feat(environment): product intent declares service-secrets generate and may-be-empty

cmd/environment/import.go

@@ -49,9 +49,25 @@ Declared fields replace their named values. Omitted fields and unrelated map
 entries are preserved, comments included. Read from stdin with
 --coordinate-contract -.
 
---product-intent <file> additionally imports environment-specific composition
-exclusions from an authored product file. It accepts only environments and
-exclude-services; it does not interpret the legacy full delivery.intent.yaml.`,
+--product-intent <file> additionally imports what the product, not the cell,
+decides per environment, from an authored product file:
+
+  environments:
+    <env>:
+      exclude-services: []          # required; [] clears
+      service-secrets:              # optional
+        generate:                   # keys minted at random by deploy secrets
+          - {scope: service, configuration: clickhouse, keys: [CLICKHOUSE_PASSWORD], services: [tracing/evidence]}
+        may-be-empty: [<stored key>] # keys whose correct value here is empty
+
+The intent owns exclude-services, service-secrets.generate and
+service-secrets.may-be-empty: a contract declaring one of them too is refused,
+and an entry the intent no longer carries is withdrawn on re-import. The
+contract keeps the rest of service-secrets (secret-store, defaults, services),
+which names the cell's store. Each entry is checked against the secret
+configurations the environment's profile declares; an unknown service,
+configuration or key is refused by name. Any other field is refused; the legacy
+full delivery.intent.yaml is not interpreted.`,
 	Args:         cobra.ExactArgs(1),
 	SilenceUsage: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -185,21 +201,6 @@ func runImport(ctx context.Context, opts *importOptions) error {
 		return err
 	}
 	declaredEnvironment := yamledit.MapValue(declaration, "environment")
-	if opts.productIntentData != nil {
-		exclusions, exclusionsErr := productExclusions(opts.productIntentData, opts.envName)
-		if exclusionsErr != nil {
-			return exclusionsErr
-		}
-		if yamledit.MapValue(declaredEnvironment, "exclude-services") != nil {
-			return fmt.Errorf("exclude-services is declared by both coordinate contract and product intent; select one owner")
-		}
-		node, encodeErr := yamledit.Encode(exclusions)
-		if encodeErr != nil {
-			return encodeErr
-		}
-		yamledit.SetMapValue(declaredEnvironment, "exclude-services", node)
-		contract.Environment.ExcludeServices = exclusions
-	}
 
 	file := filepath.Join(opts.dir, resources.WorkspaceConfigurationName)
 	original, err := os.ReadFile(file)
@@ -215,6 +216,11 @@ func runImport(ctx context.Context, opts *importOptions) error {
 	envs := yamledit.MapValue(root, "environments")
 	envNode := findExistingEnvironment(envs, opts.envName)
 
+	intent, err := applyProductIntent(opts, declaredEnvironment, envNode, contract)
+	if err != nil {
+		return err
+	}
+
 	namespace := contract.Environment.Namespace
 	if opts.namespaceSet {
 		namespace = opts.namespace
@@ -245,7 +251,10 @@ func runImport(ctx context.Context, opts *importOptions) error {
 		// trailing lines.
 		endLine := yamledit.EndLine(envNode)
 		applyContractFields(envNode, declaredEnvironment)
-		withdrawContractOwnedAbsences(envNode, declaredEnvironment)
+		withdrawOwnedAbsences(envNode, declaredEnvironment, contractOwnedOptional, false)
+		if intent != nil {
+			withdrawOwnedAbsences(envNode, declaredEnvironment, productOwnedOptional, true)
+		}
 		stampProvenance(envNode, contract, opts)
 		updated, err = spliceEnvironment(original, envNode, endLine)
 	}
@@ -260,7 +269,7 @@ func runImport(ctx context.Context, opts *importOptions) error {
 	if _, err := environments.FromWorkspace(merged); err != nil {
 		return fmt.Errorf("merged environment declarations: %w", err)
 	}
-	if opts.productIntentData != nil {
+	if intent != nil {
 		selected, err := environments.Select(merged, opts.envName)
 		if err != nil {
 			return err
@@ -272,6 +281,9 @@ func runImport(ctx context.Context, opts *importOptions) error {
 		if err := environments.ValidateEnvironmentExclusions(ctx, workspace, selected); err != nil {
 			return fmt.Errorf("product intent: %w", err)
 		}
+		if err := environments.ValidateSecretPolicy(ctx, workspace, selected, intent.ServiceSecrets); err != nil {
+			return fmt.Errorf("product intent: %w", err)
+		}
 	}
 	if err := materializeConfigurationProfile(ctx, opts.dir, env, opts.dryRun); err != nil {
 		return err
@@ -472,13 +484,21 @@ func applyContractFields(target, declaration *yaml.Node) {
 // environment no longer holds.
 var contractOwnedOptional = map[string][]string{"registry": {"mirrors", "artifacts", "auth"}}
 
-// withdrawContractOwnedAbsences removes each contract-owned optional key the contract
-// no longer carries, under a parent the contract does declare. A parent the contract
-// omits, and every operator-owned key, are left to applyContractFields' merge.
-func withdrawContractOwnedAbsences(target, declaration *yaml.Node) {
-	for parent, keys := range contractOwnedOptional {
+// productOwnedOptional names the service-secrets keys a product intent owns
+// (environments.ProductSecretPolicyKeys). Once an import reads a product intent,
+// the intent is their only source, so a key it no longer carries is withdrawn
+// even when it declares no service-secrets at all: a generator left behind would
+// keep minting, and keep a rotation admitted for, a key the product dropped.
+var productOwnedOptional = map[string][]string{"service-secrets": environments.ProductSecretPolicyKeys}
+
+// withdrawOwnedAbsences removes each owned optional key the declaration no longer
+// carries. Contract-owned keys are withdrawn only under a parent the contract
+// declares (parentOptional false); a parent it omits, and every operator-owned
+// key, are left to applyContractFields' merge.
+func withdrawOwnedAbsences(target, declaration *yaml.Node, owned map[string][]string, parentOptional bool) {
+	for parent, keys := range owned {
 		declared, current := yamledit.MapValue(declaration, parent), yamledit.MapValue(target, parent)
-		if declared == nil || current == nil || current.Kind != yaml.MappingNode {
+		if (declared == nil && !parentOptional) || current == nil || current.Kind != yaml.MappingNode {
 			continue
 		}
 		for _, key := range keys {
@@ -495,6 +515,74 @@ func withdrawContractOwnedAbsences(target, declaration *yaml.Node) {
 	}
 }
 
+// applyProductIntent adds the product intent's decisions, when one was passed,
+// to the contract declaration being merged and to the contract's environment.
+// It returns nil when the import reads no product intent.
+func applyProductIntent(opts *importOptions, declared, existing *yaml.Node, contract *environments.CoordinateContract) (*productIntent, error) {
+	if opts.productIntentData == nil {
+		return nil, nil
+	}
+	intent, err := parseProductIntent(opts.productIntentData, opts.envName)
+	if err != nil {
+		return nil, err
+	}
+	if yamledit.MapValue(declared, "exclude-services") != nil {
+		return nil, fmt.Errorf("exclude-services is declared by both coordinate contract and product intent; select one owner")
+	}
+	node, err := yamledit.Encode(intent.ExcludeServices)
+	if err != nil {
+		return nil, err
+	}
+	yamledit.SetMapValue(declared, "exclude-services", node)
+	contract.Environment.ExcludeServices = intent.ExcludeServices
+	if err := applyProductSecretPolicy(declared, existing, &contract.Environment, intent.ServiceSecrets); err != nil {
+		return nil, err
+	}
+	return intent, nil
+}
+
+// applyProductSecretPolicy adds the intent's service-secrets policy to the
+// declaration being merged, and to the contract's environment for an
+// environment the workspace does not hold yet. The contract keeps every other
+// service-secrets field; one that also declares a product-owned key is refused,
+// as exclude-services is, rather than one owner silently winning.
+func applyProductSecretPolicy(declared, existing *yaml.Node, contractEnv *environments.Environment, policy *environments.ProductSecretPolicy) error {
+	secrets := yamledit.MapValue(declared, "service-secrets")
+	for _, key := range environments.ProductSecretPolicyKeys {
+		if yamledit.MapValue(secrets, key) != nil {
+			return fmt.Errorf("service-secrets.%s is declared by both coordinate contract and product intent; select one owner", key)
+		}
+	}
+	if policy.Empty() {
+		return nil
+	}
+	if contractEnv.ServiceSecrets == nil && yamledit.MapValue(existing, "service-secrets") == nil {
+		return fmt.Errorf("product intent declares service-secrets for environment %q, but neither the coordinate contract nor the workspace declares its service-secrets secret-store", contractEnv.Name)
+	}
+	if contractEnv.ServiceSecrets != nil {
+		contractEnv.ServiceSecrets.Generate, contractEnv.ServiceSecrets.MayBeEmpty = policy.Generate, policy.MayBeEmpty
+	}
+	if secrets == nil {
+		secrets = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
+		yamledit.SetMapValue(declared, "service-secrets", secrets)
+	}
+	for _, field := range []struct {
+		key   string
+		value any
+		set   bool
+	}{{"generate", policy.Generate, len(policy.Generate) > 0}, {"may-be-empty", policy.MayBeEmpty, len(policy.MayBeEmpty) > 0}} {
+		if !field.set {
+			continue
+		}
+		node, err := yamledit.Encode(field.value)
+		if err != nil {
+			return err
+		}
+		yamledit.SetMapValue(secrets, field.key, node)
+	}
+	return nil
+}
+
 // stampProvenance writes (or, on re-import, replaces) the provenance comment
 // above the environment item, keeping any operator comment lines around it.
 func stampProvenance(envNode *yaml.Node, contract *environments.CoordinateContract, opts *importOptions) {
@@ -519,7 +607,7 @@ func stampProvenance(envNode *yaml.Node, contract *environments.CoordinateContra
 
 func init() {
 	importCmd.Flags().String("coordinate-contract", "", "Path to a codefly/coordinate/v1 descriptor, or - for stdin")
-	importCmd.Flags().String("product-intent", "", "Authored environment exclusions; accepts environments.<env>.exclude-services only")
+	importCmd.Flags().String("product-intent", "", "Authored product decisions: environments.<env>.exclude-services and service-secrets generate / may-be-empty")
 	importCmd.Flags().String("cell-contract", "", "Former spelling of --coordinate-contract")
 	_ = importCmd.Flags().MarkDeprecated("cell-contract", "use --coordinate-contract")
 	importCmd.Flags().StringVar(&importNamespace, "namespace", "", "Assert the namespace declared by the producer")

cmd/environment/product_intent.go

@@ -9,13 +9,23 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-// productExclusions reads only the product-owned composition decision. Cloud
-// placement and runtime bindings remain in the coordinate producer's contract.
-// This deliberately does not interpret the historical full launch intent.
-func productExclusions(data []byte, name string) ([]environments.EnvironmentExcludedService, error) {
+// productIntent is what an authored product file decides for one environment:
+// its composition exclusions and the product-owned part of its service-secrets
+// (environments.ProductSecretPolicy). Cloud placement, runtime bindings and the
+// cell's secret store remain in the coordinate producer's contract. This
+// deliberately does not interpret the historical full launch intent.
+type productIntent struct {
+	ExcludeServices []environments.EnvironmentExcludedService
+	// ServiceSecrets is nil when the intent declares none for the environment,
+	// which withdraws every product-owned service-secrets entry on import.
+	ServiceSecrets *environments.ProductSecretPolicy
+}
+
+func parseProductIntent(data []byte, name string) (*productIntent, error) {
 	var intent struct {
 		Environments map[string]struct {
 			ExcludeServices *[]environments.EnvironmentExcludedService `yaml:"exclude-services"`
+			ServiceSecrets  *environments.ProductSecretPolicy          `yaml:"service-secrets"`
 		} `yaml:"environments"`
 	}
 	decoder := yaml.NewDecoder(bytes.NewReader(data))
@@ -35,10 +45,13 @@ func productExclusions(data []byte, name string) ([]environments.EnvironmentExcl
 		if err := env.Validate(); err != nil {
 			return nil, fmt.Errorf("product intent environment %q: %w", environment, err)
 		}
+		if err := selected.ServiceSecrets.Validate(); err != nil {
+			return nil, fmt.Errorf("product intent environment %q: %w", environment, err)
+		}
 	}
 	selected, ok := intent.Environments[name]
 	if !ok {
 		return nil, fmt.Errorf("product intent does not declare environment %q", name)
 	}
-	return *selected.ExcludeServices, nil
+	return &productIntent{ExcludeServices: *selected.ExcludeServices, ServiceSecrets: selected.ServiceSecrets}, nil
 }

cmd/environment/product_intent_test.go

@@ -111,3 +111,106 @@ func TestProductIntentFlagDryRun(t *testing.T) {
 	require.NoError(t, executeImport(t, dir, "--product-intent", path, "--dry-run"))
 	require.Equal(t, before, readFile(t, file))
 }
+
+const productSecretsIntent = `environments:
+  azure:
+    exclude-services: []
+    service-secrets:
+      generate:
+        - {scope: service, configuration: clickhouse, keys: [CLICKHOUSE_PASSWORD], services: [shop/api]}
+      may-be-empty: [CODEFLY__SERVICE_SECRET_CONFIGURATION__SHOP__API__CLICKHOUSE__SENTRY_DSN]
+`
+
+// productSecretsWorkspace is productImportWorkspace whose shop/api declares a
+// secret clickhouse configuration for the azure profile.
+func productSecretsWorkspace(t *testing.T) string {
+	t.Helper()
+	dir := productImportWorkspace(t)
+	path := filepath.Join(dir, "modules/shop/services/api/configurations/azure/clickhouse.secret.env")
+	require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755))
+	require.NoError(t, os.WriteFile(path, []byte("CLICKHOUSE_PASSWORD=\nSENTRY_DSN=\n"), 0o600))
+	return dir
+}
+
+func TestProductIntentImportsAndWithdrawsSecretPolicy(t *testing.T) {
+	dir := productSecretsWorkspace(t)
+	contract := strings.Replace(fixtureContract,
+		`"service-secrets": {"secret-store": {"name": "cell-secrets", "kind": "ClusterSecretStore"}},`,
+		`"service-secrets": {"secret-store": {"name": "cell-secrets", "kind": "ClusterSecretStore"}, "services": {"shop/api": {"remote-keys": {"K": "api/k"}}}},`, 1)
+	doImport(t, dir, importOptions{contractData: []byte(contract), productIntentData: []byte(productSecretsIntent)})
+	secrets := loadWorkspace(t, dir).FindEnvironment("azure").ServiceSecrets
+	require.Equal(t, "cell-secrets", secrets.SecretStore.Name)
+	require.Equal(t, "api/k", secrets.Services["shop/api"].RemoteKeys["K"].Key)
+	require.Len(t, secrets.Generate, 1)
+	require.Equal(t, []string{"CLICKHOUSE_PASSWORD"}, secrets.Generate[0].Keys)
+	require.Equal(t, []string{"shop/api"}, secrets.Generate[0].Services)
+	require.Equal(t, []string{"CODEFLY__SERVICE_SECRET_CONFIGURATION__SHOP__API__CLICKHOUSE__SENTRY_DSN"}, secrets.MayBeEmpty)
+
+	// A producer-only refresh keeps the product's policy.
+	doImport(t, dir, importOptions{contractData: []byte(contract)})
+	require.Len(t, loadWorkspace(t, dir).FindEnvironment("azure").ServiceSecrets.Generate, 1)
+
+	// An intent that drops may-be-empty withdraws it and keeps generate.
+	withoutEmpty := strings.Replace(productSecretsIntent, "      may-be-empty: [CODEFLY__SERVICE_SECRET_CONFIGURATION__SHOP__API__CLICKHOUSE__SENTRY_DSN]\n", "", 1)
+	doImport(t, dir, importOptions{contractData: []byte(contract), productIntentData: []byte(withoutEmpty)})
+	secrets = loadWorkspace(t, dir).FindEnvironment("azure").ServiceSecrets
+	require.Empty(t, secrets.MayBeEmpty)
+	require.Len(t, secrets.Generate, 1)
+
+	// An intent with no service-secrets withdraws the rest; the contract's
+	// fields stay.
+	doImport(t, dir, importOptions{contractData: []byte(contract), productIntentData: []byte("environments:\n  azure:\n    exclude-services: []\n")})
+	secrets = loadWorkspace(t, dir).FindEnvironment("azure").ServiceSecrets
+	require.Empty(t, secrets.Generate)
+	require.Empty(t, secrets.MayBeEmpty)
+	require.Equal(t, "cell-secrets", secrets.SecretStore.Name)
+	require.Equal(t, "api/k", secrets.Services["shop/api"].RemoteKeys["K"].Key)
+}
+
+func TestProductIntentSecretPolicyRefusalsDoNotWrite(t *testing.T) {
+	for name, tc := range map[string]struct{ intent, contract, want string }{
+		"unknown service": {intent: strings.Replace(productSecretsIntent, "services: [shop/api]", "services: [shop/missing]", 1),
+			want: `"shop/missing", which the workspace does not compose`},
+		"undeclared configuration": {intent: strings.Replace(productSecretsIntent, "configuration: clickhouse", "configuration: postgres", 1),
+			want: `service shop/api declares no secret configuration "postgres"`},
+		"undeclared key": {intent: strings.Replace(productSecretsIntent, "keys: [CLICKHOUSE_PASSWORD]", "keys: [CLICKHOUSE_USER]", 1),
+			want: `declares no key "CLICKHOUSE_USER"`},
+		"workspace group": {intent: strings.Replace(productSecretsIntent, "{scope: service, configuration: clickhouse, keys: [CLICKHOUSE_PASSWORD], services: [shop/api]}", "{scope: workspace, configuration: clickhouse, keys: [CLICKHOUSE_PASSWORD]}", 1),
+			want: `the workspace declares no secret configuration "clickhouse"`},
+		"undeclared may-be-empty": {intent: strings.Replace(productSecretsIntent, "SENTRY_DSN]", "OTHER]", 1),
+			want: `may-be-empty names "CODEFLY__SERVICE_SECRET_CONFIGURATION__SHOP__API__CLICKHOUSE__OTHER"`},
+		"unknown generator field": {intent: strings.Replace(productSecretsIntent, "keys:", "kyes:", 1), want: "kyes"},
+		"unknown policy field":    {intent: strings.Replace(productSecretsIntent, "      generate:", "      secret-store: {name: x, kind: y}\n      generate:", 1), want: "secret-store"},
+		"invalid scope":           {intent: strings.Replace(productSecretsIntent, "scope: service", "scope: module", 1), want: "scope must be"},
+		"dual ownership": {intent: productSecretsIntent,
+			contract: strings.Replace(fixtureContract, `"kind": "ClusterSecretStore"}}`, `"kind": "ClusterSecretStore"}, "may-be-empty": ["X"]}`, 1),
+			want:     "service-secrets.may-be-empty is declared by both coordinate contract and product intent"},
+		"no secret store": {intent: productSecretsIntent,
+			contract: strings.Replace(fixtureContract, `"service-secrets": {"secret-store": {"name": "cell-secrets", "kind": "ClusterSecretStore"}},`, "", 1),
+			want:     "declares its service-secrets secret-store"},
+	} {
+		t.Run(name, func(t *testing.T) {
+			dir := productSecretsWorkspace(t)
+			file := filepath.Join(dir, resources.WorkspaceConfigurationName)
+			before := readFile(t, file)
+			contract := tc.contract
+			if contract == "" {
+				contract = fixtureContract
+			}
+			err := runImport(context.Background(), &importOptions{dir: dir, envName: "azure", contractData: []byte(contract), productIntentData: []byte(tc.intent), stdout: io.Discard})
+			require.ErrorContains(t, err, tc.want)
+			require.Equal(t, before, readFile(t, file))
+		})
+	}
+}
+
+func TestProductIntentGeneratorWithoutServicesNeedsADeclaringService(t *testing.T) {
+	dir := productSecretsWorkspace(t)
+	intent := strings.Replace(productSecretsIntent, ", services: [shop/api]", "", 1)
+	doImport(t, dir, importOptions{productIntentData: []byte(intent)})
+	require.Empty(t, loadWorkspace(t, dir).FindEnvironment("azure").ServiceSecrets.Generate[0].Services)
+
+	missing := strings.Replace(intent, "configuration: clickhouse", "configuration: postgres", 1)
+	err := runImport(t.Context(), &importOptions{dir: dir, envName: "azure", contractData: []byte(fixtureContract), productIntentData: []byte(missing), stdout: io.Discard})
+	require.ErrorContains(t, err, `no composed service declares secret configuration "postgres"`)
+}