feat(deploy): deploy secrets --rotate rotates a generated key and every stored value derived from it

cmd/deploy/secrets.go

@@ -27,6 +27,7 @@ var (
 	secretsAllowMissing bool
 	secretsYes          bool
 	secretsModules      []string
+	secretsRotate       []string
 )
 
 // SecretsCmd seeds an environment's secret store with what its render reads.
@@ -43,9 +44,9 @@ operator, and named). The store is the backend behind the SecretStore the render
 names, read from the environment's cluster; a render reading through several
 stores is planned and written store by store.
 
-A stored value is never rotated: a property the store already holds is kept even
-when a configuration file holds one too, and the two disagreeing is refused
-rather than resolved. A configured key with an empty value supplies nothing, so
+A stored value is never rotated unless --rotate names it: a property the store
+already holds is kept even when a configuration file holds one too, and the two
+disagreeing is refused rather than resolved. A configured key with an empty value supplies nothing, so
 a tracked placeholder file still leaves its properties to be supplied.
 
 Each property is resolved by the secret key the render reads out of it, not by
@@ -60,7 +61,19 @@ reads a stored value: it knows which remote keys exist, not what they hold.
 
 --module limits the plan to the remote keys the named modules' services read.
 Every other remote key is still read, so a scoped key keeps agreeing with what
-the store already holds, but nothing outside the scope is planned or written.`,
+the store already holds, but nothing outside the scope is planned or written.
+
+--rotate <module>/<service>/<configuration>:<KEY> (or <configuration>:<KEY> for a
+workspace configuration, or the CODEFLY__… secret key) replaces a GENERATED key's
+stored value with a new one, in every remote key holding it, and rewrites every
+stored value derived from it in the same apply: a stored URL whose password is
+the old value gets the new one. Any other stored value embedding the old one,
+and any key service-secrets.generate does not declare, is refused by name. Each
+write is a new version, refused if the key moved since the plan read it; a
+rotation failing part-way restores what it wrote. The CLI changes the store
+only: force-sync the ExternalSecrets, make the server accept the new credential
+(a restart only does that for an agent that re-applies it on every start), then
+restart its consumers. An identifier or signing key is never rotated.`,
 	Args: cobra.NoArgs,
 	RunE: func(cmd *cobra.Command, _ []string) error {
 		ctx, done := common.NewContext()
@@ -92,6 +105,11 @@ the store already holds, but nothing outside the scope is planned or written.`,
 				return fmt.Errorf("--module %s: not a module rendered for %s (rendered: %s)", module, env.Name, strings.Join(rendered.Modules, ", "))
 			}
 		}
+		groups := renderedByStore(rendered)
+		rotate, err := rotationKeys(rendered, groups, secretsRotate)
+		if err != nil {
+			return err
+		}
 		// The operator's own configuration files, read once for the whole
 		// environment: they are a property of the workspace, not of a store, so
 		// every per-store plan below resolves against the same index.
@@ -103,7 +121,7 @@ the store already holds, but nothing outside the scope is planned or written.`,
 		// per-key store such as an identity provider's) is planned and written
 		// store by store: one plan writes one store, and each is confirmed on
 		// its own.
-		for _, group := range renderedByStore(rendered) {
+		for _, group := range groups {
 			store, err := resolveSecretStore(ctx, env, group)
 			if err != nil {
 				return err
@@ -117,6 +135,7 @@ the store already holds, but nothing outside the scope is planned or written.`,
 				Modules:      secretsModules,
 				MayBeEmpty:   env.ServiceSecrets.MayBeEmpty,
 				Configured:   configured,
+				Rotate:       readBy(group, rotate),
 			})
 			if err != nil {
 				return err
@@ -176,12 +195,48 @@ func finishSecretsPlan(ctx context.Context, plan *deploysecrets.Plan, store depl
 		return fmt.Errorf("write not confirmed")
 	}
 	written, err := plan.Apply(ctx, store, allowMissing)
-	for _, key := range written {
-		cli.Info("Wrote %s", key)
+	for _, write := range written {
+		verb := "Wrote"
+		if write.Restored {
+			verb = "Restored the previous document of"
+		}
+		cli.Info("%s %s as version %s (%s)", verb, write.Key, write.Version, strings.Join(write.Properties, ", "))
 	}
 	return err
 }
 
+// rotationKeys resolves every --rotate selector, refusing one no rendered
+// ExternalSecret reads or whose configuration spans stores.
+func rotationKeys(rendered gitops.RenderedEnvironment, groups []gitops.RenderedEnvironment, selectors []string) ([]string, error) {
+	keys := make([]string, 0, len(selectors))
+	for _, selector := range selectors {
+		key, err := deploysecrets.RotationKey(selector)
+		if err != nil {
+			return nil, err
+		}
+		if !deploysecrets.ReadsKey(rendered, key) {
+			return nil, fmt.Errorf("--rotate %s: no rendered ExternalSecret reads %s", selector, key)
+		}
+		if err = deploysecrets.CheckRotationStores(groups, key); err != nil {
+			return nil, err
+		}
+		keys = append(keys, key)
+	}
+	return keys, nil
+}
+
+// readBy keeps the rotated keys one store group reads; each is rotated by the
+// plan of the store holding it.
+func readBy(group gitops.RenderedEnvironment, keys []string) []string {
+	var read []string
+	for _, key := range keys {
+		if deploysecrets.ReadsKey(group, key) {
+			read = append(read, key)
+		}
+	}
+	return read
+}
+
 // resolveSecretStore resolves the one store every ExternalSecret of a
 // renderedByStore group reads through, from the environment's cluster. A mixed
 // group is a programming error and is refused by name.
@@ -271,4 +326,5 @@ func init() {
 	SecretsCmd.Flags().BoolVar(&secretsAllowMissing, "allow-missing", false, "Write what can be resolved even when some properties must still be supplied")
 	SecretsCmd.Flags().BoolVarP(&secretsYes, "yes", "y", false, "Write without an interactive confirmation")
 	SecretsCmd.Flags().StringSliceVar(&secretsModules, "module", nil, "Plan only these modules' remote keys (comma-separated or repeated)")
+	SecretsCmd.Flags().StringSliceVar(&secretsRotate, "rotate", nil, "Replace a generated key's stored value and every stored value derived from it: <module>/<service>/<configuration>:<KEY>, <configuration>:<KEY>, or the CODEFLY__ key (repeatable)")
 }

cmd/deploy/secrets_test.go

@@ -125,9 +125,9 @@ func (store *outputSafetyStore) Describe(context.Context, string) (deploysecrets
 func (store *outputSafetyStore) Read(context.Context, string) (map[string]string, error) {
 	return store.document, nil
 }
-func (store *outputSafetyStore) Write(context.Context, string, map[string]string, bool) error {
+func (store *outputSafetyStore) Write(context.Context, string, map[string]string, bool) (string, error) {
 	store.writes++
-	return nil
+	return "1", nil
 }
 
 func TestSecretsCommandRefusesMissingNoOpBeforeConfirmation(t *testing.T) {
@@ -152,3 +152,22 @@ func TestSecretsCommandRefusesMissingNoOpBeforeConfirmation(t *testing.T) {
 		t.Fatal("explicit allow-missing should permit a no-op")
 	}
 }
+
+// A --rotate selector resolves to the key a rendered ExternalSecret reads, is
+// refused when none reads it, and reaches only the store group reading it.
+func TestRotationKeysResolveAgainstTheRender(t *testing.T) {
+	const key = "CODEFLY__SERVICE_SECRET_CONFIGURATION__TRACING__EVIDENCE__CLICKHOUSE__PASSWORD"
+	rendered := gitops.RenderedEnvironment{Secrets: []gitops.RenderedServiceSecret{{RemoteKey: "evidence", Services: []string{"tracing/evidence"},
+		Properties: []gitops.RenderedSecretProperty{{Property: key, Keys: []string{key}}}}}}
+	groups := renderedByStore(rendered)
+	keys, err := rotationKeys(rendered, groups, []string{"tracing/evidence/clickhouse:PASSWORD"})
+	if err != nil || len(keys) != 1 || keys[0] != key {
+		t.Fatalf("rotationKeys = %v, %v; want %s", keys, err, key)
+	}
+	if got := readBy(gitops.RenderedEnvironment{}, keys); len(got) != 0 {
+		t.Errorf("a group reading nothing was handed %v", got)
+	}
+	if _, err = rotationKeys(rendered, groups, []string{"tracing/evidence/clickhouse:USER"}); err == nil || !strings.Contains(err.Error(), "no rendered ExternalSecret reads") {
+		t.Errorf("an unread key = %v, want a refusal", err)
+	}
+}

docs/cli-reference.md

@@ -2295,9 +2295,9 @@ operator, and named). The store is the backend behind the SecretStore the render
 names, read from the environment's cluster; a render reading through several
 stores is planned and written store by store.
 
-A stored value is never rotated: a property the store already holds is kept even
-when a configuration file holds one too, and the two disagreeing is refused
-rather than resolved. A configured key with an empty value supplies nothing, so
+A stored value is never rotated unless --rotate names it: a property the store
+already holds is kept even when a configuration file holds one too, and the two
+disagreeing is refused rather than resolved. A configured key with an empty value supplies nothing, so
 a tracked placeholder file still leaves its properties to be supplied.
 
 Each property is resolved by the secret key the render reads out of it, not by
@@ -2313,6 +2313,18 @@ reads a stored value: it knows which remote keys exist, not what they hold.
 --module limits the plan to the remote keys the named modules' services read.
 Every other remote key is still read, so a scoped key keeps agreeing with what
 the store already holds, but nothing outside the scope is planned or written.
+
+--rotate <module>/<service>/<configuration>:<KEY> (or <configuration>:<KEY> for a
+workspace configuration, or the CODEFLY__… secret key) replaces a GENERATED key's
+stored value with a new one, in every remote key holding it, and rewrites every
+stored value derived from it in the same apply: a stored URL whose password is
+the old value gets the new one. Any other stored value embedding the old one,
+and any key service-secrets.generate does not declare, is refused by name. Each
+write is a new version, refused if the key moved since the plan read it; a
+rotation failing part-way restores what it wrote. The CLI changes the store
+only: force-sync the ExternalSecrets, make the server accept the new credential
+(a restart only does that for an agent that re-applies it on every start), then
+restart its consumers. An identifier or signing key is never rotated.
 ```
 
 ```
@@ -2327,6 +2339,7 @@ Flags:
       --env string       Environment whose rendered ExternalSecrets to seed (default "local")
       --metadata-only    With --dry-run: never read a stored value, only which remote keys exist
       --module strings   Plan only these modules' remote keys (comma-separated or repeated)
+      --rotate strings   Replace a generated key's stored value and every stored value derived from it: <module>/<service>/<configuration>:<KEY>, <configuration>:<KEY>, or the CODEFLY__ key (repeatable)
   -y, --yes              Write without an interactive confirmation
 ```
 

docs/commands.md

@@ -1214,6 +1214,7 @@ codefly deploy secrets --env staging --dry-run --metadata-only  # which keys exi
 codefly deploy secrets --env staging --dry-run                  # full plan; reads the store in memory
 codefly deploy secrets --env staging                            # write it (confirms; --yes to skip)
 codefly deploy secrets --env staging --module runtime --dry-run  # one module's keys only
+codefly deploy secrets --env staging --rotate tracing/evidence/clickhouse:CLICKHOUSE_PASSWORD  # rotate a generated key
 ```
 
 It reads every `ExternalSecret` the render projected for `--env` under
@@ -1224,11 +1225,12 @@ resolves every remote property to a source, never printing a value:
 
 | action | source |
 | --- | --- |
-| `keep` | the store already holds it; nothing stored is ever rotated |
+| `keep` | the store already holds it; nothing stored is rotated unless `--rotate` names it |
 | `propagate` | a configuration value (`CODEFLY__WORKSPACE_SECRET_CONFIGURATION__…`, `CODEFLY__SERVICE_SECRET_CONFIGURATION__…`) is one value however many services read it, so it is copied from the remote key that holds it |
 | `configured` | the environment's own `configurations/<profile>/<group>.secret.env` holds the value — the file `codefly config set` writes — and the plan seeds the store from it, naming the file as its source |
 | `generate` | declared random by `service-secrets.generate` (below) |
 | `require` | nothing produces it: the operator supplies it — an external credential, or a value its producing agent derives |
+| `rotate` | `--rotate` names its generated key, or its stored value is derived from one that is ([below](#rotating-a-generated-key)) |
 
 **`configured`: the environment's own files are a source.** `codefly config set
 <group> KEY=VALUE --env <env>` writes
@@ -1251,8 +1253,9 @@ key reaches its value whichever way the group or key was spelled.
 
 Four rules bound it:
 
-- **A stored value still wins and is never rotated.** A property the store
-  already holds is `keep`, file or no file.
+- **A stored value still wins and is never rotated** (bar an explicit
+  `--rotate`, which refuses a configured key). A property the store already
+  holds is `keep`, file or no file.
 - **A stored value that differs from the configured one is refused**, like two
   remote keys disagreeing: keeping it would leave the deployed workload on one
   value and every local run on the other, reported as `keep`. Change the
@@ -1356,6 +1359,83 @@ service already signs with is never replaced by a second run. It is one
 workload's key: a plan in which more than one service reads it is refused, and
 the remedy is to remove the extra reader, never to re-mint the key.
 
+#### Rotating a generated key
+
+`--rotate` replaces a stored value. It is explicit and narrow:
+
+```bash
+codefly deploy secrets --env blue --rotate tracing/evidence/clickhouse:CLICKHOUSE_PASSWORD --dry-run
+codefly deploy secrets --env blue --rotate tracing/evidence/clickhouse:CLICKHOUSE_PASSWORD
+```
+
+The selector is spelled like the `service-secrets.generate` entry that declares
+the key — `<module>/<service>/<configuration>:<KEY>`, `<configuration>:<KEY>`
+for a workspace configuration — or is the `CODEFLY__…_SECRET_CONFIGURATION__…`
+key itself; it is repeatable. Only a key `service-secrets.generate` declares is
+rotated. Each of these is refused with its reason: a supplied key (`require`); a
+configured one (change the file instead); an `ed25519-pkcs8` signing key (its
+public half is registered elsewhere); an `identifier` key (a new user or
+database name in the store renames nothing on the server); and a key no rendered
+ExternalSecret reads.
+
+The new value is minted as the generator would mint it and written to **every**
+remote key holding the old one; a holder outside `--module` is refused rather
+than left behind. Then every **derived** value moves in the same apply. What is
+derived is read from the store, not guessed from a name: every stored property
+of the store is examined, and one that parses as a URL whose **decoded** userinfo
+password is the old value — the connection an agent exports without a template,
+stored by hand because the plan reported it `require` — has that password
+replaced in its userinfo and every other byte kept. Any other stored value
+holding the old one, raw or percent-encoded, is refused, naming
+`remote-key#property`, because codefly cannot recompute it and a stale derived
+value is the failure rotation must not leave. A connection its producer declares
+as a template (below) is not stored at all: External Secrets re-assembles it from
+the rotated primitive on its next sync. A configuration read through more than
+one store is refused: a plan sees one store. **What is not seen:** a copy of the
+value in another store, or one that no rendered ExternalSecret reads (a value
+outside codefly's render), is not examined; find those before rotating.
+
+`--dry-run` shows `rotate` rows by property name, no value; with
+`--metadata-only` it shows the rotated key's rows only, since derived values are
+found by reading the store. Each write adds a **new version** (nothing is
+destroyed) and the result lists each remote key, the properties it set and the
+version the store assigned. Before each write the key's newest version is
+compared with the one the plan read, and a key that moved is refused by name
+(the store has no conditional write, so this narrows the race; it does not close
+it). A rotation failing part-way writes the previous document of every key it
+wrote back as a new version — including the failing key when its write landed
+anyway, and never a key someone else wrote since — on a context of its own, so
+the source and its derived values still agree and the same command can be run
+again. If a restore fails too, the error names the keys that may still hold the
+rotated document. After success a plain `deploy secrets` re-run writes nothing;
+running `--rotate` again rotates again.
+
+**The CLI changes the store only — not the server's credential.** Whether a new
+value in the store changes the credential the server accepts is the producing
+agent's behaviour, and the CLI cannot tell which an agent does:
+
+- An agent that **re-applies the credential from its environment on every
+  start** follows a restart. The ClickHouse agent is one: its entrypoint
+  rewrites `users.d` from `CLICKHOUSE_USER`/`CLICKHOUSE_PASSWORD` on every start
+  (service-clickhouse `main.go`).
+- An agent whose image applies it **only when the data directory is first
+  created** (the official Postgres image, for one) does not: after a restart the
+  store and every consumer URL carry the new password while the database user
+  keeps the old one. Change the credential on the server yourself (for Postgres,
+  `ALTER ROLE … PASSWORD …` with the new value) before restarting consumers, or
+  do not rotate that key here.
+
+Then, in order:
+
+1. force-sync the affected ExternalSecrets (`kubectl annotate externalsecret <name> force-sync=$(date +%s) --overwrite`, or wait for their refresh interval);
+2. restart (or re-credential, above) the server that reads the rotated value;
+3. restart its consumers, which read the derived connection.
+
+**Undo.** Every earlier version is kept. To go back, make each written key's
+previous version current again by adding it as a new version, without printing
+it — `gcloud secrets versions access <previous> --secret <key> --project <project> | gcloud secrets versions add <key> --project <project> --data-file=-`
+for each key the result listed — then repeat the three steps above.
+
 A key its producer declares as a template (core's `ConfigurationValue.template`
 — a Postgres agent's `READ_WRITE_CONNECTION`, say: role, address and database
 around a reference to its read-write password) is never read from the store.

docs/deployment-secret-safety.md

@@ -22,7 +22,15 @@ Older rendered ExternalSecrets already contain the `secretKey` and `remoteRef`
 needed for resolution. There is no render-inventory migration.
 
 Valid stored values remain unchanged. A seed operation does not rotate them or
-prove that a manually seeded value authenticates to its intended service.
+prove that a manually seeded value authenticates to its intended service. Only
+`--rotate` replaces a stored value, and only a generated one: it rewrites a
+stored URL whose decoded password is the old value and refuses any other stored
+value holding it, raw or percent-encoded, so no derived value in the store is
+left on the old one. It does not change the credential the server accepts, and
+it does not see a copy outside the store it plans. Its output names keys,
+properties and store versions; the old and new values are never printed, and a
+derived URL that fails to parse is refused without echoing it
+([commands](commands.md#rotating-a-generated-key)).
 Configuration aliases must agree across every referenced property, including
 properties in one document. Aliases use one generated value when none exists,
 or reuse the consistent value already held anywhere in that alias group.