From 8dc8c1c6540573555440725a2396105b5a549bd1 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint Date: Sat, 10 Oct 2026 17:21:49 -0400 Subject: [PATCH] fix(deploy): the durable server provisions from its own command, not a postStart hook A cell's execution admission refuses any container with an exec probe or hook, so a durable (restricted) Vault could not start on staging. The container command, provision/server.sh, now starts the server, runs the unchanged provision/provision.sh (durable mode, script and key name as $1 and $2) against it, and waits on it; a provisioning failure stops the server and exits non-zero so the container restarts and provisions again, as a failed hook did. A termination trap waits for the server to shut down. The local dev render keeps its postStart hook (no admission there). The Docker harness provisions through the container's own command: it waits for 'vault: provisioned' or the container's exit; idempotence is re-checked by running the script again in the live server; refusing to initialize over an init record is checked by the record being untouched. Mutation-checked: a provisioning failure that leaves the server running fails the tests. Co-Authored-By: Claude Opus 5.5 --- builder.go | 15 +++-- durable_test.go | 62 +++++++++++++++---- provision.go | 20 +++--- provision/server.sh | 31 +++++++++- provision_test.go | 10 ++- .../kustomize/base/stateful-set.yaml.tmpl | 17 ++--- 6 files changed, 107 insertions(+), 48 deletions(-) diff --git a/builder.go b/builder.go index 87cd649..c1a23df 100644 --- a/builder.go +++ b/builder.go @@ -38,7 +38,7 @@ type deploymentTemplateParameters struct { // the Service publishes it and folds it onto 8200, the port the container // listens on. Zero leaves the template on 8200. ServicePort uint32 - // ProvisionCommand is the vault container's postStart exec command, as a + // ProvisionCommand is the local dev render's postStart exec command, as a // JSON array (a valid YAML flow sequence): provision/provision.sh run with // its mode and the configured transit key name. Empty renders no hook. ProvisionCommand string @@ -146,20 +146,19 @@ func (s *Builder) Deploy(ctx context.Context, req *builderv0.DeploymentRequest) parameters.External = true case restricted: parameters.Durable = true - if parameters.ServerCommand, err = serverCommand(); err != nil { - return s.Builder.DeployError(err) - } } if !parameters.External { key, err := s.transitKeyName() if err != nil { return s.Builder.DeployError(err) } - mode := provisionModeDev if parameters.Durable { - mode = provisionModeDurable - } - if parameters.ProvisionCommand, err = provisionCommand(mode, key); err != nil { + // The durable server provisions itself from its own command: a + // cell's execution admission refuses an exec hook. + if parameters.ServerCommand, err = serverCommand(key); err != nil { + return s.Builder.DeployError(err) + } + } else if parameters.ProvisionCommand, err = provisionCommand(provisionModeDev, key); err != nil { return s.Builder.DeployError(err) } } diff --git a/durable_test.go b/durable_test.go index 800b948..1a45ce2 100644 --- a/durable_test.go +++ b/durable_test.go @@ -73,8 +73,8 @@ func TestDurableProvisioningRefusesBadInputs(t *testing.T) { } // durableHarness runs the pinned image exactly as the restricted render does: -// the rendered container command on a read-only root filesystem with a -// persistent data volume, and the rendered postStart hook executed inside it. +// the rendered container command — which starts the server and provisions it — +// on a read-only root filesystem with a persistent data volume. // The seal is a transit seal served by a second, throwaway Vault — the local // stand-in for the cloud KMS an environment supplies (gcpckms on GCP), passed // the way the environment passes it: as environment variables. @@ -84,7 +84,6 @@ type durableHarness struct { suffix string network string server []string - hook []string } func newDurableHarness(t *testing.T) *durableHarness { @@ -93,7 +92,7 @@ func newDurableHarness(t *testing.T) *durableHarness { require.Equal(t, builderv0.DeploymentStatus_SUCCESS, response.GetState().GetState(), response.GetState().GetMessage()) container := renderedVaultContainer(t, destination) require.NotEmpty(t, container.Command) - require.NotEmpty(t, container.Lifecycle.PostStart.Exec.Command) + require.Empty(t, container.Lifecycle.PostStart.Exec.Command, "the server's own command provisions; no exec hook") ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute) t.Cleanup(cancel) @@ -102,7 +101,7 @@ func newDurableHarness(t *testing.T) *durableHarness { require.NoError(t, err) h := &durableHarness{ t: t, ctx: ctx, suffix: hex.EncodeToString(suffix), - server: container.Command, hook: container.Lifecycle.PostStart.Exec.Command, + server: container.Command, } h.network = "vault-durable-" + h.suffix h.docker("network", "create", h.network) @@ -188,9 +187,46 @@ func (h *durableHarness) start(volume, accessToken string, extraEnv ...string) ( return name, "http://" + h.docker("port", name, "8200/tcp") } -// provision runs the rendered hook in the container, as the kubelet does. +// provision waits for the container's own command to finish provisioning the +// server it started: success is its "vault: provisioned" line with the server +// still running; failure is the container exiting. It returns the provisioning +// lines of the container's log, which is what a hook's output used to be. func (h *durableHarness) provision(container string) (string, error) { - return h.try(append([]string{"exec", container}, h.hook...)...) + h.t.Helper() + for { + logs, _ := h.try("logs", container) + state := strings.TrimSpace(h.docker("inspect", "-f", "{{.State.Running}} {{.State.ExitCode}}", container)) + lines := provisioningLines(logs) + switch { + case strings.Contains(logs, "vault: provisioned"): + return lines, nil + case strings.HasPrefix(state, "false"): + return lines, fmt.Errorf("the container exited before provisioning (state %s)", state) + } + select { + case <-h.ctx.Done(): + return lines, h.ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } +} + +// reprovision runs the provisioning script again against the running server, +// as the next start of the process would: it must find nothing to do. +func (h *durableHarness) reprovision(container string) (string, error) { + return h.try("exec", container, "/bin/sh", "-c", provisionScript, "vault-provision", "durable", "api-keys") +} + +// provisioningLines keeps the lines the provisioning script and the server +// script print about provisioning, leaving out the server's own log. +func provisioningLines(logs string) string { + var kept []string + for _, line := range strings.Split(logs, "\n") { + if strings.Contains(line, "vault provisioning:") || strings.HasPrefix(line, "vault: ") { + kept = append(kept, line) + } + } + return strings.Join(kept, "\n") } func (h *durableHarness) stop(container string) { @@ -269,7 +305,7 @@ func TestDurableVaultSurvivesRestart(t *testing.T) { // Twice more on the same process: nothing to do. for range 2 { - out, err = h.provision(first) + out, err = h.reprovision(first) require.NoError(t, err, out) require.Equal(t, "vault provisioning: transit engine and key \"api-keys\" ready", strings.TrimSpace(out)) } @@ -279,7 +315,7 @@ func TestDurableVaultSurvivesRestart(t *testing.T) { second, address := h.start(volume, token) out, err = h.provision(second) require.NoError(t, err, out) - require.Equal(t, "vault provisioning: transit engine and key \"api-keys\" ready", strings.TrimSpace(out), + require.Equal(t, "vault provisioning: transit engine and key \"api-keys\" ready\nvault: provisioned", strings.TrimSpace(out), "a restarted Vault unseals itself and needs no initialization, token or key") status, decrypted := h.call(address, token, http.MethodPost, "/v1/transit/decrypt/api-keys", fmt.Sprintf(`{"ciphertext":%q}`, encrypted["ciphertext"])) @@ -316,8 +352,12 @@ func TestDurableVaultRefusesToInitializeOverAnInitRecord(t *testing.T) { out, err := h.provision(container) require.Error(t, err) require.Contains(t, out, "vault storage is uninitialized but /vault/data/bootstrap/init.json exists") - raw := h.docker("exec", container, "wget", "-qO-", "http://127.0.0.1:8200/v1/sys/init") - require.JSONEq(t, `{"initialized":false}`, raw, "the server was left uninitialized") + require.Contains(t, out, "vault: provisioning failed; stopping the server") + // Initialization writes its response to the init record before anything + // else, so a record still holding exactly what was planted is a server + // that was never initialized. + planted := h.docker("run", "--rm", "-u", "100:1000", "-v", volume+":/vault/data", "--entrypoint", "cat", image.FullName(), "/vault/data/bootstrap/init.json") + require.JSONEq(t, `{"root_token": ""}`, planted, "the server was left uninitialized") } // With VAULT_RECOVERY_PGP_KEY the recovery key is only ever written encrypted diff --git a/provision.go b/provision.go index 1a0a5bb..56fb86f 100644 --- a/provision.go +++ b/provision.go @@ -27,9 +27,10 @@ const ( // provisionScript provisions, in a deployed Vault, the state the local runtime // seeds (Runtime.enableTransit and localVaultState.bootstrap): in durable mode // initialization, the composition-supplied access token and KV v2 at secret/; -// in both modes the transit engine at transit/ and the configured key. The -// deployment renders it as the vault container's postStart hook, so it runs on -// every start of the Vault process. +// in both modes the transit engine at transit/ and the configured key. It runs +// on every start of the Vault process: in durable mode from the container's +// own command (provision/server.sh), because a cell's execution admission +// refuses an exec hook; in the local dev render as its postStart hook. // //go:embed provision/provision.sh var provisionScript string @@ -64,11 +65,14 @@ func provisionCommand(mode provisionMode, key string) (string, error) { return execCommand("/bin/sh", "-c", provisionScript, "vault-provision", string(mode), key) } -// serverCommand renders the durable container's command. dumb-init stays PID 1, -// as under the image's own entrypoint, so signals reach vault and zombies are -// reaped. -func serverCommand() (string, error) { - return execCommand("/usr/bin/dumb-init", "--", "/bin/sh", "-c", serverScript, "vault-server") +// serverCommand renders the durable container's command: provision/server.sh, +// which starts the server and provisions it with provision/provision.sh in +// durable mode. The provisioning script and the key name reach it as its +// positional $1 and $2, so the configured value is data, not script text. +// dumb-init stays PID 1, as under the image's own entrypoint, so signals reach +// vault and zombies are reaped. +func serverCommand(key string) (string, error) { + return execCommand("/usr/bin/dumb-init", "--", "/bin/sh", "-c", serverScript, "vault-server", provisionScript, key) } // execCommand renders an exec argv as a JSON array, which is a valid YAML flow diff --git a/provision/server.sh b/provision/server.sh index d68d684..0f92dde 100644 --- a/provision/server.sh +++ b/provision/server.sh @@ -1,8 +1,16 @@ #!/bin/sh # Starts the durable Vault server a restricted (deployed) render runs: integrated # raft storage on the pod's persistent volume, auto-unsealed by the seal the -# environment supplies. It is the container's command; provision/provision.sh -# (durable mode) initializes the server once and provisions it on every start. +# environment supplies. It is the container's command, and it provisions the +# server it started: provision/provision.sh (durable mode, passed as $1 with the +# transit key name as $2) initializes it once and provisions it on every start. +# +# Provisioning runs here, not as a postStart hook: a cell's execution admission +# refuses any container with an exec probe or hook (a command of the delivering +# party's choosing, run inside a pod that already holds the workload identity), +# so the start-up work lives in the container's own command. A provisioning +# failure stops the server and exits non-zero, so the container restarts and +# the next start provisions again — what a failed hook did. # # The seal is environment configuration, never part of this agent's render: the # composition declares it as the vault service's non-secret configuration @@ -53,4 +61,21 @@ disable_mlock = true ui = false EOF -exec vault server -config="$config" +provision="$1" +key="$2" + +vault server -config="$config" & +server=$! +# dumb-init forwards a termination signal to the whole process group, so the +# server receives it too; wait for it to shut down rather than leave it to be +# killed when this shell exits. +trap 'wait "$server"; exit $?' TERM INT + +if ! /bin/sh -c "$provision" vault-provision durable "$key"; then + echo "vault: provisioning failed; stopping the server so the container restarts and provisions again" >&2 + kill -TERM "$server" 2>/dev/null || true + wait "$server" || true + exit 1 +fi +echo "vault: provisioned" +wait "$server" diff --git a/provision_test.go b/provision_test.go index 9833787..9bb8303 100644 --- a/provision_test.go +++ b/provision_test.go @@ -158,13 +158,11 @@ func TestRestrictedRenderRunsADurableVault(t *testing.T) { container := renderedVaultContainer(t, destination) require.Equal(t, "vault", container.Name) require.Equal(t, - []string{"/usr/bin/dumb-init", "--", "/bin/sh", "-c", serverScript, "vault-server"}, - container.Command, "the durable server script is the container's command") + []string{"/usr/bin/dumb-init", "--", "/bin/sh", "-c", serverScript, "vault-server", provisionScript, test.want}, + container.Command, "the durable server script is the container's command; it provisions with the embedded script verbatim, key name as $2") require.Empty(t, container.Args, "no `server -dev` arguments") - require.Equal(t, - []string{"/bin/sh", "-c", provisionScript, "vault-provision", "durable", test.want}, - container.Lifecycle.PostStart.Exec.Command, - "the hook runs the embedded script verbatim, durable mode, key name as $2") + require.Empty(t, container.Lifecycle.PostStart.Exec.Command, + "no exec hook: a cell's execution admission refuses one, so the server's own command provisions") // Storage: one claim, mounted where raft and the init record live, // on the environment's default StorageClass, retained when the diff --git a/templates/deployment/kustomize/base/stateful-set.yaml.tmpl b/templates/deployment/kustomize/base/stateful-set.yaml.tmpl index 5d60b00..6f6bacf 100644 --- a/templates/deployment/kustomize/base/stateful-set.yaml.tmpl +++ b/templates/deployment/kustomize/base/stateful-set.yaml.tmpl @@ -110,15 +110,9 @@ spec: value: "true" {{- end }} {{- if .Deployment.Parameters.ProvisionCommand }} -{{- if .Deployment.Parameters.Durable }} - # Initializes the server once, installs the composition-supplied - # access token, and makes sure KV v2 at secret/ and the transit - # engine and key exist (provision/provision.sh, durable mode). It - # runs on every start and is idempotent: existing state, above all an - # existing transit key, is never rewritten. The container does not - # reach Running until it succeeds; a failed hook restarts the - # container. -{{- else }} + # The local dev render only: a durable render provisions from its own + # command (provision/server.sh), because a cell's execution admission + # refuses an exec hook. # Consumers encrypt and HMAC through transit/ (the configured key), and # `vault server -dev` starts with no transit engine. The hook # provisions it over loopback (provision/provision.sh, dev mode) on @@ -126,7 +120,6 @@ spec: # restart therefore mints a NEW key: everything encrypted or HMACed # before it is unreadable. That is acceptable only for this # disposable local render. -{{- end }} lifecycle: postStart: exec: @@ -153,8 +146,8 @@ spec: # 501 — not initialized # 503 — sealed # Startup and liveness only ask whether the process answers: a - # durable server is uninitialized until its first postStart hook - # completes, and sealed while its seal is unreachable — restarting it + # durable server is uninitialized until its own command has + # provisioned it, and sealed while its seal is unreachable — restarting it # fixes neither. Readiness is strict: only an initialized, unsealed, # active server receives traffic. startupProbe: