fix(deploy): the durable server provisions from its own command, not a postStart hook

builder.go

@@ -38,7 +38,7 @@ type deploymentTemplateParameters struct {
 	// the Service publishes it and folds it onto 8200, the port the container
 	// listens on. Zero leaves the template on 8200.
 	ServicePort uint32
-	// ProvisionCommand is the vault container's postStart exec command, as a
+	// ProvisionCommand is the local dev render's postStart exec command, as a
 	// JSON array (a valid YAML flow sequence): provision/provision.sh run with
 	// its mode and the configured transit key name. Empty renders no hook.
 	ProvisionCommand string
@@ -146,20 +146,19 @@ func (s *Builder) Deploy(ctx context.Context, req *builderv0.DeploymentRequest)
 		parameters.External = true
 	case restricted:
 		parameters.Durable = true
-		if parameters.ServerCommand, err = serverCommand(); err != nil {
-			return s.Builder.DeployError(err)
-		}
 	}
 	if !parameters.External {
 		key, err := s.transitKeyName()
 		if err != nil {
 			return s.Builder.DeployError(err)
 		}
-		mode := provisionModeDev
 		if parameters.Durable {
-			mode = provisionModeDurable
-		}
-		if parameters.ProvisionCommand, err = provisionCommand(mode, key); err != nil {
+			// The durable server provisions itself from its own command: a
+			// cell's execution admission refuses an exec hook.
+			if parameters.ServerCommand, err = serverCommand(key); err != nil {
+				return s.Builder.DeployError(err)
+			}
+		} else if parameters.ProvisionCommand, err = provisionCommand(provisionModeDev, key); err != nil {
 			return s.Builder.DeployError(err)
 		}
 	}

durable_test.go

@@ -73,8 +73,8 @@ func TestDurableProvisioningRefusesBadInputs(t *testing.T) {
 }
 
 // durableHarness runs the pinned image exactly as the restricted render does:
-// the rendered container command on a read-only root filesystem with a
-// persistent data volume, and the rendered postStart hook executed inside it.
+// the rendered container command — which starts the server and provisions it —
+// on a read-only root filesystem with a persistent data volume.
 // The seal is a transit seal served by a second, throwaway Vault — the local
 // stand-in for the cloud KMS an environment supplies (gcpckms on GCP), passed
 // the way the environment passes it: as environment variables.
@@ -84,7 +84,6 @@ type durableHarness struct {
 	suffix  string
 	network string
 	server  []string
-	hook    []string
 }
 
 func newDurableHarness(t *testing.T) *durableHarness {
@@ -93,7 +92,7 @@ func newDurableHarness(t *testing.T) *durableHarness {
 	require.Equal(t, builderv0.DeploymentStatus_SUCCESS, response.GetState().GetState(), response.GetState().GetMessage())
 	container := renderedVaultContainer(t, destination)
 	require.NotEmpty(t, container.Command)
-	require.NotEmpty(t, container.Lifecycle.PostStart.Exec.Command)
+	require.Empty(t, container.Lifecycle.PostStart.Exec.Command, "the server's own command provisions; no exec hook")
 
 	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute)
 	t.Cleanup(cancel)
@@ -102,7 +101,7 @@ func newDurableHarness(t *testing.T) *durableHarness {
 	require.NoError(t, err)
 	h := &durableHarness{
 		t: t, ctx: ctx, suffix: hex.EncodeToString(suffix),
-		server: container.Command, hook: container.Lifecycle.PostStart.Exec.Command,
+		server: container.Command,
 	}
 	h.network = "vault-durable-" + h.suffix
 	h.docker("network", "create", h.network)
@@ -188,9 +187,46 @@ func (h *durableHarness) start(volume, accessToken string, extraEnv ...string) (
 	return name, "http://" + h.docker("port", name, "8200/tcp")
 }
 
-// provision runs the rendered hook in the container, as the kubelet does.
+// provision waits for the container's own command to finish provisioning the
+// server it started: success is its "vault: provisioned" line with the server
+// still running; failure is the container exiting. It returns the provisioning
+// lines of the container's log, which is what a hook's output used to be.
 func (h *durableHarness) provision(container string) (string, error) {
-	return h.try(append([]string{"exec", container}, h.hook...)...)
+	h.t.Helper()
+	for {
+		logs, _ := h.try("logs", container)
+		state := strings.TrimSpace(h.docker("inspect", "-f", "{{.State.Running}} {{.State.ExitCode}}", container))
+		lines := provisioningLines(logs)
+		switch {
+		case strings.Contains(logs, "vault: provisioned"):
+			return lines, nil
+		case strings.HasPrefix(state, "false"):
+			return lines, fmt.Errorf("the container exited before provisioning (state %s)", state)
+		}
+		select {
+		case <-h.ctx.Done():
+			return lines, h.ctx.Err()
+		case <-time.After(500 * time.Millisecond):
+		}
+	}
+}
+
+// reprovision runs the provisioning script again against the running server,
+// as the next start of the process would: it must find nothing to do.
+func (h *durableHarness) reprovision(container string) (string, error) {
+	return h.try("exec", container, "/bin/sh", "-c", provisionScript, "vault-provision", "durable", "api-keys")
+}
+
+// provisioningLines keeps the lines the provisioning script and the server
+// script print about provisioning, leaving out the server's own log.
+func provisioningLines(logs string) string {
+	var kept []string
+	for _, line := range strings.Split(logs, "\n") {
+		if strings.Contains(line, "vault provisioning:") || strings.HasPrefix(line, "vault: ") {
+			kept = append(kept, line)
+		}
+	}
+	return strings.Join(kept, "\n")
 }
 
 func (h *durableHarness) stop(container string) {
@@ -269,7 +305,7 @@ func TestDurableVaultSurvivesRestart(t *testing.T) {
 
 	// Twice more on the same process: nothing to do.
 	for range 2 {
-		out, err = h.provision(first)
+		out, err = h.reprovision(first)
 		require.NoError(t, err, out)
 		require.Equal(t, "vault provisioning: transit engine and key \"api-keys\" ready", strings.TrimSpace(out))
 	}
@@ -279,7 +315,7 @@ func TestDurableVaultSurvivesRestart(t *testing.T) {
 	second, address := h.start(volume, token)
 	out, err = h.provision(second)
 	require.NoError(t, err, out)
-	require.Equal(t, "vault provisioning: transit engine and key \"api-keys\" ready", strings.TrimSpace(out),
+	require.Equal(t, "vault provisioning: transit engine and key \"api-keys\" ready\nvault: provisioned", strings.TrimSpace(out),
 		"a restarted Vault unseals itself and needs no initialization, token or key")
 
 	status, decrypted := h.call(address, token, http.MethodPost, "/v1/transit/decrypt/api-keys", fmt.Sprintf(`{"ciphertext":%q}`, encrypted["ciphertext"]))
@@ -316,8 +352,12 @@ func TestDurableVaultRefusesToInitializeOverAnInitRecord(t *testing.T) {
 	out, err := h.provision(container)
 	require.Error(t, err)
 	require.Contains(t, out, "vault storage is uninitialized but /vault/data/bootstrap/init.json exists")
-	raw := h.docker("exec", container, "wget", "-qO-", "http://127.0.0.1:8200/v1/sys/init")
-	require.JSONEq(t, `{"initialized":false}`, raw, "the server was left uninitialized")
+	require.Contains(t, out, "vault: provisioning failed; stopping the server")
+	// Initialization writes its response to the init record before anything
+	// else, so a record still holding exactly what was planted is a server
+	// that was never initialized.
+	planted := h.docker("run", "--rm", "-u", "100:1000", "-v", volume+":/vault/data", "--entrypoint", "cat", image.FullName(), "/vault/data/bootstrap/init.json")
+	require.JSONEq(t, `{"root_token": ""}`, planted, "the server was left uninitialized")
 }
 
 // With VAULT_RECOVERY_PGP_KEY the recovery key is only ever written encrypted

provision.go

@@ -27,9 +27,10 @@ const (
 // provisionScript provisions, in a deployed Vault, the state the local runtime
 // seeds (Runtime.enableTransit and localVaultState.bootstrap): in durable mode
 // initialization, the composition-supplied access token and KV v2 at secret/;
-// in both modes the transit engine at transit/ and the configured key. The
-// deployment renders it as the vault container's postStart hook, so it runs on
-// every start of the Vault process.
+// in both modes the transit engine at transit/ and the configured key. It runs
+// on every start of the Vault process: in durable mode from the container's
+// own command (provision/server.sh), because a cell's execution admission
+// refuses an exec hook; in the local dev render as its postStart hook.
 //
 //go:embed provision/provision.sh
 var provisionScript string
@@ -64,11 +65,14 @@ func provisionCommand(mode provisionMode, key string) (string, error) {
 	return execCommand("/bin/sh", "-c", provisionScript, "vault-provision", string(mode), key)
 }
 
-// serverCommand renders the durable container's command. dumb-init stays PID 1,
-// as under the image's own entrypoint, so signals reach vault and zombies are
-// reaped.
-func serverCommand() (string, error) {
-	return execCommand("/usr/bin/dumb-init", "--", "/bin/sh", "-c", serverScript, "vault-server")
+// serverCommand renders the durable container's command: provision/server.sh,
+// which starts the server and provisions it with provision/provision.sh in
+// durable mode. The provisioning script and the key name reach it as its
+// positional $1 and $2, so the configured value is data, not script text.
+// dumb-init stays PID 1, as under the image's own entrypoint, so signals reach
+// vault and zombies are reaped.
+func serverCommand(key string) (string, error) {
+	return execCommand("/usr/bin/dumb-init", "--", "/bin/sh", "-c", serverScript, "vault-server", provisionScript, key)
 }
 
 // execCommand renders an exec argv as a JSON array, which is a valid YAML flow

provision/server.sh

@@ -1,8 +1,16 @@
 #!/bin/sh
 # Starts the durable Vault server a restricted (deployed) render runs: integrated
 # raft storage on the pod's persistent volume, auto-unsealed by the seal the
-# environment supplies. It is the container's command; provision/provision.sh
-# (durable mode) initializes the server once and provisions it on every start.
+# environment supplies. It is the container's command, and it provisions the
+# server it started: provision/provision.sh (durable mode, passed as $1 with the
+# transit key name as $2) initializes it once and provisions it on every start.
+#
+# Provisioning runs here, not as a postStart hook: a cell's execution admission
+# refuses any container with an exec probe or hook (a command of the delivering
+# party's choosing, run inside a pod that already holds the workload identity),
+# so the start-up work lives in the container's own command. A provisioning
+# failure stops the server and exits non-zero, so the container restarts and
+# the next start provisions again — what a failed hook did.
 #
 # The seal is environment configuration, never part of this agent's render: the
 # composition declares it as the vault service's non-secret configuration
@@ -53,4 +61,21 @@ disable_mlock = true
 ui            = false
 EOF
 
-exec vault server -config="$config"
+provision="$1"
+key="$2"
+
+vault server -config="$config" &
+server=$!
+# dumb-init forwards a termination signal to the whole process group, so the
+# server receives it too; wait for it to shut down rather than leave it to be
+# killed when this shell exits.
+trap 'wait "$server"; exit $?' TERM INT
+
+if ! /bin/sh -c "$provision" vault-provision durable "$key"; then
+	echo "vault: provisioning failed; stopping the server so the container restarts and provisions again" >&2
+	kill -TERM "$server" 2>/dev/null || true
+	wait "$server" || true
+	exit 1
+fi
+echo "vault: provisioned"
+wait "$server"

provision_test.go

@@ -158,13 +158,11 @@ func TestRestrictedRenderRunsADurableVault(t *testing.T) {
 			container := renderedVaultContainer(t, destination)
 			require.Equal(t, "vault", container.Name)
 			require.Equal(t,
-				[]string{"/usr/bin/dumb-init", "--", "/bin/sh", "-c", serverScript, "vault-server"},
-				container.Command, "the durable server script is the container's command")
+				[]string{"/usr/bin/dumb-init", "--", "/bin/sh", "-c", serverScript, "vault-server", provisionScript, test.want},
+				container.Command, "the durable server script is the container's command; it provisions with the embedded script verbatim, key name as $2")
 			require.Empty(t, container.Args, "no `server -dev` arguments")
-			require.Equal(t,
-				[]string{"/bin/sh", "-c", provisionScript, "vault-provision", "durable", test.want},
-				container.Lifecycle.PostStart.Exec.Command,
-				"the hook runs the embedded script verbatim, durable mode, key name as $2")
+			require.Empty(t, container.Lifecycle.PostStart.Exec.Command,
+				"no exec hook: a cell's execution admission refuses one, so the server's own command provisions")
 
 			// Storage: one claim, mounted where raft and the init record live,
 			// on the environment's default StorageClass, retained when the

templates/deployment/kustomize/base/stateful-set.yaml.tmpl

@@ -110,23 +110,16 @@ spec:
               value: "true"
 {{- end }}
 {{- if .Deployment.Parameters.ProvisionCommand }}
-{{- if .Deployment.Parameters.Durable }}
-          # Initializes the server once, installs the composition-supplied
-          # access token, and makes sure KV v2 at secret/ and the transit
-          # engine and key exist (provision/provision.sh, durable mode). It
-          # runs on every start and is idempotent: existing state, above all an
-          # existing transit key, is never rewritten. The container does not
-          # reach Running until it succeeds; a failed hook restarts the
-          # container.
-{{- else }}
+          # The local dev render only: a durable render provisions from its own
+          # command (provision/server.sh), because a cell's execution admission
+          # refuses an exec hook.
           # Consumers encrypt and HMAC through transit/ (the configured key), and
           # `vault server -dev` starts with no transit engine. The hook
           # provisions it over loopback (provision/provision.sh, dev mode) on
           # every container start, because the dev store is in-memory. A
           # restart therefore mints a NEW key: everything encrypted or HMACed
           # before it is unreadable. That is acceptable only for this
           # disposable local render.
-{{- end }}
           lifecycle:
             postStart:
               exec:
@@ -153,8 +146,8 @@ spec:
           #   501 — not initialized
           #   503 — sealed
           # Startup and liveness only ask whether the process answers: a
-          # durable server is uninitialized until its first postStart hook
-          # completes, and sealed while its seal is unreachable — restarting it
+          # durable server is uninitialized until its own command has
+          # provisioned it, and sealed while its seal is unreachable — restarting it
           # fixes neither. Readiness is strict: only an initialized, unsealed,
           # active server receives traffic.
           startupProbe: