feat(policy-log): the witness starts with no warehouse mirror, and locks every entry under its delivered retention (#1154)

module/configurations/local/policy-log.env

@@ -11,4 +11,8 @@
 # against a dedicated bucket whose IAM forbids deleting or replacing anything
 # under `entries/`.
 POLICY_LOG_BUCKET=
+# Days every entry is held under a LOCKED object retention (nobody can delete or
+# replace it before then; the bucket must have object retention enabled). A
+# whole number of days, at most ten years. Required with the bucket.
+POLICY_LOG_RETENTION_DAYS=
 POLICY_LOG_WAREHOUSE_ENDPOINT=

module/configurations/local/policy-log.secret.env

@@ -3,10 +3,13 @@
 # the database could rewrite the evidence that database's narrowings are
 # witnessed by.
 #
-# A placeholder, never a usable value. Production supplies a unique,
-# high-entropy token through Codefly's secret configuration provider, over
-# HTTPS, with redirects refused.
-POLICY_LOG_WAREHOUSE_TOKEN=local-dev-only-replace-me
+# POLICY_LOG_WAREHOUSE_TOKEN is optional, together with
+# POLICY_LOG_WAREHOUSE_ENDPOINT: both set configures the warehouse mirror,
+# neither runs the witness with no mirror (stated at boot), and one without the
+# other is refused. It is not assigned here because this local composition
+# configures no mirror, and a shipped secret default may only hold a placeholder.
+# Where a mirror is configured, it is a unique, high-entropy token delivered
+# through Codefly's secret configuration provider.
 
 # Persistent witness identity, provisioned by the composition secret provider.
 # Canonical standard base64 of PKCS#8 Ed25519 DER; no generated boot fallback.

module/services/policy-log/README.md

@@ -30,19 +30,45 @@ database restore must not restore the witness. Accounts must have no object
 write, signing-key read, warehouse writer, or witness administration capability.
 The witness holds no accounts database credential or dependency.
 
-Delivery must provision a dedicated bucket, workload identity, private signing
-key, and warehouse ingestion credential. Bucket IAM must confine the workload
+Delivery must provision a dedicated bucket, workload identity and private
+signing key, and — only where a warehouse mirror is configured — the warehouse
+ingestion credential. Bucket IAM must confine the workload
 to that bucket and forbid deleting or replacing `entries/` objects; enforce the
-immutable prefix with retention and IAM, in addition to application checks.
+immutable prefix with retention and IAM, in addition to application checks. The
+retention half is this service's at write time: every entry is created under a
+**Locked** object retention for `POLICY_LOG_RETENTION_DAYS` (a whole number of
+days, at most ten years), so the bucket must have object retention enabled, and
+a bucket that cannot hold the lock refuses every append. The head carries no
+retention, because it is rewritten on every append.
+
+Uploading under a retention lock needs `storage.objects.setRetention`. Grant it
+**only** under an IAM condition on `resource.name.startsWith(".../objects/entries/")`,
+never bucket-wide and never through `roles/storage.objectAdmin`: with it on the
+head, a compromised workload could lock `head.json` far into the future and the
+log could never advance again. The head's write authority (create and delete,
+for the generation-matched replace) is conditioned on `head.json` alone, and
+nothing is granted delete under `entries/`. Enabling object retention on a
+bucket cannot be undone.
+
+A bucket without object retention, or an identity without `setRetention`,
+still boots — `witness.Open` only reads — and then refuses every append with a
+400 or 403 from the store. Those two symptoms name a provisioning defect, not
+a witness one.
+
+An entry written whose head commit was lost (a timeout after the entry landed)
+holds its sequence slot under the lock for the whole horizon, and it cannot be
+removed out of band. The log is unstuck by retrying that operation's identical
+entry — rebuildable from the orphaned object itself — which completes the head
+commit; every other operation is answered `ErrPending` until then.
 The head's write authority is scoped to `head.json`. Bucket lifecycle deletion,
 rewriting an existing sequence, or restoring the head from a database backup
 would defeat the trust boundary and are not recovery procedures. Provision
 versioning and deletion protection outside the workload's administrative reach.
 
 The runtime exposes no delete, reset, unconditional overwrite, ambient signing-key
 source, insecure warehouse transport, or emulator option. The GCS constructor
-refuses a storage emulator setting. The warehouse uses its own mandatory bearer
-credential over HTTPS and refuses redirects.
+refuses a storage emulator setting. A configured warehouse mirror uses its own
+bearer credential, mandatory with its endpoint, over HTTPS and refuses redirects.
 
 The listener is private and must stay private. Composition must admit only the
 host's declared client through workload-authenticated mesh policy, resolve its
@@ -201,18 +227,23 @@ and the detail goes to this service's own failure reporter instead.
 [code/main.go](code/main.go) is the binary. Every construction happens **before**
 `net.Listen`, in this order, and any failure is fatal: resolve the whole
 configuration group, read the Codefly-allocated port, parse the delivered signing-key
-secret, open the bucket, build the warehouse writer, start the mirror
-worker, then `witness.Open`, which verifies the entire committed chain under
+secret, open the bucket, build the warehouse writer and start the mirror
+worker (or wire `mirror.None` and say so, when no mirror is configured), then
+`witness.Open`, which verifies the entire committed chain under
 that key. A witness that accepted an append it could not sign, store, or verify
 the history of would be worse than one that is down, because the caller would
 commit against it.
 
-`POLICY_LOG_BUCKET`, `POLICY_LOG_WAREHOUSE_ENDPOINT` and
+`POLICY_LOG_BUCKET`, `POLICY_LOG_RETENTION_DAYS`, `POLICY_LOG_WAREHOUSE_ENDPOINT` and
 `POLICY_LOG_WAREHOUSE_TOKEN` come from the `policy-log` configuration group and
 from nowhere else. There is no process-environment fallback, no default bucket
 and no development bypass, so an ambient value cannot make this service witness
 somewhere nobody provisioned — and the refusal names the missing keys, never the
-delivered values, because one of the three is a credential. The native composition cannot yet start this service with its empty group.
+delivered values, because one of them is a credential. The bucket and its entry
+retention are always required. The two warehouse settings are optional, but only together: both
+delivered starts the mirror, neither starts the witness with no mirror, and one
+without the other is refused as a half-provisioned group. The native composition
+cannot yet start this service with its empty group.
 Local execution must use the same signing and verification protocol with durable
 custody independent of Accounts; missing native storage and bootstrap are owner
 tooling gaps, not a reason to create a host-private log or fake receipts.
@@ -292,6 +323,15 @@ qualification before the transport can claim the full deadline requirement.
 
 ## Warehouse mirror
 
+**The mirror is optional** (owner decision 2026-10-18, reversing the 2026-10-17
+call that the witness stays down until a warehouse front exists). The object
+store is the authority — every receipt is signed, chained and committed there
+before it is returned — so a witness deployed with no warehouse configured
+witnesses fully; what it lacks is the projection. It wires `mirror.None`, a
+named value rather than a nil, and prints one line at boot naming
+`POLICY_LOG_WAREHOUSE_ENDPOINT` and `POLICY_LOG_WAREHOUSE_TOKEN`, so the pod log
+says why the warehouse is empty. A warehouse added later backfills from `Read`.
+
 The [warehouse writer](code/mirror/warehouse.go) follows the repository's JSON
 POST ingestion convention with a dedicated credential. The service offers an
 entry only after the head commits. One worker consumes a bounded 64-entry

module/services/policy-log/code/main.go

@@ -1,7 +1,9 @@
 // policy-log is the independent signed witness for authority changes. This
 // binary wires the delivered signing-key secret, the object store, the warehouse
 // mirror and the transport-independent service onto the gRPC endpoint Codefly
-// allocated, and refuses to start if any one of them is not provisioned.
+// allocated, and refuses to start if the key or the store is not provisioned.
+// The mirror is the one optional part: with neither of its settings delivered
+// the witness runs without one and says so at boot (owner decision 2026-10-18).
 //
 // Nothing here may start a listener it cannot witness with, so every
 // construction happens BEFORE net.Listen and any failure is fatal. A witness
@@ -20,6 +22,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	"policy-log/identity"
 	"policy-log/mirror"
@@ -42,13 +45,22 @@ const grpcEndpointName = "grpc"
 // provisions for this service. It is the ONLY source of these settings.
 const policyLogConfiguration = "policy-log"
 
-var settingKeys = struct{ bucket, warehouseEndpoint, warehouseToken string }{
+var settingKeys = struct{ bucket, retentionDays, warehouseEndpoint, warehouseToken string }{
 	bucket:            "POLICY_LOG_BUCKET",
+	retentionDays:     "POLICY_LOG_RETENTION_DAYS",
 	warehouseEndpoint: "POLICY_LOG_WAREHOUSE_ENDPOINT",
 	warehouseToken:    "POLICY_LOG_WAREHOUSE_TOKEN",
 }
 
-type settings struct{ bucket, warehouseEndpoint, warehouseToken string }
+type settings struct {
+	bucket                            string
+	retention                         time.Duration
+	warehouseEndpoint, warehouseToken string
+}
+
+// mirrored reports whether a warehouse mirror was configured. resolveSettings
+// guarantees the endpoint and token are delivered together or not at all.
+func (s settings) mirrored() bool { return s.warehouseEndpoint != "" }
 
 // resolveSettings reads the whole configuration from the delivered group and
 // from nowhere else.
@@ -59,34 +71,97 @@ type settings struct{ bucket, warehouseEndpoint, warehouseToken string }
 // successfully, which is the one failure mode this service cannot have. Every
 // missing key is named at once, so a half-provisioned group takes one restart
 // to diagnose rather than three.
-func resolveSettings(value func(group, key string) (string, error)) (settings, error) {
-	var resolved settings
-	var missing []string
-	for _, field := range []struct {
-		key  string
-		into *string
-	}{
-		{settingKeys.bucket, &resolved.bucket},
-		{settingKeys.warehouseEndpoint, &resolved.warehouseEndpoint},
-		{settingKeys.warehouseToken, &resolved.warehouseToken},
-	} {
-		// A group that is absent and a key that is empty are the same thing to
-		// this service: nothing was provisioned. Neither value is reported back,
-		// because one of the three is a credential.
-		got, err := value(policyLogConfiguration, field.key)
-		if err != nil || got == "" {
-			missing = append(missing, field.key)
-			continue
+//
+// The bucket is the witness's authority and is always required. The warehouse
+// mirror is a best-effort projection of what the bucket already committed
+// (README, "Warehouse mirror"), so it is optional — but only WHOLE: its endpoint
+// and its token are delivered together or not at all. One without the other is
+// a half-provisioned mirror, and it is refused rather than read as "no mirror",
+// because that would silently drop a projection someone meant to configure.
+func resolveSettings(value func(group, key string) (string, bool, error)) (settings, error) {
+	// A value no namespace carries and an empty value are the same thing to this
+	// service: nothing was provisioned. A value that is there and could not be
+	// READ is not: it is refused by name, never read as absent — otherwise a
+	// broken delivery of both mirror settings would boot claiming no mirror was
+	// configured. No value is ever reported back, because one is a credential.
+	var unreadable []string
+	read := func(key string) string {
+		got, set, err := value(policyLogConfiguration, key)
+		if err != nil {
+			unreadable = append(unreadable, key)
+			return ""
 		}
-		*field.into = got
+		if !set {
+			return ""
+		}
+		return got
+	}
+	retentionDays := read(settingKeys.retentionDays)
+	resolved := settings{
+		bucket:            read(settingKeys.bucket),
+		warehouseEndpoint: read(settingKeys.warehouseEndpoint),
+		warehouseToken:    read(settingKeys.warehouseToken),
+	}
+	if len(unreadable) > 0 {
+		return settings{}, fmt.Errorf("configuration group %q delivered %s but it could not be read: this service refuses rather than guess what was provisioned",
+			policyLogConfiguration, strings.Join(unreadable, ", "))
+	}
+	var missing []string
+	if resolved.bucket == "" {
+		missing = append(missing, settingKeys.bucket)
+	}
+	// The entry retention is the bucket's other half: how long every entry is
+	// Locked. Required, a whole number of days, and bounded — a Locked
+	// retention can never be shortened, so a malformed value is refused rather
+	// than rounded.
+	// Bounded in DAYS before it is multiplied: a large count would overflow
+	// time.Duration into a short, accepted retention.
+	if days, err := strconv.Atoi(retentionDays); err != nil || days <= 0 ||
+		days > int(gcs.MaxRetention/(24*time.Hour)) || strconv.Itoa(days) != retentionDays {
+		missing = append(missing, settingKeys.retentionDays)
+	} else {
+		resolved.retention = time.Duration(days) * 24 * time.Hour
+	}
+	switch {
+	case resolved.warehouseEndpoint != "" && resolved.warehouseToken == "":
+		missing = append(missing, settingKeys.warehouseToken)
+	case resolved.warehouseEndpoint == "" && resolved.warehouseToken != "":
+		missing = append(missing, settingKeys.warehouseEndpoint)
 	}
 	if len(missing) > 0 {
-		return settings{}, fmt.Errorf("configuration group %q delivered no %s: composition must provision the whole group before this service can witness",
+		return settings{}, fmt.Errorf("configuration group %q delivered no usable %s: composition must provision the bucket and its entry retention, and the warehouse mirror whole or not at all, before this service can witness",
 			policyLogConfiguration, strings.Join(missing, ", "))
 	}
 	return resolved, nil
 }
 
+// noMirrorWarning is the boot line a witness with no warehouse mirror prints,
+// so a reader of the pod log learns what the deployment chose rather than
+// inferring it from a warehouse that stays empty.
+const noMirrorWarning = "policy-log: no warehouse mirror is configured: this witness WITNESSES normally — " +
+	"the object store is the authority and every receipt is signed, chained and committed there — " +
+	"but committed entries are not projected to the warehouse until POLICY_LOG_WAREHOUSE_ENDPOINT and " +
+	"POLICY_LOG_WAREHOUSE_TOKEN are delivered together; Read is the source to backfill it from"
+
+// startMirror builds the warehouse projection the settings ask for: the
+// bounded best-effort worker when one is configured, or mirror.None, stated at
+// boot, when none is.
+func startMirror(ctx context.Context, resolved settings, warn func(string)) (service.Mirror, error) {
+	if !resolved.mirrored() {
+		warn(noMirrorWarning)
+		return mirror.None{}, nil
+	}
+	warehouse, err := mirror.NewWriter(resolved.warehouseEndpoint, resolved.warehouseToken)
+	if err != nil {
+		return nil, err
+	}
+	// A dropped or failed mirror is reported and repairable from Read. It never
+	// changes an append's outcome, so it is logged, not fatal.
+	return mirror.Start(ctx, warehouse, func(err error) {
+		log.Printf("policy-log: warehouse mirror: %v", err)
+	})
+}
+
 // resolveSigningKey reads only the declared secret group. The public pin is
 // independently delivered to clients; this service cannot choose or replace it.
 func resolveSigningKey(secret func(string, string) (string, error)) (ed25519.PrivateKey, error) {
@@ -102,9 +177,7 @@ func resolveSigningKey(secret func(string, string) (string, error)) (ed25519.Pri
 }
 
 func run(ctx context.Context, stop func()) error {
-	resolved, err := resolveSettings(func(group, key string) (string, error) {
-		return codefly.For(ctx).WorkspaceValue(group, key)
-	})
+	resolved, err := resolveSettings(codefly.For(ctx).WorkspaceValueIfSet)
 	if err != nil {
 		return err
 	}
@@ -120,20 +193,12 @@ func run(ctx context.Context, stop func()) error {
 	if err != nil {
 		return err
 	}
-	store, err := gcs.New(ctx, resolved.bucket)
+	store, err := gcs.New(ctx, resolved.bucket, resolved.retention)
 	if err != nil {
 		return err
 	}
 	defer func() { _ = store.Close() }()
-	warehouse, err := mirror.NewWriter(resolved.warehouseEndpoint, resolved.warehouseToken)
-	if err != nil {
-		return err
-	}
-	// A dropped or failed mirror is reported and repairable from Read. It never
-	// changes an append's outcome, so it is logged, not fatal.
-	worker, err := mirror.Start(ctx, warehouse, func(err error) {
-		log.Printf("policy-log: warehouse mirror: %v", err)
-	})
+	worker, err := startMirror(ctx, resolved, func(line string) { log.Print(line) })
 	if err != nil {
 		return err
 	}