fix(store): retention drops audit partitions by their real bounds, under lock

.github/workflows/ci.yml

@@ -112,6 +112,26 @@ jobs:
         run: node module/tools/migration-pairing-gate.mjs check
       - name: Validate every service test-target catalog
         run: node module/tools/test-targets-gate.mjs check
+      # The alert pack is PromQL that no service runs, so an alert that cannot
+      # fire goes unnoticed until the failure it was for. promtool evaluates
+      # each alert in slo_pack.json, as the Prometheus rule its id and
+      # expression make, over the series the pack's rule tests feed it.
+      - name: Test the alert pack's rules with promtool
+        env:
+          PROMETHEUS_VERSION: "3.15.0"
+          PROMETHEUS_SHA256: 2a542df32eac02ee17b9d844fb2aa1de00dafa5476579ba8a3ba862e9d572ea0
+        run: |
+          work="$(mktemp -d)"
+          archive="prometheus-${PROMETHEUS_VERSION}.linux-amd64"
+          curl -fsSL -o "${work}/${archive}.tar.gz" \
+            "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/${archive}.tar.gz"
+          echo "${PROMETHEUS_SHA256}  ${work}/${archive}.tar.gz" | sha256sum -c -
+          tar -xzf "${work}/${archive}.tar.gz" -C "${work}" --strip-components=1 "${archive}/promtool"
+          pack=module/services/accounts/code/pkg/metrics
+          jq '{groups: [{name: "slo_pack", rules: [.alerts[] | {alert: .id, expr: .expression}]}]}' \
+            "${pack}/slo_pack.json" > "${work}/slo_pack.rules.json"
+          cp "${pack}/slo_pack_test.yml" "${work}/"
+          "${work}/promtool" test rules "${work}/slo_pack_test.yml"
       # The author and committer email are the half of the naming boundary the tree scan cannot
       # see, and the half that cannot be scrubbed: they are part of the commit object, so
       # changing one rewrites every descendant hash. Scoped to the commits this change proposes

PLATFORM_REFERENCE.md

@@ -203,12 +203,17 @@ Gated by tenant-admin or platform role (`src/components/auth/role-gate.tsx`).
   analog of the audit's `delegation_jti` + `caused_by` chain + ceiling-snapshot
   hash.
 - **Retention/redaction, 🟡:** `RunRetention` deletes per
-  `data_retention_policies`; export payloads are PII-redacted. The audit's named
-  two-tier (7-year content-free / 90-day full) split is not a distinct construct
-  yet — it is the compliance tier planned under
+  `data_retention_policies`, counted from the database's clock, and retires
+  audit events by dropping each monthly partition whose recorded range ends
+  before the window
+  ([DATABASE_AUTHORITY.md](./module/DATABASE_AUTHORITY.md#audit-retention-drops-a-partition-by-its-recorded-range-under-lock));
+  a run that fails in any part is an error the alert pack watches
+  ([MEASUREMENT_RUNBOOKS.md](./module/MEASUREMENT_RUNBOOKS.md#retention));
+  export payloads are PII-redacted. The audit's named two-tier (7-year
+  content-free / 90-day full) split is not a distinct construct yet — it is the
+  compliance tier planned under
   [ADR 0006](./module/docs/adr/0006-audit-sink-and-retention-tiers.md) (tiered
-  retention + a swappable compliance sink; note today's `RunRetention` is a
-  no-op because it collides with the immutability trigger).
+  retention + a swappable compliance sink).
 - **Read auditing, ⬜:** bulk reads are RLS-scoped but not themselves logged as
   audit events. The audit's "audit the access set (IDs, capped, dedupe window)"
   pattern — and its 11k-rows-from-a-poller cautionary tale — is unbuilt.

RELEASE_GATES.md

@@ -437,7 +437,7 @@ reverse, fails the `release-contract` job.
 
 | Job | What it guards | What it runs |
 | --- | --- | --- |
-| `base-integrity` | the frontend workspace install graph and public-claim scan (`module-verify.mjs`), plus the RLS-migration, migration-pairing, reference-aware migration upgrade, and generated-pin gates | `node --test` for each gate's own suite, then `node module/tools/<gate>.mjs check`; migration reference validation runs on every candidate, with PostgreSQL upgrade and clean-install replay scoped to migration/runner inputs (see [migration policy](module/services/store/migrations/README.md)) |
+| `base-integrity` | the frontend workspace install graph and public-claim scan (`module-verify.mjs`), plus the RLS-migration, migration-pairing, reference-aware migration upgrade, and generated-pin gates, and the alert pack's rules | `node --test` for each gate's own suite, then `node module/tools/<gate>.mjs check`; migration reference validation runs on every candidate, with PostgreSQL upgrade and clean-install replay scoped to migration/runner inputs (see [migration policy](module/services/store/migrations/README.md)); `promtool test rules` over the alerts in `slo_pack.json`, with the tests in `slo_pack_test.yml` beside it, on a pinned, checksum-verified promtool |
 | `authz-coverage` | the generated authorization catalog: RBAC coverage, audit coverage, and permission no-broadening against `main` | `node module/tools/authz-coverage-gate.mjs`, plus `go test` for the gateway header-lockstep and adapter enforcement tests |
 | `release-contract` | this gating graph itself, and the Dependabot configuration that feeds it | `node --test scripts/ci/release-gates.test.mjs`, `node --test scripts/ci/dependabot-coverage.test.mjs`, `node scripts/ci/release-gates.mjs check`, `node scripts/ci/dependabot-coverage.mjs check` |
 | `docs-sync` | the generated interface docs and the story-trace tests (#516) | `node module/tools/interface-docs-gate.mjs check`, `node module/tools/story-trace-gate.mjs tests` |

module/DATABASE_AUTHORITY.md

@@ -588,6 +588,104 @@ attached or detached, now needs a `BYPASSRLS` role to read every row, which the
 partition offload planned in
 [ADR 0006](./docs/adr/0006-audit-sink-and-retention-tiers.md) has to provision.
 
+## Audit retention drops a partition by its recorded range, under lock
+
+Audit retention is a partition drop, and a dropped partition cannot be
+recovered. `audit_events_drop_partitions_before(cutoff)`, as migration
+`34_audit_retention_partition_bounds` defines it, drops a monthly partition only
+when the upper bound PostgreSQL records for it ends by the cutoff. Earlier
+releases cut a partition's bounds at the midnights of the session zone that
+created it, so the function reads each bound from `pg_get_expr(relpartbound)`,
+never from the month in the name, and renders and reads it under a `DateStyle`
+and `TimeZone` it pins, never the caller's. It never drops a partition whose
+bounds are not a bounded range (`DEFAULT`, `MINVALUE`, `MAXVALUE`) or whose
+detach has begun.
+
+The cutoff is one the `audit_events` retention policy allows by the database's
+clock. The function refuses (SQLSTATE 22023) a cutoff later than the start of
+its transaction less the policy's window, a missing one, and any when there is
+no policy, so a caller whose clock runs ahead cannot drop a partition still
+inside the window. Once something is due it reads the policy again, with a row
+lock (`FOR SHARE NOWAIT`) it holds until it commits, and checks the cutoff
+against that, so the window it keeps to is the one in force when it commits: a
+change to the policy made while a run waits for its locks waits in turn for the
+run, and one already under way when the run reads the policy makes the run
+refuse (55P03) rather than wait. A window is a positive number of days:
+`data_retention_policies` refuses any other with a check constraint, and the
+migration that added it refuses to apply while a policy holds one, naming it,
+rather than leave retention to guess what it meant.
+
+When something is due the function locks `audit_events` `ACCESS EXCLUSIVE` and
+decides again from the catalog as it stands once the lock is held, so a
+partition detached while it waited is left alone. That lock stops a partition
+being created, attached, detached or given other bounds, but not renamed, and
+`LOCK TABLE` finds the table it locks by name: a partition renamed away while a
+`LOCK TABLE` waits for it, and another renamed to its name, would lead that
+`LOCK TABLE` on to the other one, and to a second wait. So the function locks
+each partition it chose by identity, under the name it has at that moment, with
+`LOCK TABLE ... NOWAIT` tried again until its deadline rather than a wait in the
+lock queue, and refuses unless the lock it holds is on that very partition.
+Holding them all, it decides a third time. It drops them only if exactly those
+are still due, and otherwise drops nothing (55000). Only a READ COMMITTED
+statement reads the catalog as it stands, so the function refuses any other
+isolation level (25000). `DropAuditPartitionsBefore` opens its own control-plane
+transaction at READ COMMITTED when its caller has none, whatever the database's
+default isolation, and that is how `RunRetention` calls it. The function waits
+ten seconds in all for its locks, not ten for each: it takes the policy's table
+and `audit_events` each in a statement of its own, after setting `lock_timeout`
+to what remains of ten seconds from its start, and each partition it locks with
+`NOWAIT`, tried again until then. Each `DROP` runs under what remains too, which
+bounds its wait for the DEFAULT partition, if there is one, which dropping a
+range partition also locks. It gives up with 55P03, dropping nothing, once
+nothing remains. When nothing is due it takes no lock at all: it reads bounds
+with `pg_get_expr(relpartbound, 0)`, which does not open the partition.
+
+`RunRetention` counts every window back from the database's clock, in days of
+24 hours, so the cutoff it passes is exactly the window before the database's
+present, and always in the past. It refuses a window that is not a positive
+number of days. A run goes on past a part that fails and returns every failure
+together; the scheduler logs that as an error and sets the gauge
+`saas.retention.failing` to 1 for each failed part, and back to 0 once a run no
+longer fails it. The alert pack's `retention_failures` alert fires while a part
+reads 1, from the first scrape after the failure, and CI tests it with
+`promtool` ([MEASUREMENT_RUNBOOKS.md](./MEASUREMENT_RUNBOOKS.md#retention)).
+`pkg/infra/postgres_audit_retention_test.go` holds the function's rules against
+PostgreSQL, and the retention tests in `pkg/business` hold the window's.
+
+## Audit partitions are provisioned at UTC midnights, within ten seconds of waiting
+
+`audit_events_provision_partitions(months_ahead)` is the store's one
+provisioning call, at startup and on every retention run, whatever the
+policies. It creates the partitions of the months from the previous one to
+`months_ahead` ahead, by the database's clock in UTC, then brings every
+partition's row-level security in step with the parent's.
+`audit_events_ensure_partition(month)` and `audit_events_secure_all_partitions()`
+remain for migrations and operators. Execute on all three stays with
+`app_control_plane`.
+
+Creating a partition locks `audit_events` `ACCESS EXCLUSIVE`, which audit writes
+queue behind, so provisioning takes that lock only when a month lacks its
+partition, decides again under the lock what to create, and waits ten seconds in
+all for every lock it takes, like retention, before giving up with 55P03 and
+creating nothing. The months ahead are provisioned in advance, so a call that
+gives up leaves the next one time to try again. Bringing a partition in step
+locks it only when it is out of step, by identity as retention does, and decides
+again under that lock what to change. A partition locked that way is tried again
+rather than waited for in the queue, so one that other sessions keep in use
+throughout can make the call give up at its deadline, and the next run tries
+again. Bringing every partition in step reads the list of partitions holding
+`audit_events` `ACCESS SHARE`, so a retention run cannot drop a partition the
+call has listed before it reaches it.
+
+A new partition's bounds are UTC midnights, whatever the zone of the session
+that provisions it. A partition an earlier release cut at another zone's
+midnight still borders it, and a range from UTC midnight would overlap one cut
+west of UTC or leave a gap after one cut east of it, where an insert would find
+no partition. So a new partition starts where its predecessor ends, and ends
+where its successor starts, when that neighbour's bound lies within a day of
+the UTC midnight. `pkg/infra/postgres_audit_provisioning_test.go` holds these
+rules against PostgreSQL.
+
 ## SECURITY DEFINER functions list pg_temp last
 
 A `SECURITY DEFINER` function runs with its owner's authority. Unless its

module/MEASUREMENT_RUNBOOKS.md

@@ -70,3 +70,29 @@ provider responses into an incident channel.
 4. Re-send the corrected provider quantity with a stable reconciliation key.
 5. Verify source events, aggregate, customer history, entitlement limit, and
    provider quantity agree after the late-arrival window.
+
+## `retention`
+
+1. Read the accounts `retention` log line `scheduled run failed`. It names each
+   failed resource, the step (`load`, `provision`, `policy` or `remove`) and the
+   database's error. `saas.retention.failing` is 1 for the same resource and
+   step until a run no longer fails it, and carries no record.
+2. SQLSTATE 55P03 means retention or audit partition provisioning waited ten
+   seconds in all for its locks and gave up, or found the `audit_events` policy
+   being changed: nothing was dropped or created. Find the transaction holding
+   `audit_events`, one of its partitions or the policy's row in `pg_locks` and
+   `pg_stat_activity`. The next run, a day later or at the next restart, tries
+   again, and partitions are provisioned three months ahead, so one refusal does
+   not stop audit writes.
+3. SQLSTATE 55000 means an audit partition retention chose was renamed out of
+   its names, or a partition it or provisioning was locking was dropped, while
+   it waited for the lock, and nothing was dropped or changed. Look for the
+   session renaming or dropping audit partitions; the next run decides again
+   from the catalog as it stands.
+4. A `policy` failure is a `data_retention_policies` window that is not a
+   positive number of days, and SQLSTATE 22023 from the audit drop is a cutoff
+   later than the `audit_events` policy allows by the database's clock. Correct
+   the policy or the caller; never widen the check.
+5. Confirm the next run reports no failure, which clears the alert, and that the
+   partitions past the `audit_events` window are gone while the months ahead
+   exist.

module/services/accounts/code/pkg/business/audit_typed_integration_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"accounts/pkg/business"
+	"accounts/pkg/infra/storetx"
 )
 
 // TestAuditEventTypes_Parity verifies the DB projection reconciles exactly to
@@ -126,15 +127,21 @@ func TestQueryAuditLog_FiltersByNamespace(t *testing.T) {
 // TestAuditRetention_DropsOldPartitions proves retention actually removes data
 // now: it drops whole partitions whose range is entirely older than the cutoff
 // (which the append-only trigger would have blocked as a row DELETE), while
-// leaving current-month data intact.
+// leaving current-month data intact. The old month lies far past any window
+// the audit_events policy keeps, since a cutoff the policy does not allow is
+// refused.
 func TestAuditRetention_DropsOldPartitions(t *testing.T) {
 	clearData(t)
 	ctx := testCtx
 
 	_, org := mustUserAndOrg(t, ctx, "ret@audit-test.com", "ret-audit", "Retention Co")
 
 	require.NoError(t, testStore.WithControlPlane(ctx, func(ctx context.Context) error {
-		return testStore.EnsureAuditPartitions(ctx, 3)
+		if err := testStore.EnsureAuditPartitions(ctx, 3); err != nil {
+			return err
+		}
+		_, err := storetx.Tx(ctx).Exec(ctx, `SELECT audit_events_ensure_partition('2000-02-15'::date)`)
+		return err
 	}))
 
 	// A current-month event we expect to survive retention.
@@ -144,18 +151,17 @@ func TestAuditRetention_DropsOldPartitions(t *testing.T) {
 		})
 	}))
 
-	// Cutoff = first instant of the current month. The empty prior-month
-	// partitions (seeded at migration + EnsureAuditPartitions) are entirely
-	// older than the cutoff and must be dropped; the current partition must not.
-	now := time.Now().UTC()
-	cutoff := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)
+	// The February 2000 partition is entirely older than the cutoff and must be
+	// dropped; the current partition must not.
+	cutoff := time.Date(2000, 3, 1, 0, 0, 0, 0, time.UTC)
 	var dropped int64
 	require.NoError(t, testStore.WithControlPlane(ctx, func(ctx context.Context) error {
 		var err error
 		dropped, err = testStore.DropAuditPartitionsBefore(ctx, cutoff)
 		return err
 	}))
-	require.Positive(t, dropped, "expected at least one old empty partition to be dropped")
+	require.Positive(t, dropped, "expected the old partition to be dropped")
+	now := time.Now().UTC()
 
 	// Current-month data survives.
 	past := now.Add(-time.Hour)