From 965ad885b851957ad9f3ddec2d092dfc9a8dad06 Mon Sep 17 00:00:00 2001 From: Ferhat Elmas Date: Thu, 29 Oct 2026 20:25:16 +0100 Subject: [PATCH 1/9] feat: add db support for noncurrent lifecycle execution Signed-off-by: Ferhat Elmas --- src/storage/database/adapter.ts | 53 + src/storage/database/lifecycle.test.ts | 177 +- src/storage/database/lifecycle.ts | 123 +- src/storage/database/pg.test.ts | 1 + src/storage/database/pg.ts | 779 ++++++- src/storage/lifecycle/configuration.ts | 4 - src/storage/lifecycle/continuation.test.ts | 492 +++-- src/storage/lifecycle/continuation.ts | 318 +-- src/storage/lifecycle/counters.test.ts | 81 + src/storage/lifecycle/counters.ts | 47 + src/storage/lifecycle/evaluation.test.ts | 71 + src/storage/lifecycle/evaluation.ts | 29 + src/storage/schemas/lifecycle.ts | 104 +- .../bucket-lifecycle-configuration.test.ts | 110 +- src/test/bucket-lifecycle.test.ts | 23 +- .../noncurrent-lifecycle-database.test.ts | 1902 +++++++++++++++++ src/test/noncurrent-lifecycle-schema.test.ts | 2 +- src/test/utils/lifecycle-continuation.ts | 10 +- 18 files changed, 3664 insertions(+), 662 deletions(-) create mode 100644 src/storage/lifecycle/counters.test.ts create mode 100644 src/storage/lifecycle/counters.ts create mode 100644 src/storage/lifecycle/evaluation.test.ts create mode 100644 src/storage/lifecycle/evaluation.ts create mode 100644 src/test/noncurrent-lifecycle-database.test.ts diff --git a/src/storage/database/adapter.ts b/src/storage/database/adapter.ts index af7f2e82a..3ae7818bf 100644 --- a/src/storage/database/adapter.ts +++ b/src/storage/database/adapter.ts @@ -5,8 +5,18 @@ import { Bucket, BucketLifecycleConfiguration, BucketVersioningStatus, + EvaluateNoncurrentLifecyclePageInput, IcebergCatalog, + LifecycleArmAttemptInput, LifecycleBucket, + LifecycleClaimedState, + LifecycleClaimIdentity, + LifecycleClaimInput, + LifecycleCommitAttemptInput, + LifecycleDispatchState, + LifecycleEvaluationPage, + LifecycleReleaseClaimInput, + LifecycleState, Obj, ObjectListEntry, ObjectListingFilterMode, @@ -207,6 +217,49 @@ export interface Database { deleteLifecycleConfiguration(bucketId: string): Promise + listDueLifecycleBuckets(limit: number, options?: { inFlightOnly?: boolean }): Promise + + findLifecycleDispatchState(options: { + inFlightOnly?: boolean + runBudgetMs: number + }): Promise + + claimLifecycleBucket(input: LifecycleClaimInput): Promise + + revalidateLifecycleClaim(input: LifecycleClaimIdentity): Promise + + evaluateNoncurrentLifecyclePage( + input: EvaluateNoncurrentLifecyclePageInput + ): Promise + + armLifecycleAttempt(input: LifecycleArmAttemptInput): Promise + + revalidateLifecycleRecoveryAttempt( + input: LifecycleClaimIdentity, + attemptId: string, + leaseMs: number + ): Promise + + commitLifecycleAttempt(input: LifecycleCommitAttemptInput): Promise + + listLifecycleAbandonedArtifacts( + attemptedBefore: string, + limit: number + ): Promise<{ bucketId: string; key: string; live: boolean }[]> + + resolveLifecycleAbandonedArtifacts( + deleted: { bucketId: string; key: string }[], + failed: { bucketId: string; key: string }[] + ): Promise + + releaseLifecycleClaim(input: LifecycleReleaseClaimInput): Promise + + completeLifecycleRun( + input: LifecycleClaimIdentity, + lastResult: Record, + nextRunAt: string + ): Promise + countObjectsInBucket(bucketId: string, limit?: number): Promise deleteBucket(bucketId: string | string[]): Promise diff --git a/src/storage/database/lifecycle.test.ts b/src/storage/database/lifecycle.test.ts index 3f964f014..c5f3f9f84 100644 --- a/src/storage/database/lifecycle.test.ts +++ b/src/storage/database/lifecycle.test.ts @@ -1,11 +1,12 @@ import { compileLifecycleEvaluationRules } from '../lifecycle/configuration' import { buildEvaluateNoncurrentLifecyclePageStatement, - decodeCandidate, decodeLifecycleCandidateIdentity, mapLifecycleEvaluationPage, } from './lifecycle' +const cursor = { name: 'last', archivedAt: '2026-08-01T00:00:00.000Z' } + describe('noncurrent lifecycle evaluation query', () => { test.each([ { @@ -82,80 +83,19 @@ describe('noncurrent lifecycle evaluation query', () => { expect(statement.values?.[4]).toEqual(['-infinity']) }) - test('builds a page variant without a cursor', () => { - const statement = buildEvaluateNoncurrentLifecyclePageStatement({ - bucketId: 'bucket-a', - snapshotAt: '2026-08-15T00:00:00.000Z', - pageSize: 500, - rules: [ - { cutoffAt: '2026-07-16T00:00:00.000Z' }, - { - cutoffAt: '2026-08-08T00:00:00.000Z', - newerNoncurrentVersions: 3, - }, - ], - }) - - expect(statement.text).toContain('AS MATERIALIZED') - expect(statement.text).toContain('evaluated.newer_count >= rules.newer_noncurrent_versions') - expect(statement.text).toContain('object_row.archived_at <') - expect(statement.text).not.toContain('object_row.metadata') - expect(statement.text).not.toContain('object_row.created_at') - expect(statement.text).not.toContain('object_row.id') - expect(statement.text).toMatch( - /ORDER BY\s+object_row\.archived_at,\s+object_row\.name COLLATE "C"/ - ) - expect(statement.values).toEqual([ - 'bucket-a', - '2026-08-15T00:00:00.000Z', - '2026-08-08T00:00:00.000Z', - 500, - ['2026-07-16T00:00:00.000Z', '2026-08-08T00:00:00.000Z'], - [null, 3], - 3, - ]) - }) - - test('builds a continuation variant with the exact keyset tuple', () => { - const statement = buildEvaluateNoncurrentLifecyclePageStatement({ - bucketId: 'bucket-a', - snapshotAt: '2026-08-15T00:00:00.000123Z', - cursor: { - name: 'logs/a', - archivedAt: '2026-08-01T00:00:00.000123Z', - }, - pageSize: 20, - rules: [{ cutoffAt: '2026-08-08T00:00:00.000Z', newerNoncurrentVersions: 3 }], - }) - - expect(statement.text).toMatch( - /\(\s*object_row\.archived_at,\s*object_row\.name COLLATE "C"\s*\) > \(/ - ) - expect(statement.values).toEqual([ - 'bucket-a', - '2026-08-15T00:00:00.000123Z', - '2026-08-08T00:00:00.000Z', - '2026-08-01T00:00:00.000123Z', - 'logs/a', - 20, - ['2026-08-08T00:00:00.000Z'], - [3], - 3, - ]) - }) - test('maps page progress independently from candidate selectivity', () => { expect( mapLifecycleEvaluationPage( { raw_rows_examined: 2, + exhausted: true, candidates: [], page_end: { name: 'last', archivedAt: '2026-08-01T00:00:00.000Z', }, }, - 500 + 480 ) ).toEqual({ rawRowsExamined: 2, @@ -168,18 +108,42 @@ describe('noncurrent lifecycle evaluation query', () => { }) }) + test.each([ + [ + 'more raw rows than the page size', + { raw_rows_examined: 3, candidates: [], page_end: cursor }, + ], + [ + 'more candidates than raw rows', + { + raw_rows_examined: 0, + candidates: [{ id: 'a', name: 'n', version: null, isDeleteMarker: false }], + page_end: null, + }, + ], + ['rows without a page end', { raw_rows_examined: 1, candidates: [], page_end: null }], + ['a page end without rows', { raw_rows_examined: 0, candidates: [], page_end: cursor }], + ])('rejects %s', (_label, row) => { + expect(() => mapLifecycleEvaluationPage({ ...row, exhausted: true }, 2)).toThrow() + }) + test.each([null, 'version-a'])( 'maps the physical candidate identity with version %s', (version) => { + const id = '5a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d' expect( mapLifecycleEvaluationPage( { raw_rows_examined: 1, + exhausted: true, candidates: [ { + id, name: 'history/object.txt', version, isDeleteMarker: false, + archivedAt: '2026-08-01T00:00:00.000123Z', + rawRowNumber: 1, }, ], page_end: { @@ -187,15 +151,15 @@ describe('noncurrent lifecycle evaluation query', () => { archivedAt: '2026-08-01T00:00:00.000Z', }, }, - 500 + 480 ) ).toMatchObject({ - candidates: [{ name: 'history/object.txt', version, isDeleteMarker: false }], + candidates: [{ id, name: 'history/object.txt', version, isDeleteMarker: false }], }) } ) - test.each([0, 501])('rejects page size %s', (pageSize) => { + test.each([0, 481])('rejects page size %s', (pageSize) => { expect(() => buildEvaluateNoncurrentLifecyclePageStatement({ bucketId: 'bucket-a', @@ -203,45 +167,78 @@ describe('noncurrent lifecycle evaluation query', () => { pageSize, rules: [{ cutoffAt: '2026-08-01T00:00:00.000Z' }], }) - ).toThrow('between 1 and 500') + ).toThrow('between 1 and 480') + }) + + test.each([0, -1, 1.5, 3, Number.NaN, '1', undefined])( + 'rejects invalid candidate position %s', + (rawRowNumber) => { + expect(() => + mapLifecycleEvaluationPage( + { + raw_rows_examined: 2, + exhausted: true, + page_end: cursor, + candidates: [ + { + id: 'a', + name: 'a', + version: null, + isDeleteMarker: false, + archivedAt: cursor.archivedAt, + rawRowNumber, + }, + ], + }, + 100 + ) + ).toThrow('invalid candidate row number') + } + ) + + test.each([ + [1, 1], + [2, 1], + ])('rejects unordered candidate positions %j', (...positions) => { + expect(() => + mapLifecycleEvaluationPage( + { + raw_rows_examined: 2, + exhausted: true, + page_end: cursor, + candidates: positions.map((rawRowNumber) => ({ + id: 'a', + name: 'a', + version: null, + isDeleteMarker: false, + archivedAt: cursor.archivedAt, + rawRowNumber, + })), + }, + 100 + ) + ).toThrow('invalid candidate row number') }) }) describe('lifecycle candidate decoding', () => { const row = { - id: 'row-id', - bucketId: 'bucket-a', + id: '5a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d', name: 'history/object.txt', version: null, - isVersioned: false, isDeleteMarker: false, - metadata: { size: 12 }, - createdAt: '2026-07-01T00:00:00.000Z', - archivedAt: '2026-08-01T00:00:00.000123Z', } - test('preserves a nullable physical version and microsecond archive ordering', () => { - expect(decodeCandidate(row)).toEqual(row) - }) - - test('accepts a delete marker with null metadata and a physical UUID', () => { - const marker = { - ...row, - version: '856d94e5-5241-42ae-9c0b-d06332a7ae15', - isVersioned: true, - isDeleteMarker: true, - metadata: null, - } - expect(decodeCandidate(marker)).toEqual(marker) - }) - test.each([undefined, '', 1])( 'rejects an omitted or malformed physical version %s', (version) => { - expect(() => decodeCandidate({ ...row, version })).toThrow('candidate.version') expect(() => decodeLifecycleCandidateIdentity({ ...row, version })).toThrow( 'candidate.version' ) } ) + + test.each([undefined, '', 42])('rejects an omitted or malformed row id %s', (id) => { + expect(() => decodeLifecycleCandidateIdentity({ ...row, id })).toThrow('candidate.id') + }) }) diff --git a/src/storage/database/lifecycle.ts b/src/storage/database/lifecycle.ts index c629e4062..909a4f26d 100644 --- a/src/storage/database/lifecycle.ts +++ b/src/storage/database/lifecycle.ts @@ -1,20 +1,21 @@ import type { DatabaseStatement } from '@internal/database' import type { QueryResultRow } from 'pg' import { lifecycleCutoffTime } from '../lifecycle/configuration' +import { decodeCursor } from '../lifecycle/continuation' import { type EvaluateNoncurrentLifecyclePageInput, LIFECYCLE_MAX_NEWER_NONCURRENT_VERSIONS, LIFECYCLE_MAX_PAGE_SIZE, - type LifecycleCandidate, type LifecycleCandidateIdentity, type LifecycleEvaluationPage, - type NoncurrentLifecycleShardCursor, + type LifecycleEvaluationRule, } from '../schemas/lifecycle' interface LifecycleEvaluationResultRow extends QueryResultRow { raw_rows_examined: number candidates: unknown page_end: unknown | null + exhausted: boolean } // Builds an oldest-first page up to the least restrictive rule cutoff. Cursor ordering @@ -41,6 +42,8 @@ export function buildEvaluateNoncurrentLifecyclePageStatement( const conditions = [ 'object_row.bucket_id = $1', 'object_row.archived_at IS NOT NULL', + `object_row.name <> ''`, + `object_row.archived_at >= '0001-01-01T00:00:00Z'`, `object_row.archived_at <= ${snapshotParameter}::timestamptz`, `object_row.archived_at < ${latestCutoffParameter}::timestamptz`, ] @@ -57,13 +60,10 @@ export function buildEvaluateNoncurrentLifecyclePageStatement( } const pageSizeParameter = parameter(input.pageSize) - const cutoffsParameter = parameter(input.rules.map((rule) => timestampParameter(rule.cutoffAt))) - const newerCountsParameter = parameter( - input.rules.map((rule) => rule.newerNoncurrentVersions ?? null) - ) - const maximumNewerProbeParameter = parameter( - Math.max(1, ...input.rules.map((rule) => rule.newerNoncurrentVersions ?? 0)) - ) + const { cutoffs, newerCounts, maximumNewerProbe } = lifecycleRuleParameters(input.rules) + const cutoffsParameter = parameter(cutoffs) + const newerCountsParameter = parameter(newerCounts) + const maximumNewerProbeParameter = parameter(maximumNewerProbe) return { text: ` @@ -76,6 +76,7 @@ export function buildEvaluateNoncurrentLifecyclePageStatement( ), raw_page AS MATERIALIZED ( SELECT + object_row.id, object_row.name, object_row.version, object_row.is_delete_marker, @@ -86,7 +87,8 @@ export function buildEvaluateNoncurrentLifecyclePageStatement( LIMIT ${pageSizeParameter}::integer ), evaluated AS MATERIALIZED ( - SELECT raw_page.*, COALESCE(newer.newer_count, 0) AS newer_count + SELECT raw_page.*, COALESCE(newer.newer_count, 0) AS newer_count, + row_number() OVER (ORDER BY raw_page.archived_at, raw_page.name COLLATE "C")::integer AS raw_row_number FROM raw_page LEFT JOIN LATERAL ( SELECT count(*)::integer AS newer_count @@ -108,35 +110,43 @@ export function buildEvaluateNoncurrentLifecyclePageStatement( AND raw_page.archived_at < rules.cutoff_at ) ) AS newer ON true + ), + eligible AS MATERIALIZED ( + SELECT evaluated.* FROM evaluated + WHERE evaluated.version IS DISTINCT FROM '' + AND EXISTS ( + SELECT 1 FROM rules + WHERE evaluated.archived_at < rules.cutoff_at + AND (rules.newer_noncurrent_versions IS NULL + OR evaluated.newer_count >= rules.newer_noncurrent_versions) + ) ) SELECT (SELECT count(*)::integer FROM raw_page) AS raw_rows_examined, COALESCE(( SELECT jsonb_agg( jsonb_build_object( - 'name', evaluated.name, - 'version', evaluated.version, - 'isDeleteMarker', evaluated.is_delete_marker + 'id', eligible.id, + 'name', eligible.name, + 'version', eligible.version, + 'isDeleteMarker', eligible.is_delete_marker, + 'archivedAt', to_char(eligible.archived_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'), + 'rawRowNumber', eligible.raw_row_number ) - ORDER BY evaluated.archived_at, evaluated.name COLLATE "C" - ) - FROM evaluated - WHERE EXISTS ( - SELECT 1 - FROM rules - WHERE evaluated.archived_at < rules.cutoff_at - AND ( - rules.newer_noncurrent_versions IS NULL - OR evaluated.newer_count >= rules.newer_noncurrent_versions - ) + ORDER BY eligible.archived_at, eligible.name COLLATE "C" ) + FROM eligible ), '[]'::jsonb) AS candidates, ( - SELECT jsonb_build_object('archivedAt', raw_page.archived_at, 'name', raw_page.name) + SELECT jsonb_build_object( + 'archivedAt', to_char(raw_page.archived_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'), + 'name', raw_page.name + ) FROM raw_page ORDER BY raw_page.archived_at DESC, raw_page.name COLLATE "C" DESC LIMIT 1 - ) AS page_end + ) AS page_end, + (SELECT count(*) FROM raw_page) < ${pageSizeParameter}::integer AS exhausted `, values, } @@ -154,12 +164,30 @@ export function mapLifecycleEvaluationPage( if (!Array.isArray(row.candidates)) { throw new Error('Lifecycle evaluation returned invalid candidates') } - const candidates = row.candidates.map(decodeLifecycleCandidateIdentity) + let previousRowNumber = 0 + const candidates = row.candidates.map((value: unknown) => { + const candidate = record(value, 'candidate') + const rawRowNumber = candidate.rawRowNumber + if ( + typeof rawRowNumber !== 'number' || + !Number.isSafeInteger(rawRowNumber) || + rawRowNumber <= previousRowNumber || + rawRowNumber > rawRowsExamined + ) { + throw new Error('Lifecycle evaluation returned an invalid candidate row number') + } + previousRowNumber = rawRowNumber + return { + ...decodeLifecycleCandidateIdentity(value), + archivedAt: timestamp(candidate.archivedAt, 'candidate.archivedAt'), + rawRowNumber, + } + }) if (candidates.length > rawRowsExamined) { throw new Error('Lifecycle evaluation returned more candidates than raw rows') } - const pageEnd = row.page_end === null ? undefined : decodeCursor(row.page_end) + const pageEnd = row.page_end === null ? undefined : decodeCursor(row.page_end, 'pageEnd') if ((rawRowsExamined === 0) !== (pageEnd === undefined)) { throw new Error('Lifecycle evaluation returned an invalid page end') } @@ -168,11 +196,19 @@ export function mapLifecycleEvaluationPage( rawRowsExamined, candidates, ...(pageEnd === undefined ? {} : { pageEnd }), - exhausted: rawRowsExamined < pageSize, + exhausted: boolean(row.exhausted, 'page.exhausted'), } } -function timestampParameter(value: string): string { +export function lifecycleRuleParameters(rules: LifecycleEvaluationRule[]) { + return { + cutoffs: rules.map((rule) => timestampParameter(rule.cutoffAt)), + newerCounts: rules.map((rule) => rule.newerNoncurrentVersions ?? null), + maximumNewerProbe: Math.max(1, ...rules.map((rule) => rule.newerNoncurrentVersions ?? 0)), + } +} + +export function timestampParameter(value: string): string { if (value === '-infinity') return value const date = new Date(value) const year = date.getUTCFullYear() @@ -220,37 +256,12 @@ function validateEvaluationInput(input: EvaluateNoncurrentLifecyclePageInput) { } export function decodeLifecycleCandidateIdentity(value: unknown): LifecycleCandidateIdentity { - const candidate = record(value, 'candidate') - return { - name: string(candidate.name, 'candidate.name'), - version: nullableString(candidate.version, 'candidate.version'), - isDeleteMarker: boolean(candidate.isDeleteMarker, 'candidate.isDeleteMarker'), - } -} - -export function decodeCandidate(value: unknown): LifecycleCandidate { const candidate = record(value, 'candidate') return { id: string(candidate.id, 'candidate.id'), - bucketId: string(candidate.bucketId, 'candidate.bucketId'), name: string(candidate.name, 'candidate.name'), version: nullableString(candidate.version, 'candidate.version'), - isVersioned: boolean(candidate.isVersioned, 'candidate.isVersioned'), isDeleteMarker: boolean(candidate.isDeleteMarker, 'candidate.isDeleteMarker'), - metadata: - candidate.metadata === null - ? null - : (record(candidate.metadata, 'candidate.metadata') as Record), - createdAt: timestamp(candidate.createdAt, 'candidate.createdAt'), - archivedAt: timestamp(candidate.archivedAt, 'candidate.archivedAt'), - } -} - -function decodeCursor(value: unknown): NoncurrentLifecycleShardCursor { - const cursor = record(value, 'pageEnd') - return { - archivedAt: timestamp(cursor.archivedAt, 'pageEnd.archivedAt'), - name: string(cursor.name, 'pageEnd.name'), } } @@ -276,7 +287,7 @@ function boolean(value: unknown, label: string): boolean { } function timestamp(value: unknown, label: string): string { - const parsed = value instanceof Date ? value.toISOString() : string(value, label) + const parsed = string(value, label) if (Number.isNaN(Date.parse(parsed))) throw new Error(`${label} must be an ISO timestamp`) return parsed } diff --git a/src/storage/database/pg.test.ts b/src/storage/database/pg.test.ts index bcbc2c8c4..cbae3c9ce 100644 --- a/src/storage/database/pg.test.ts +++ b/src/storage/database/pg.test.ts @@ -1689,6 +1689,7 @@ describe('StoragePgDB lifecycle mutation permissions', () => { id: 'bucket', name: 'bucket', type: 'STANDARD', + versioning_status: 'DISABLED', lifecycle_configuration: lifecycleConfiguration, lifecycle_configuration_generation: lifecycleConfiguration === null ? null : 'generation', } diff --git a/src/storage/database/pg.ts b/src/storage/database/pg.ts index 86fa34fea..32dc16eaf 100644 --- a/src/storage/database/pg.ts +++ b/src/storage/database/pg.ts @@ -1,4 +1,5 @@ import { randomUUID } from 'node:crypto' +import { isDeepStrictEqual } from 'node:util' import { type DatabaseExecutor, type DatabaseStatement, @@ -14,7 +15,21 @@ import { hashStringToInt } from '@internal/hashing' import { logger, logSchema } from '@internal/monitoring' import { dbQueryPerformance } from '@internal/monitoring/metrics' import { ObjectMetadata } from '@storage/backend' -import { assertLifecycleSchemaReady, lifecycleConfigurationsEqual } from '@storage/lifecycle' +import { + assertLifecycleSchemaReady, + commitLifecycleBatchResults, + compileLifecycleEvaluationRules, + decodeLifecycleContinuation, + encodeLifecycleContinuation, + lifecycleConfigurationsEqual, + pendingArtifacts, + recordLifecycleMetadataDeletions, + skipLifecycleBatch, +} from '@storage/lifecycle' +import { + type LifecycleDeletedObjectRow, + lifecycleDeletionCounters, +} from '@storage/lifecycle/counters' import { DatabaseError, QueryResultRow } from 'pg' import { DatabaseEngine, getConfig } from '../../config' import { isUuid } from '../limits' @@ -22,8 +37,20 @@ import { Bucket, BucketLifecycleConfiguration, BucketVersioningStatus, + EvaluateNoncurrentLifecyclePageInput, IcebergCatalog, + LIFECYCLE_MAX_DELETE_BATCH_SIZE, + LifecycleArmAttemptInput, LifecycleBucket, + LifecycleClaimedState, + LifecycleClaimIdentity, + LifecycleClaimInput, + LifecycleCommitAttemptInput, + LifecycleContinuation, + LifecycleDispatchState, + LifecycleEvaluationPage, + LifecycleReleaseClaimInput, + LifecycleState, Obj, ObjectListEntry, ObjectListingFilterMode, @@ -46,6 +73,13 @@ import { } from './adapter' import { SelectColumnPolicy, selectColumns } from './columns' import { DBError, mapPgTransactionAbortedError, PgErrorContext } from './errors' +import { + buildEvaluateNoncurrentLifecyclePageStatement, + type LifecycleEvaluationResultRow, + lifecycleRuleParameters, + mapLifecycleEvaluationPage, + timestampParameter, +} from './lifecycle' const { databaseEngine, databaseStatementTimeout, isMultitenant, databaseHealthcheckUnscoped } = getConfig() @@ -125,13 +159,35 @@ const HEALTHCHECK_SQL = 'SELECT id from storage.buckets limit 1' const HEALTHCHECK_QUERY_OPTIONS: UnscopedQueryOptions = Object.freeze({ timeoutMs: databaseStatementTimeout, }) +const LIFECYCLE_LOCK_TIMEOUT_MS = 5000 +const IN_FLIGHT_ONLY = "AND state.continuation #> '{batch,inFlight}' IS NOT NULL" +const LIFECYCLE_BUCKET_MATCH = `bucket.type = 'STANDARD'` +const LIFECYCLE_CLAIM_IDENTITY = `state.bucket_id = $1 AND state.claim_id = $2::uuid` const LIFECYCLE_BUCKET_COLUMNS = [ 'id', 'name', 'type', + 'versioning_status', 'lifecycle_configuration', 'lifecycle_configuration_generation', ].join(',') + +interface LifecycleStateRow extends QueryResultRow { + bucket_id: string + next_run_at: Date | string | null + claim_id: string | null + claim_until: Date | string | null + continuation: unknown | null + failure_count: number +} + +interface LifecycleClaimRow + extends LifecycleStateRow, + Pick< + LifecycleBucket, + 'versioning_status' | 'lifecycle_configuration' | 'lifecycle_configuration_generation' + > {} + async function executeQuery( db: DatabaseExecutor, statement: string | DatabaseStatement, @@ -730,6 +786,7 @@ export class StoragePgDB implements Database { const bucket = await this.findBucketById(bucketId, LIFECYCLE_BUCKET_COLUMNS) assertStandardLifecycleBucket(bucket) + return bucket as LifecycleBucket } @@ -1887,18 +1944,30 @@ export class StoragePgDB implements Database { const targetColumn = by === 'name' ? 'name COLLATE "C"' : quoteIdentifier(String(by)) const conditions = ['bucket_id = $1', `${targetColumn} = ANY($2)`] + const versioned = await this.hasMigration('object-versioning-core') - if (by === 'name' && (await this.hasMigration('object-versioning-core'))) { + if (by === 'name' && versioned) { conditions.push('archived_at IS NULL') } + const orderedLock = + by === 'id' + ? `WITH locked AS MATERIALIZED ( + SELECT id FROM storage.objects + WHERE ${conditions.join(' AND ')} + ORDER BY name COLLATE "C"${versioned ? ', version COLLATE "C"' : ''} + FOR UPDATE + )` + : '' + const result = await this.runQuery('DeleteObjects', async (db, signal) => { return this.query( db, { text: ` + ${orderedLock} DELETE FROM storage.objects - WHERE ${conditions.join(' AND ')} + WHERE ${orderedLock ? 'id IN (SELECT id FROM locked)' : conditions.join(' AND ')} RETURNING * `, values: [bucketId, objectNames], @@ -3161,6 +3230,18 @@ export class StoragePgDB implements Database { await this.testLifecycleWritePermission(statement) if (unchanged) { + if (await serviceDatabase.hasMigration('lifecycle-writer-coordination')) { + await serviceDatabase.runQuery('ReconcileLifecycleState', (db, signal) => + serviceDatabase.query( + db, + { + text: 'SELECT storage.reconcile_lifecycle_state($1, false)', + values: [bucketId], + }, + signal + ) + ) + } return locked } @@ -3220,6 +3301,668 @@ export class StoragePgDB implements Database { } } } + + private async lockLifecycleObjects( + bucketId: string, + objects: Array<{ id: string; name: string; version: string | null }> + ): Promise<{ bucket: LifecycleBucket; lockedIds: string[] }> { + if (!this.options.tnx) { + throw ERRORS.InvalidRequest('Lifecycle object locks require a transaction') + } + + await this.runQuery('SetLifecycleLockTimeout', (db, signal) => + db.query( + { + text: `SELECT set_config('lock_timeout', $1, true)`, + values: [`${LIFECYCLE_LOCK_TIMEOUT_MS}ms`], + }, + { signal } + ) + ) + + try { + await this.waitObjectLocks(objects.map(({ name }) => ({ bucketId, objectName: name }))) + const bucket = await this.findBucketById(bucketId, LIFECYCLE_BUCKET_COLUMNS, { + forShare: true, + }) + assertStandardLifecycleBucket(bucket) + // Row locks also serialize current promotions with journal publication. + // Eligibility is checked in the next statement after any writer has committed. + const locked = await this.runQuery('LockLifecycleObjects', (db, signal) => + this.query<{ id: string }>( + db, + { + text: `SELECT object_row.id + FROM unnest($2::uuid[], $3::text[], $4::text[]) AS target(id, name, version) + JOIN storage.objects AS object_row ON object_row.id = target.id + WHERE object_row.bucket_id = $1 + AND (object_row.name, object_row.version) + IS NOT DISTINCT FROM (target.name, target.version) + ORDER BY object_row.name COLLATE "C", object_row.version COLLATE "C" + FOR UPDATE OF object_row`, + values: [ + bucketId, + objects.map(({ id }) => id), + objects.map(({ name }) => name), + objects.map(({ version }) => version), + ], + }, + signal + ) + ) + return { + bucket: bucket as LifecycleBucket, + lockedIds: locked.rows.map((row) => row.id), + } + } catch (error) { + if (isStorageError(ErrorCode.ResourceLocked, error)) { + throw ERRORS.LockTimeout(error.originalError as Error | undefined) + } + throw error + } + } + + async listDueLifecycleBuckets( + limit: number, + options: { inFlightOnly?: boolean } = {} + ): Promise { + const result = await this.runQuery('ListDueLifecycleBuckets', async (db, signal) => + this.query<{ bucket_id: string }>( + db, + { + text: `SELECT state.bucket_id + FROM storage.bucket_lifecycle_states AS state + JOIN storage.buckets AS bucket ON bucket.id = state.bucket_id + AND ${LIFECYCLE_BUCKET_MATCH} + WHERE state.next_run_at <= statement_timestamp() + AND (state.claim_until IS NULL OR state.claim_until < statement_timestamp()) + ${options.inFlightOnly ? IN_FLIGHT_ONLY : ''} + ORDER BY state.next_run_at, state.bucket_id + LIMIT $1`, + values: [limit], + }, + signal + ) + ) + return result.rows.map((row) => row.bucket_id) + } + + async findLifecycleDispatchState(options: { + inFlightOnly?: boolean + runBudgetMs: number + }): Promise { + const result = await this.runQuery('FindLifecycleDispatchState', async (db, signal) => { + return this.query<{ + next_dispatch_at: Date | null + max_failure_count: number + oldest_in_flight_started_at: string | null + has_abandoned_artifacts: boolean + }>( + db, + { + text: ` + SELECT MIN(GREATEST( + state.next_run_at, + CASE WHEN state.claim_until IS NOT NULL THEN LEAST( + state.claim_until, + state.last_started_at + $1::bigint * interval '1 millisecond' + ) END + )) FILTER ( + WHERE state.next_run_at IS NOT NULL + ${options.inFlightOnly ? IN_FLIGHT_ONLY : ''} + ) AS next_dispatch_at, + COALESCE(MAX(state.failure_count), 0) AS max_failure_count, + MIN(state.continuation #>> '{batch,inFlight,startedAt}') + AS oldest_in_flight_started_at, + EXISTS (SELECT 1 FROM storage.lifecycle_abandoned_artifacts) + AS has_abandoned_artifacts + FROM storage.bucket_lifecycle_states AS state + JOIN storage.buckets AS bucket + ON bucket.id = state.bucket_id + AND ${LIFECYCLE_BUCKET_MATCH} + `, + values: [options.runBudgetMs], + }, + signal + ) + }) + + const row = result.rows[0] + const startedAt = row.oldest_in_flight_started_at + return { + nextDispatchAt: row.next_dispatch_at, + maxFailureCount: row.max_failure_count, + oldestInFlightStartedAt: + startedAt && Number.isFinite(Date.parse(startedAt)) + ? new Date(startedAt).toISOString() + : null, + hasAbandonedArtifacts: row.has_abandoned_artifacts, + } + } + + async claimLifecycleBucket( + input: LifecycleClaimInput + ): Promise { + const result = await this.runQuery('ClaimLifecycleBucket', async (db, signal) => { + return this.query( + db, + { + text: ` + UPDATE storage.bucket_lifecycle_states AS state + SET claim_id = $2::uuid, + claim_until = clock_timestamp() + $3::bigint * interval '1 millisecond', + last_started_at = clock_timestamp(), + updated_at = clock_timestamp() + FROM storage.buckets AS bucket + WHERE state.bucket_id = $1 + AND state.next_run_at <= clock_timestamp() + AND (state.claim_until IS NULL OR state.claim_until < clock_timestamp()) + AND bucket.id = state.bucket_id + AND ${LIFECYCLE_BUCKET_MATCH} + RETURNING state.*, + bucket.versioning_status, + bucket.lifecycle_configuration, + bucket.lifecycle_configuration_generation + `, + values: [input.bucketId, input.claimId, input.leaseMs], + }, + signal + ) + }) + + const row = result.rows[0] + if (!row) return undefined + try { + return { + ...mapLifecycleState(row), + bucket: { + versioning_status: row.versioning_status, + lifecycle_configuration: row.lifecycle_configuration, + lifecycle_configuration_generation: row.lifecycle_configuration_generation, + }, + } + } catch (error) { + throw Object.assign(error as Error, { failureCount: row.failure_count }) + } + } + + async revalidateLifecycleClaim( + input: LifecycleClaimIdentity, + options: { attemptId?: string; forUpdate?: boolean } = {} + ): Promise { + const result = await this.runQuery('RevalidateLifecycleClaim', async (db, signal) => { + return this.query( + db, + { + text: ` + SELECT state.* + FROM storage.bucket_lifecycle_states AS state + JOIN storage.buckets AS bucket + ON bucket.id = state.bucket_id + AND ${LIFECYCLE_BUCKET_MATCH} + WHERE ${LIFECYCLE_CLAIM_IDENTITY} + AND state.claim_until > clock_timestamp() + AND ($3::text IS NULL OR state.continuation #>> '{batch,inFlight,attemptId}' = $3) + ${options.forUpdate ? 'FOR UPDATE OF state' : ''} + `, + values: [...lifecycleClaimIdentityValues(input), options.attemptId ?? null], + }, + signal + ) + }) + + return result.rows[0] ? mapLifecycleState(result.rows[0]) : undefined + } + + async evaluateNoncurrentLifecyclePage( + input: EvaluateNoncurrentLifecyclePageInput + ): Promise { + const statement = buildEvaluateNoncurrentLifecyclePageStatement(input) + const result = await this.runQuery('EvaluateNoncurrentLifecyclePage', (db, signal) => + this.query(db, statement, signal) + ) + + const row = result.rows[0] + if (!row) throw ERRORS.InternalError(undefined, 'Lifecycle evaluation returned no result') + return mapLifecycleEvaluationPage(row, input.pageSize) + } + + async armLifecycleAttempt(input: LifecycleArmAttemptInput): Promise { + if (!this.options.tnx) { + return this.withTransaction((db) => db.armLifecycleAttempt(input)) + } + const continuation = decodeLifecycleContinuation(input.continuation) + const batch = continuation.batch + if (!batch?.inFlight || batch.versions.length === 0) { + throw ERRORS.InvalidParameter('lifecycle attempt continuation') + } + if (batch.versions.length > LIFECYCLE_MAX_DELETE_BATCH_SIZE) { + throw ERRORS.InvalidParameter( + `lifecycle delete batch exceeds ${LIFECYCLE_MAX_DELETE_BATCH_SIZE} versions` + ) + } + + const serviceDatabase = this.asSuperUser() + const { bucket, lockedIds } = await serviceDatabase.lockLifecycleObjects( + input.bucketId, + batch.versions + ) + const rules = + bucket.lifecycle_configuration === null + ? [] + : compileLifecycleEvaluationRules( + bucket.lifecycle_configuration, + new Date(continuation.snapshotAt) + ) + + const { cutoffs, newerCounts, maximumNewerProbe } = lifecycleRuleParameters(rules) + + // Recheck the exact archived rows and policy under the writer locks before + // durably authorizing backend deletion. + const result = await this.runQuery('ArmLifecycleAttempt', (db, signal) => + this.query( + db, + { + text: ` + WITH targets AS MATERIALIZED ( + SELECT * + FROM unnest($14::uuid[], $6::text[], $7::text[], $8::boolean[]) + AS target(id, name, version, is_delete_marker) + ), + rules(cutoff_at, newer_noncurrent_versions) AS MATERIALIZED ( + SELECT * FROM unnest($9::timestamptz[], $10::integer[]) + ), + eligible_targets AS MATERIALIZED ( + SELECT count(*)::integer AS eligible_count + FROM targets AS target + JOIN storage.objects AS object_row ON object_row.id = target.id + WHERE object_row.id = ANY($13::uuid[]) + AND object_row.bucket_id = $1 + AND (object_row.name, object_row.version) + IS NOT DISTINCT FROM (target.name, target.version) + AND object_row.archived_at IS NOT NULL + AND object_row.is_delete_marker = target.is_delete_marker + AND object_row.archived_at <= $11::timestamptz + AND EXISTS ( + SELECT 1 FROM rules + WHERE object_row.archived_at < rules.cutoff_at + AND (rules.newer_noncurrent_versions IS NULL OR ( + SELECT count(*) FROM ( + SELECT 1 FROM storage.objects AS newer + WHERE newer.bucket_id = $1 + AND newer.name COLLATE "C" = object_row.name COLLATE "C" + AND newer.archived_at > object_row.archived_at + AND newer.archived_at <= $11::timestamptz + ORDER BY newer.archived_at LIMIT $12::integer + ) AS bounded_newer + ) >= rules.newer_noncurrent_versions) + ) + + ) + UPDATE storage.bucket_lifecycle_states AS state + SET continuation = CASE WHEN eligible_targets.eligible_count = cardinality($6::text[]) + THEN $5::jsonb ELSE state.continuation END, + claim_id = CASE WHEN eligible_targets.eligible_count = cardinality($6::text[]) + THEN state.claim_id ELSE NULL END, + claim_until = CASE WHEN eligible_targets.eligible_count = cardinality($6::text[]) + THEN clock_timestamp() + $4::bigint * interval '1 millisecond' + ELSE NULL END, + next_run_at = clock_timestamp(), + updated_at = clock_timestamp() + FROM storage.buckets AS bucket, eligible_targets + WHERE ${LIFECYCLE_CLAIM_IDENTITY} + AND state.claim_until > clock_timestamp() + AND bucket.id = state.bucket_id + AND ${LIFECYCLE_BUCKET_MATCH} + AND bucket.versioning_status <> 'DISABLED' + AND bucket.lifecycle_configuration_generation = $3::uuid + AND state.continuation #> '{batch,inFlight}' IS NULL + RETURNING state.* + `, + values: [ + ...lifecycleClaimIdentityValues(input), + continuation.generation, + input.leaseMs, + JSON.stringify(encodeLifecycleContinuation(continuation)), + batch.versions.map((version) => version.name), + batch.versions.map((version) => version.version), + batch.versions.map((version) => version.artifacts.length === 0), + cutoffs, + newerCounts, + timestampParameter(continuation.snapshotAt), + maximumNewerProbe, + lockedIds, + batch.versions.map((version) => version.id), + ], + }, + signal + ) + ) + + if (!result.rows[0]?.claim_id) return undefined + + // The metadata delete and armed journal commit together. + // No new foreign-key reference can be added after this transaction commits. + const deleted = await serviceDatabase.runQuery('DeleteLifecycleMetadata', (db, signal) => + serviceDatabase.query( + db, + { text: 'SELECT * FROM storage.delete_lifecycle_objects($1::uuid[])', values: [lockedIds] }, + signal + ) + ) + const deletedIds = new Set(deleted.rows.map((row) => row.id)) + const versions = batch.versions.filter((version) => deletedIds.has(version.id)) + const vetoed = batch.versions.length - versions.length + if (vetoed > 0) { + logSchema.warning(logger, '[Lifecycle] Kept archived versions whose deletion was vetoed', { + type: 'event', + tenantId: this.tenantId, + project: this.tenantId, + metadata: JSON.stringify({ bucketId: input.bucketId, vetoed }), + }) + } + const next = + versions.length === 0 + ? skipLifecycleBatch(continuation) + : recordLifecycleMetadataDeletions( + { ...continuation, batch: { ...batch, versions } }, + lifecycleDeletionCounters(deleted.rows, { + tenantId: this.tenantId, + bucketId: input.bucketId, + }) + ) + const saved = await serviceDatabase.runQuery('SaveLifecycleMetadataDeletions', (db, signal) => + serviceDatabase.query( + db, + { + text: `UPDATE storage.bucket_lifecycle_states AS state + SET continuation = $3::jsonb, updated_at = clock_timestamp() + WHERE ${LIFECYCLE_CLAIM_IDENTITY} + RETURNING state.*`, + values: [ + ...lifecycleClaimIdentityValues(input), + JSON.stringify(encodeLifecycleContinuation(next)), + ], + }, + signal + ) + ) + if (!saved.rows[0]) + throw new Error('Lifecycle claim disappeared while arming metadata deletions') + return mapLifecycleState(saved.rows[0]) + } + + async revalidateLifecycleRecoveryAttempt( + input: LifecycleClaimIdentity, + attemptId: string, + leaseMs: number + ): Promise { + const result = await this.runQuery('RevalidateLifecycleRecoveryAttempt', (db, signal) => + this.query( + db, + { + text: ` + UPDATE storage.bucket_lifecycle_states AS state + SET claim_until = clock_timestamp() + $4::bigint * interval '1 millisecond', + next_run_at = clock_timestamp(), + updated_at = clock_timestamp() + FROM storage.buckets AS bucket + WHERE ${LIFECYCLE_CLAIM_IDENTITY} + AND state.claim_until > clock_timestamp() + AND state.continuation #>> '{batch,inFlight,attemptId}' = $3 + AND bucket.id = state.bucket_id + AND ${LIFECYCLE_BUCKET_MATCH} + RETURNING state.* + `, + values: [...lifecycleClaimIdentityValues(input), attemptId, leaseMs], + }, + signal + ) + ) + return result.rows[0] ? mapLifecycleState(result.rows[0]) : undefined + } + + async commitLifecycleAttempt( + input: LifecycleCommitAttemptInput + ): Promise { + if (!this.options.tnx) { + return this.withTransaction((database) => database.commitLifecycleAttempt(input)) + } + + const { continuation } = input + if (continuation.batch?.inFlight?.attemptId !== input.attemptId) { + throw ERRORS.InvalidParameter('lifecycle attempt identity') + } + + const serviceDatabase = this.asSuperUser() + + const state = await serviceDatabase.revalidateLifecycleClaim(input, { + attemptId: input.attemptId, + forUpdate: true, + }) + if (!state) return undefined + if (!state.continuation || !sameFrozenLifecycleBatch(state.continuation, continuation)) { + throw ERRORS.InvalidRequest('Lifecycle attempt artifact set changed before completion') + } + + const persisted = commitLifecycleBatchResults(continuation, input.abandonPending) + + const result = await serviceDatabase.runQuery('CommitLifecycleAttemptState', (db, signal) => + serviceDatabase.query( + db, + { + text: ` + UPDATE storage.bucket_lifecycle_states AS state + SET continuation = $4::jsonb, + next_run_at = clock_timestamp(), + updated_at = clock_timestamp() + WHERE ${LIFECYCLE_CLAIM_IDENTITY} + AND continuation #>> '{batch,inFlight,attemptId}' = $3 + RETURNING * + `, + values: [ + ...lifecycleClaimIdentityValues(input), + input.attemptId, + JSON.stringify(encodeLifecycleContinuation(persisted)), + ], + }, + signal + ) + ) + if (!result.rows[0]) return undefined + if (input.abandonPending) { + const pending = pendingArtifacts(continuation) + if (pending.length > 0) { + await serviceDatabase.runQuery('PersistLifecycleAbandonedArtifacts', (db, signal) => + serviceDatabase.query( + db, + { + text: ` + INSERT INTO storage.lifecycle_abandoned_artifacts (bucket_id, key, name, version) + SELECT $1, artifact.key, artifact.name, artifact.version + FROM unnest($2::text[], $3::text[], $4::text[]) AS artifact(key, name, version) + ON CONFLICT DO NOTHING + `, + values: [ + input.bucketId, + pending.map(({ key }) => key), + pending.map(({ name }) => name), + pending.map(({ version }) => version), + ], + }, + signal + ) + ) + } + } + return mapLifecycleState(result.rows[0]) + } + + async listLifecycleAbandonedArtifacts( + attemptedBefore: string, + limit: number + ): Promise<{ bucketId: string; key: string; live: boolean }[]> { + const serviceDatabase = this.asSuperUser() + const result = await serviceDatabase.runQuery('ListLifecycleAbandonedArtifacts', (db, signal) => + serviceDatabase.query<{ bucket_id: string; key: string; live: boolean }>( + db, + { + text: ` + WITH page AS ( + SELECT bucket_id, key, name, version, attempted_at + FROM storage.lifecycle_abandoned_artifacts + WHERE attempted_at < $1::timestamptz + ORDER BY attempted_at + LIMIT $2 + ), + checked AS ( + SELECT page.bucket_id, page.key, page.attempted_at, + CASE + WHEN page.version IS NOT NULL THEN EXISTS ( + SELECT 1 FROM storage.objects o + WHERE o.bucket_id = page.bucket_id + AND o.name COLLATE "C" = page.name + AND o.version = page.version + ) + ELSE EXISTS ( + SELECT 1 FROM storage.objects o + WHERE o.bucket_id = page.bucket_id + AND o.name COLLATE "C" = page.name + AND o.version IS NULL + ) + END AS live + FROM page + ), + dropped AS ( + DELETE FROM storage.lifecycle_abandoned_artifacts AS artifact + USING checked + WHERE checked.live + AND artifact.bucket_id = checked.bucket_id + AND artifact.key = checked.key + ) + SELECT bucket_id, key, live FROM checked ORDER BY attempted_at + `, + values: [attemptedBefore, limit], + }, + signal + ) + ) + return result.rows.map((row) => ({ bucketId: row.bucket_id, key: row.key, live: row.live })) + } + + async resolveLifecycleAbandonedArtifacts( + deleted: { bucketId: string; key: string }[], + failed: { bucketId: string; key: string }[] + ): Promise { + const serviceDatabase = this.asSuperUser() + await serviceDatabase.runQuery('ResolveLifecycleAbandonedArtifacts', (db, signal) => + serviceDatabase.query( + db, + { + text: ` + WITH deleted AS ( + DELETE FROM storage.lifecycle_abandoned_artifacts AS artifact + USING unnest($1::text[], $2::text[]) AS target(bucket_id, key) + WHERE artifact.bucket_id = target.bucket_id AND artifact.key = target.key + ) + UPDATE storage.lifecycle_abandoned_artifacts AS artifact + SET attempted_at = clock_timestamp() + FROM unnest($3::text[], $4::text[]) AS target(bucket_id, key) + WHERE artifact.bucket_id = target.bucket_id AND artifact.key = target.key + `, + values: [ + deleted.map((artifact) => artifact.bucketId), + deleted.map((artifact) => artifact.key), + failed.map((artifact) => artifact.bucketId), + failed.map((artifact) => artifact.key), + ], + }, + signal + ) + ) + } + + async releaseLifecycleClaim(input: LifecycleReleaseClaimInput): Promise { + const { continuation } = input + if ((continuation === undefined || continuation?.batch?.inFlight) && input.nextRunAt === null) { + throw ERRORS.InvalidParameter('lifecycle recovery schedule') + } + const result = await this.runQuery('ReleaseLifecycleClaim', (db, signal) => + this.query( + db, + { + text: ` + UPDATE storage.bucket_lifecycle_states AS state + SET continuation = CASE WHEN $6::boolean THEN $3::jsonb ELSE continuation END, + next_run_at = CASE + WHEN NOT $6::boolean AND continuation #> '{batch,inFlight}' IS NOT NULL + THEN LEAST($4::timestamptz, $7::timestamptz) + ELSE $4::timestamptz + END, + claim_id = NULL, + claim_until = NULL, + last_completed_at = clock_timestamp(), + failure_count = CASE + WHEN $5::jsonb IS NOT NULL THEN failure_count + 1 + WHEN (CASE WHEN $6::boolean THEN $3::jsonb ELSE continuation END) #> '{batch,inFlight}' IS NULL THEN 0 + ELSE failure_count + END, + last_error = COALESCE($5::jsonb, last_error), + updated_at = clock_timestamp() + WHERE ${LIFECYCLE_CLAIM_IDENTITY} + `, + values: [ + ...lifecycleClaimIdentityValues(input), + continuation == null ? null : JSON.stringify(encodeLifecycleContinuation(continuation)), + input.nextRunAt, + input.error === undefined ? null : JSON.stringify(input.error), + continuation !== undefined, + input.maxInFlightRetryAt ?? null, + ], + }, + signal + ) + ) + return result.rowCount === 1 + } + + async completeLifecycleRun( + input: LifecycleClaimIdentity, + lastResult: Record, + nextRunAt: string + ): Promise { + const result = await this.runQuery('CompleteLifecycleRun', (db, signal) => + this.query( + db, + { + text: ` + UPDATE storage.bucket_lifecycle_states AS state + SET continuation = NULL, + next_run_at = CASE + WHEN bucket.lifecycle_configuration_generation::text + IS NOT DISTINCT FROM $4::jsonb ->> 'generation' + THEN $3::timestamptz + ELSE clock_timestamp() + END, + claim_id = NULL, + claim_until = NULL, + last_completed_at = clock_timestamp(), + last_success_at = clock_timestamp(), + last_result = $4::jsonb, + failure_count = 0, + last_error = NULL, + updated_at = clock_timestamp() + FROM storage.buckets AS bucket + WHERE ${LIFECYCLE_CLAIM_IDENTITY} + AND bucket.id = state.bucket_id + `, + values: [...lifecycleClaimIdentityValues(input), nextRunAt, JSON.stringify(lastResult)], + }, + signal + ) + ) + return result.rowCount === 1 + } } function normalizeTimeoutMs(timeoutMs: number | undefined): number | undefined { @@ -3495,3 +4238,33 @@ async function rollbackSavepoint(tnx: DatabaseTransaction, savepoint: string): P await tnx.query(`ROLLBACK TO SAVEPOINT ${savepoint}`) await tnx.query(`RELEASE SAVEPOINT ${savepoint}`) } + +function mapLifecycleState(row: LifecycleStateRow): LifecycleState { + return { + bucketId: row.bucket_id, + claimId: row.claim_id, + continuation: row.continuation === null ? null : decodeLifecycleContinuation(row.continuation), + failureCount: row.failure_count, + } +} + +function lifecycleClaimIdentityValues(input: LifecycleClaimIdentity): unknown[] { + return [input.bucketId, input.claimId] +} + +function sameFrozenLifecycleBatch( + persisted: LifecycleContinuation, + proposed: LifecycleContinuation +): boolean { + const frozen = ({ batch }: LifecycleContinuation) => + batch && [ + batch.keySeparator, + batch.versions.map(({ id, name, version, artifacts }) => [ + id, + name, + version, + artifacts.length, + ]), + ] + return isDeepStrictEqual(frozen(persisted), frozen(proposed)) +} diff --git a/src/storage/lifecycle/configuration.ts b/src/storage/lifecycle/configuration.ts index 8354991f5..b253de213 100644 --- a/src/storage/lifecycle/configuration.ts +++ b/src/storage/lifecycle/configuration.ts @@ -111,10 +111,6 @@ export function lifecycleConfigurationToS3( } } -export function hasEnabledLifecycleRule(configuration: BucketLifecycleConfiguration): boolean { - return configuration.rules.some((rule) => rule.status === 'Enabled') -} - export function compileLifecycleEvaluationRules( configuration: BucketLifecycleConfiguration, snapshotAt: Date diff --git a/src/storage/lifecycle/continuation.test.ts b/src/storage/lifecycle/continuation.test.ts index 9696357f7..eb9312ce2 100644 --- a/src/storage/lifecycle/continuation.test.ts +++ b/src/storage/lifecycle/continuation.test.ts @@ -1,35 +1,28 @@ import { randomUUID } from 'node:crypto' import { withOptionalVersion } from '@storage/backend/adapter' import { - advanceLifecycleEvaluationPage, + armedContinuation, + newContinuation, + stagedContinuation, +} from '../../test/utils/lifecycle-continuation' +import { + advanceEmptyLifecyclePage, armLifecycleAttempt, commitLifecycleBatchResults, - completeLifecycleBatch, - createLifecycleContinuation, decodeLifecycleContinuation, - filterStagedLifecycleBatch, + encodeLifecycleContinuation, freezeLifecycleBatchVersion, LifecycleContinuationDecodeError, recordLifecycleArtifactOutcomes, - refreshStagedLifecycleBatchArtifacts, + recordLifecycleMetadataDeletions, + skipLifecycleBatch, stageLifecycleBatch, } from './continuation' function validContinuation() { const generation = randomUUID() return { - continuationVersion: 1, - runId: randomUUID(), - trigger: 'scheduled', - generation, - snapshotAt: '2026-08-06T00:00:00.000Z', - mode: 'DELETE', - topology: { - scanKind: 'NONCURRENT', - epoch: '1', - shardId: 0, - shardCount: 1, - }, + ...newContinuation(generation, { snapshotAt: '2026-08-06T00:00:00.000Z' }), pageEnd: { name: 'logs/example.json', archivedAt: '2026-08-05T12:00:00.000Z', @@ -43,19 +36,17 @@ function validContinuation() { batchesCompleted: 0, }, batch: { + keySeparator: '/' as const, versions: [ { + id: randomUUID(), name: 'logs/example.json', - version: randomUUID(), - artifacts: [ - { key: 'logs/example.json/version', outcome: 'UNRESOLVED' }, - { key: 'logs/example.json/version.info', outcome: 'UNRESOLVED' }, - ], + version: 'version', + artifacts: [{ outcome: 'UNRESOLVED' }, { outcome: 'UNRESOLVED' }], }, ], inFlight: { attemptId: randomUUID(), - authorizedGeneration: generation, startedAt: '2026-08-06T00:01:00.000Z', }, }, @@ -65,7 +56,50 @@ function validContinuation() { describe('decodeLifecycleContinuation', () => { test('decodes a bounded v1 continuation', () => { const input = validContinuation() - expect(decodeLifecycleContinuation(input)).toEqual(input) + const decoded = decodeLifecycleContinuation(input) + expect(decoded.batch?.versions[0].artifacts).toEqual([ + { key: 'logs/example.json/version', outcome: 'UNRESOLVED' }, + { key: 'logs/example.json/version.info', outcome: 'UNRESOLVED' }, + ]) + expect(decoded).toMatchObject({ ...input, batch: { inFlight: input.batch.inFlight } }) + }) + + test('ignores a stored artifact key and derives it from the name and version', () => { + const input = validContinuation() + input.batch.versions[0].artifacts[0] = { key: 'tampered', outcome: 'UNRESOLVED' } as never + expect(decodeLifecycleContinuation(input).batch?.versions[0].artifacts[0].key).toBe( + 'logs/example.json/version' + ) + }) + + test('encodes without artifact keys and decodes back to the same continuation', () => { + const decoded = decodeLifecycleContinuation(validContinuation()) + const encoded = JSON.parse(JSON.stringify(encodeLifecycleContinuation(decoded))) + expect(encoded.batch.versions[0].artifacts).toEqual([ + { outcome: 'UNRESOLVED' }, + { outcome: 'UNRESOLVED' }, + ]) + expect(decodeLifecycleContinuation(encoded)).toEqual(decoded) + }) + + test('recovers keys using the separator frozen in the journal', () => { + const input = validContinuation() + const encoded = { ...input, batch: { ...input.batch, keySeparator: '-$v-' } } + const recovered = decodeLifecycleContinuation(encoded) + expect(recovered.batch?.versions[0].artifacts.map(({ key }) => key)).toEqual([ + 'logs/example.json-$v-version', + 'logs/example.json-$v-version.info', + ]) + expect(decodeLifecycleContinuation(encodeLifecycleContinuation(recovered))).toEqual(recovered) + expect(() => + decodeLifecycleContinuation({ ...input, batch: { ...input.batch, keySeparator: undefined } }) + ).toThrow('batch.keySeparator') + }) + + test('encodes a continuation without a batch unchanged', () => { + const { batch: _batch, ...input } = validContinuation() + const decoded = decodeLifecycleContinuation(input) + expect(encodeLifecycleContinuation(decoded)).toEqual(decoded) }) test('rejects a resolved artifact with a missing companion', () => { @@ -79,26 +113,14 @@ describe('decodeLifecycleContinuation', () => { ) }) - test('round-trips an armed legacy physical version without replacing null with a sentinel', () => { - const input = validContinuation() - const legacy = freezeLifecycleBatchVersion({ - name: 'logs/example.json', - version: null, - isDeleteMarker: false, - }) - const journal = { ...input, batch: { ...input.batch, versions: [legacy] } } - expect(decodeLifecycleContinuation(JSON.parse(JSON.stringify(journal)))).toEqual(journal) - expect(legacy.artifacts).toEqual([ - { key: 'logs/example.json', outcome: 'UNRESOLVED' }, - { key: 'logs/example.json.info', outcome: 'UNRESOLVED' }, - ]) - }) - test('rejects an omitted physical version selector in a durable batch', () => { const input = validContinuation() const { version: _version, ...entry } = input.batch.versions[0] expect(() => - decodeLifecycleContinuation({ ...input, batch: { ...input.batch, versions: [entry] } }) + decodeLifecycleContinuation({ + ...input, + batch: { ...input.batch, versions: [{ ...entry, artifacts: entry.artifacts.slice(0, 1) }] }, + }) ).toThrow('batch.versions[0].version') }) @@ -124,6 +146,18 @@ describe('decodeLifecycleContinuation', () => { delete (value as { pageEnd?: unknown }).pageEnd }, ], + [ + 'invalid snapshot timestamp', + (value: ReturnType) => { + value.snapshotAt = 'yesterday' + }, + ], + [ + 'negative counter', + (value: ReturnType) => { + value.counters.versionsExamined = -1 + }, + ], [ 'unknown artifact outcome', (value: ReturnType) => { @@ -136,28 +170,72 @@ describe('decodeLifecycleContinuation', () => { expect(() => decodeLifecycleContinuation(value)).toThrow(LifecycleContinuationDecodeError) }) + test.each([ + '00000000-0000-0000-0000-000000000000', + '6ba7b810-9dad-31d1-70b4-00c04fd430c8', + ] as const)('decodes the database uuid %s as a batch version id', (id) => { + const input = validContinuation() + input.batch.versions[0].id = id + expect(decodeLifecycleContinuation(input).batch?.versions[0].id).toBe(id) + }) + + test.each([ + '6ba7b810-9dad-31d1-70b4-00c04fd430c', + '6ba7b810-9dad-31d1-70b4-00c04fd430cg', + ] as const)('rejects the malformed batch version id %s', (id) => { + const input = validContinuation() + input.batch.versions[0].id = id + expect(() => decodeLifecycleContinuation(input)).toThrow('batch.versions[0].id must be a UUID') + }) + + test('round-trips every journal shape through the encoded JSON form', () => { + const pageEnd = { name: 'a/b', archivedAt: '2026-08-05T12:00:00.000Z' } + const versions = [ + { id: randomUUID(), name: 'a/b', version: randomUUID(), isDeleteMarker: false }, + { id: randomUUID(), name: 'a/b', version: null, isDeleteMarker: false }, + { id: randomUUID(), name: 'a/b', version: 'null', isDeleteMarker: false }, + { id: randomUUID(), name: 'd/-$v-x.info/é', version: randomUUID(), isDeleteMarker: false }, + { id: randomUUID(), name: 'a/b', version: randomUUID(), isDeleteMarker: true }, + ] + const armed = armedContinuation(randomUUID(), versions, pageEnd) + expect(armed.batch!.versions[1]).toMatchObject({ + version: null, + artifacts: [{ key: 'a/b', outcome: 'UNRESOLVED' }], + }) + const [data, info] = armed.batch!.versions[0].artifacts + const recorded = recordLifecycleArtifactOutcomes( + armed, + new Map([ + [data.key, { outcome: 'UNRESOLVED' as const, error: 'timeout' }], + [info.key, { outcome: 'FAILED' as const, error: 'denied' }], + [armed.batch!.versions[1].artifacts[0].key, { outcome: 'DELETED' as const }], + [armed.batch!.versions[2].artifacts[0].key, { outcome: 'ABSENT' as const }], + ]) + ) + for (const continuation of [ + newContinuation(randomUUID()), + { ...newContinuation(randomUUID()), cursor: pageEnd }, + stagedContinuation(randomUUID(), versions, pageEnd), + armed, + recorded, + commitLifecycleBatchResults(recorded), + ]) { + const encoded = JSON.parse(JSON.stringify(encodeLifecycleContinuation(continuation))) + expect(JSON.stringify(encoded)).not.toContain('"key"') + expect(decodeLifecycleContinuation(encoded)).toEqual(continuation) + } + }) + test('bounds the durable logical-version batch', () => { const value = validContinuation() - value.batch.versions = Array.from({ length: 501 }, () => value.batch.versions[0]) - expect(() => decodeLifecycleContinuation(value)).toThrow('at most 500 entries') + value.batch.versions = Array.from({ length: 481 }, () => value.batch.versions[0]) + expect(() => decodeLifecycleContinuation(value)).toThrow('at most 480 entries') }) }) describe('lifecycle continuation transitions', () => { test('rejects empty staging and advances pages without eligible candidates', () => { - const initial = createLifecycleContinuation({ - runId: randomUUID(), - trigger: 'scheduled', - generation: randomUUID(), - snapshotAt: '2026-08-15T00:00:00.000Z', - mode: 'DELETE', - topology: { - scanKind: 'NONCURRENT', - epoch: '1', - shardId: 0, - shardCount: 1, - }, - }) + const initial = newContinuation(randomUUID(), { snapshotAt: '2026-08-15T00:00:00.000Z' }) const page = { rawRowsExamined: 1, candidates: [], @@ -173,21 +251,85 @@ describe('lifecycle continuation transitions', () => { }) ).toThrow('Lifecycle batch cannot be empty') - expect(advanceLifecycleEvaluationPage(initial, page)).toEqual({ + expect(advanceEmptyLifecyclePage(initial, page)).toEqual({ ...initial, cursor: page.pageEnd, counters: { ...initial.counters, versionsExamined: 1 }, }) + + expect(() => + advanceEmptyLifecyclePage(initial, { + ...page, + candidates: [ + { + id: randomUUID(), + name: page.pageEnd.name, + version: randomUUID(), + isDeleteMarker: false, + archivedAt: page.pageEnd.archivedAt, + rawRowNumber: 1, + }, + ], + }) + ).toThrow('Lifecycle pages with eligible candidates must be staged before advancing') }) - test('keeps null and literal null-string identities separate through filtering and recovery', () => { + test('stages a batch with distinct examined and eligible counts, rejects restaging, and drops pageEnd on advance', () => { + const initial = newContinuation(randomUUID(), { snapshotAt: '2026-08-15T00:00:00.000Z' }) + const versions = [0, 1].map(() => + freezeLifecycleBatchVersion({ + id: randomUUID(), + name: 'logs/example.json', + version: randomUUID(), + isDeleteMarker: false, + }) + ) + const pageEnd = { name: 'logs/example.json', archivedAt: '2026-08-02T00:00:00.000Z' } + const staged = stageLifecycleBatch(initial, { pageEnd, rawRowsExamined: 5, versions }) + + expect(staged.counters).toMatchObject({ versionsExamined: 5, versionsEligible: 2 }) + expect(() => stageLifecycleBatch(staged, { pageEnd, rawRowsExamined: 5, versions })).toThrow( + 'already has a staged batch' + ) + + const armed = armLifecycleAttempt(staged, { + attemptId: randomUUID(), + startedAt: '2026-08-15T00:01:00.000Z', + }) + const resolved = recordLifecycleArtifactOutcomes( + armed, + new Map( + versions.flatMap((v) => v.artifacts.map((a) => [a.key, { outcome: 'DELETED' as const }])) + ) + ) + const advanced = commitLifecycleBatchResults(resolved) + expect(advanced).not.toHaveProperty('pageEnd') + expect(advanced).not.toHaveProperty('batch') + expect(advanced.cursor).toEqual(pageEnd) + }) + + test('skips a staged batch to its page end without counting it as completed', () => { + const pageEnd = { name: 'logs/example.json', archivedAt: '2026-08-02T00:00:00.000Z' } + const staged = stagedContinuation( + randomUUID(), + [{ id: randomUUID(), name: pageEnd.name, version: randomUUID(), isDeleteMarker: false }], + pageEnd + ) + const { batch: _batch, pageEnd: _pageEnd, ...rest } = staged + + expect(skipLifecycleBatch(staged)).toEqual({ ...rest, cursor: pageEnd }) + }) + + test('keeps null and literal null-string identities separate through recovery', () => { const initial = validContinuation() const legacy = freezeLifecycleBatchVersion({ + id: randomUUID(), name: 'logs/example.json', version: null, isDeleteMarker: false, }) const named = freezeLifecycleBatchVersion({ + id: randomUUID(), name: legacy.name, version: 'null', isDeleteMarker: false, @@ -195,35 +337,35 @@ describe('lifecycle continuation transitions', () => { const staged = decodeLifecycleContinuation({ ...initial, counters: { ...initial.counters, versionsExamined: 2, versionsEligible: 2 }, - batch: { versions: [legacy, named] }, + batch: { keySeparator: '/', versions: [legacy, named] }, }) - expect(filterStagedLifecycleBatch(staged, [legacy]).batch?.versions).toEqual([legacy]) - expect(filterStagedLifecycleBatch(staged, [named]).batch?.versions).toEqual([named]) - const armed = armLifecycleAttempt(staged, initial.batch.inFlight) const recorded = recordLifecycleArtifactOutcomes( armed, new Map(named.artifacts.map(({ key }) => [key, { outcome: 'ABSENT' as const }])) ) - const recovered = commitLifecycleBatchResults(recorded, [legacy], { - objectVersions: 1, - deleteMarkers: 0, - bytes: 12, + const recovered = commitLifecycleBatchResults(recorded) + expect(recovered.batch).toEqual({ + keySeparator: '/', + versions: [legacy], + inFlight: initial.batch.inFlight, }) - expect(recovered.batch).toEqual({ versions: [legacy], inFlight: initial.batch.inFlight }) expect(recovered.cursor).toBeUndefined() }) test('freezes exact physical keys using the backend locator helper', () => { const physicalKey = withOptionalVersion('logs/example.json', 'version-a') + const id = randomUUID() expect( freezeLifecycleBatchVersion({ + id, name: 'logs/example.json', version: 'version-a', isDeleteMarker: false, }) ).toEqual({ + id, name: 'logs/example.json', version: 'version-a', artifacts: [ @@ -234,63 +376,53 @@ describe('lifecycle continuation transitions', () => { expect( freezeLifecycleBatchVersion({ + id: randomUUID(), name: 'deleted', version: randomUUID(), isDeleteMarker: true, }).artifacts ).toEqual([]) + + expect( + freezeLifecycleBatchVersion({ + id: randomUUID(), + name: 'logs/example.json', + version: null, + isDeleteMarker: false, + }).artifacts + ).toEqual([{ key: 'logs/example.json', outcome: 'UNRESOLVED' }]) }) test('requires an armed attempt to complete a delete-marker batch', () => { const initial = validContinuation() const marker = freezeLifecycleBatchVersion({ + id: randomUUID(), name: initial.pageEnd.name, version: randomUUID(), isDeleteMarker: true, }) const staged = decodeLifecycleContinuation({ ...initial, - batch: { versions: [marker] }, + batch: { keySeparator: '/', versions: [marker] }, }) - expect(() => completeLifecycleBatch(staged)).toThrow('must be armed before completing') + expect(() => commitLifecycleBatchResults(staged)).toThrow('must be armed before committing') + expect(() => recordLifecycleArtifactOutcomes(staged, new Map())).toThrow( + 'must be armed before recording' + ) const armed = armLifecycleAttempt(staged, initial.batch.inFlight) - const completed = completeLifecycleBatch(armed, { - objectVersions: 0, - deleteMarkers: 1, - bytes: 0, - }) + const completed = commitLifecycleBatchResults(armed) expect(completed.cursor).toEqual(staged.pageEnd) expect(completed.batch).toBeUndefined() - expect(completed.counters).toEqual({ - ...staged.counters, - deleteMarkersDeleted: 1, - batchesCompleted: 1, - }) - - const filtered = filterStagedLifecycleBatch(staged, []) - expect(filtered.cursor).toEqual(staged.pageEnd) - expect(filtered.batch).toBeUndefined() - expect(filtered.counters).toEqual({ ...staged.counters, batchesCompleted: 1 }) + expect(completed.counters).toEqual({ ...staged.counters, batchesCompleted: 1 }) }) test('keeps one stable attempt as compensation authority across generation replacement', () => { const generation = randomUUID() - const initial = createLifecycleContinuation({ - runId: randomUUID(), - trigger: 'scheduled', - generation, - snapshotAt: '2026-08-15T00:00:00.000Z', - mode: 'DELETE', - topology: { - scanKind: 'NONCURRENT', - epoch: '1', - shardId: 0, - shardCount: 1, - }, - }) + const initial = newContinuation(generation, { snapshotAt: '2026-08-15T00:00:00.000Z' }) const candidate = freezeLifecycleBatchVersion({ + id: randomUUID(), name: 'logs/example.json', version: randomUUID(), isDeleteMarker: false, @@ -302,105 +434,51 @@ describe('lifecycle continuation transitions', () => { }) const attempt = { attemptId: randomUUID(), - authorizedGeneration: generation, startedAt: '2026-08-15T00:01:00.000Z', } const armed = armLifecycleAttempt(staged, attempt) - expect(armLifecycleAttempt(armed, attempt)).toBe(armed) - expect(() => completeLifecycleBatch(armed)).toThrow('artifact compensation') + expect(commitLifecycleBatchResults(armed).batch?.versions).toHaveLength(1) const partiallyResolved = recordLifecycleArtifactOutcomes( armed, - new Map([[candidate.artifacts[0].key, { outcome: 'DELETED' }]]) + new Map([[candidate.artifacts[1].key, { outcome: 'DELETED' }]]) ) expect(partiallyResolved.batch?.inFlight).toEqual(attempt) - expect(() => completeLifecycleBatch(partiallyResolved)).toThrow('artifact compensation') + expect(commitLifecycleBatchResults(partiallyResolved).batch?.versions).toHaveLength(1) const resolved = recordLifecycleArtifactOutcomes( partiallyResolved, - new Map([[candidate.artifacts[1].key, { outcome: 'ABSENT' }]]) + new Map([[candidate.artifacts[0].key, { outcome: 'ABSENT' }]]) ) - expect(completeLifecycleBatch(resolved)).toMatchObject({ + expect(commitLifecycleBatchResults(resolved)).toMatchObject({ cursor: staged.pageEnd, counters: { versionsExamined: 1, versionsEligible: 1, batchesCompleted: 1 }, }) }) - test('refreshes the exact artifact set from the locked row shape before arming', () => { - const generation = randomUUID() - const version = randomUUID() - const staged = stageLifecycleBatch( - createLifecycleContinuation({ - runId: randomUUID(), - trigger: 'scheduled', - generation, - snapshotAt: '2026-08-15T00:00:00.000Z', - mode: 'DELETE', - topology: { - scanKind: 'NONCURRENT', - epoch: '1', - shardId: 0, - shardCount: 1, - }, - }), - { - pageEnd: { - name: 'deleted', - archivedAt: '2026-08-02T00:00:00.000Z', - }, - rawRowsExamined: 1, - versions: [ - { - name: 'deleted', - version, - artifacts: [ - { key: 'stale-artifact', outcome: 'UNRESOLVED' }, - { key: 'stale-artifact.info', outcome: 'UNRESOLVED' }, - ], - }, - ], - } - ) - - expect( - refreshStagedLifecycleBatchArtifacts(staged, [ - { name: 'deleted', version, isDeleteMarker: true }, - ]).batch?.versions - ).toEqual([{ name: 'deleted', version, artifacts: [] }]) - }) - - test('rejects attempt replacement and cursor advancement with unresolved artifacts', () => { + test('rejects replacing an armed attempt', () => { const continuation = decodeLifecycleContinuation(validContinuation()) expect(() => armLifecycleAttempt(continuation, { attemptId: randomUUID(), - authorizedGeneration: continuation.generation, startedAt: '2026-08-15T00:01:00.000Z', }) - ).toThrow('different armed attempt') - expect(() => completeLifecycleBatch(continuation)).toThrow('artifact compensation') + ).toThrow('already has an armed attempt') }) - test('retains partial artifact success and records completed-version counters immediately', () => { + test('retains a version whose data artifact is unresolved even if the info sibling was deleted', () => { const continuation = decodeLifecycleContinuation(validContinuation()) const version = continuation.batch!.versions[0] const partial = recordLifecycleArtifactOutcomes( continuation, new Map([ - [version.artifacts[0].key, { outcome: 'DELETED' as const }], - [version.artifacts[1].key, { outcome: 'FAILED' as const, error: 'retry me' }], + [version.artifacts[0].key, { outcome: 'UNRESOLVED' as const, error: 'retry me' }], + [version.artifacts[1].key, { outcome: 'DELETED' as const }], ]) ) - expect(() => completeLifecycleBatch(partial)).toThrow('artifact compensation') - expect( - commitLifecycleBatchResults(partial, [version], { - objectVersions: 2, - deleteMarkers: 1, - bytes: 30, - }) - ).toMatchObject({ + expect(commitLifecycleBatchResults(partial)).toMatchObject({ batch: { inFlight: partial.batch!.inFlight, versions: [ @@ -408,52 +486,67 @@ describe('lifecycle continuation transitions', () => { name: version.name, version: version.version, artifacts: [ + expect.objectContaining({ outcome: 'UNRESOLVED' }), expect.objectContaining({ outcome: 'DELETED' }), - expect.objectContaining({ outcome: 'FAILED' }), ], }, ], }, - counters: { - objectVersionsDeleted: 2, - deleteMarkersDeleted: 1, - bytesDeleted: 30, - batchesCompleted: 0, - }, + counters: { batchesCompleted: 0 }, }) }) test.each([ - ['UNRESOLVED', 'UNRESOLVED'], - ['DELETED', 'UNRESOLVED'], - ['DELETED', 'FAILED'], - ['ABSENT', 'FAILED'], - ])('rejects dropping %s/%s artifacts while retaining another version', (main, sidecar) => { - const input = validContinuation() - const retained = input.batch.versions[0] - input.counters.versionsExamined = 2 - input.counters.versionsEligible = 2 - input.batch.versions.push({ - name: 'logs/other.json', - version: randomUUID(), - artifacts: [ - { key: 'logs/other.json/version', outcome: main }, - { key: 'logs/other.json/version.info', outcome: sidecar }, - ], + ['data FAILED with info DELETED', 'FAILED', 'DELETED'], + ['data DELETED with info FAILED', 'DELETED', 'FAILED'], + ['data ABSENT with info FAILED', 'ABSENT', 'FAILED'], + ] as const)('retains a failed artifact for retry: %s', (_label, data, info) => { + const continuation = decodeLifecycleContinuation(validContinuation()) + const version = continuation.batch!.versions[0] + const recorded = recordLifecycleArtifactOutcomes( + continuation, + new Map([ + [version.artifacts[0].key, { outcome: data, error: 'x' }], + [version.artifacts[1].key, { outcome: info, error: 'x' }], + ]) + ) + + expect(commitLifecycleBatchResults(recorded)).toMatchObject({ + batch: { versions: [expect.objectContaining({ id: version.id })] }, + counters: { batchesCompleted: 0 }, }) - const continuation = decodeLifecycleContinuation(input) + }) - expect(() => - commitLifecycleBatchResults(continuation, [retained], { + test('saturates the cumulative byte counter at the safe integer limit', () => { + const input = validContinuation() + input.counters.bytesDeleted = Number.MAX_SAFE_INTEGER - 1 + const continuation = decodeLifecycleContinuation(input) + expect( + recordLifecycleMetadataDeletions(continuation, { objectVersions: 1, deleteMarkers: 0, - bytes: 12, - }) - ).toThrow('artifact compensation') - expect(continuation).toEqual(input) + bytes: 10, + }).counters.bytesDeleted + ).toBe(Number.MAX_SAFE_INTEGER) + }) + + test('abandonPending drains FAILED and UNRESOLVED artifacts and advances the cursor', () => { + const armed = decodeLifecycleContinuation(validContinuation()) + const [data, info] = armed.batch!.versions[0].artifacts + const recorded = recordLifecycleArtifactOutcomes( + armed, + new Map([ + [data.key, { outcome: 'FAILED' as const, error: 'not deleted' }], + [info.key, { outcome: 'UNRESOLVED' as const }], + ]) + ) + const abandoned = commitLifecycleBatchResults(recorded, true) + expect(abandoned.batch).toBeUndefined() + expect(abandoned.cursor).toEqual(armed.pageEnd) + expect(abandoned.counters.batchesCompleted).toBe(armed.counters.batchesCompleted + 1) }) - test('drains definitive all-failed versions and can filter an unarmed staged batch', () => { + test('retains definitive all-failed versions for retry', () => { const armed = decodeLifecycleContinuation(validContinuation()) const failed = recordLifecycleArtifactOutcomes( armed, @@ -464,18 +557,9 @@ describe('lifecycle continuation transitions', () => { ]) ) ) - expect(completeLifecycleBatch(failed)).toMatchObject({ - cursor: armed.pageEnd, - counters: { batchesCompleted: 1 }, - }) - - const staged = decodeLifecycleContinuation({ - ...validContinuation(), - batch: { versions: armed.batch!.versions }, - }) - expect(filterStagedLifecycleBatch(staged, [])).toMatchObject({ - cursor: staged.pageEnd, - counters: { batchesCompleted: 1 }, + expect(commitLifecycleBatchResults(failed)).toMatchObject({ + batch: { versions: [expect.objectContaining({ id: armed.batch!.versions[0].id })] }, + counters: { batchesCompleted: 0 }, }) }) }) diff --git a/src/storage/lifecycle/continuation.ts b/src/storage/lifecycle/continuation.ts index 21828d910..4b08ad0ad 100644 --- a/src/storage/lifecycle/continuation.ts +++ b/src/storage/lifecycle/continuation.ts @@ -1,4 +1,4 @@ -import { withOptionalVersion } from '../backend/adapter' +import { FILE_VERSION_SEPARATOR, PATH_SEPARATOR, SEPARATOR } from '../backend/adapter' import { LIFECYCLE_CONTINUATION_VERSION, LIFECYCLE_MAX_PAGE_SIZE, @@ -8,18 +8,12 @@ import { type LifecycleContinuation, type LifecycleContinuationBatch, type LifecycleEvaluationPage, - type LifecycleExecutionMode, type LifecycleInFlightAttempt, type LifecycleRunCounters, - type LifecycleShardTopology, - type LifecycleTrigger, - type NoncurrentLifecycleShardCursor, + type NoncurrentLifecycleCursor, } from '../schemas/lifecycle' -const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i -const BIGINT = /^-?\d+$/ -const MIN_SIGNED_BIGINT = -(1n << 63n) -const MAX_SIGNED_BIGINT = (1n << 63n) - 1n +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i interface LifecycleDeletionCounts { objectVersions: number @@ -36,11 +30,8 @@ export class LifecycleContinuationDecodeError extends Error { export function createLifecycleContinuation(input: { runId: string - trigger: LifecycleTrigger generation: string snapshotAt: string - mode: LifecycleExecutionMode - topology: LifecycleShardTopology }): LifecycleContinuation { return decodeLifecycleContinuation({ continuationVersion: LIFECYCLE_CONTINUATION_VERSION, @@ -59,23 +50,46 @@ export function createLifecycleContinuation(input: { export function freezeLifecycleBatchVersion( candidate: LifecycleCandidateIdentity ): LifecycleBatchVersion { - const physicalKey = withOptionalVersion(candidate.name, candidate.version ?? undefined) return { + id: candidate.id, name: candidate.name, version: candidate.version, artifacts: candidate.isDeleteMarker ? [] - : [ - { key: physicalKey, outcome: 'UNRESOLVED' }, - { key: `${physicalKey}.info`, outcome: 'UNRESOLVED' }, - ], + : lifecycleArtifactKeys(candidate.name, candidate.version).map((key) => ({ + key, + outcome: 'UNRESOLVED' as const, + })), + } +} + +function lifecycleArtifactKeys( + name: string, + version: string | null, + separator = SEPARATOR +): string[] { + const key = version ? `${name}${separator}${version}` : name + return version ? [key, `${key}.info`] : [key] +} + +export function encodeLifecycleContinuation(continuation: LifecycleContinuation): unknown { + if (!continuation.batch) return continuation + return { + ...continuation, + batch: { + ...continuation.batch, + versions: continuation.batch.versions.map((version) => ({ + ...version, + artifacts: version.artifacts.map(({ key: _key, ...artifact }) => artifact), + })), + }, } } export function stageLifecycleBatch( continuation: LifecycleContinuation, input: { - pageEnd: NoncurrentLifecycleShardCursor + pageEnd: NoncurrentLifecycleCursor rawRowsExamined: number versions: LifecycleBatchVersion[] } @@ -86,13 +100,6 @@ export function stageLifecycleBatch( if (input.versions.length === 0) { throw new Error('Lifecycle batch cannot be empty') } - if ( - !Number.isSafeInteger(input.rawRowsExamined) || - input.rawRowsExamined < input.versions.length || - input.rawRowsExamined > LIFECYCLE_MAX_PAGE_SIZE - ) { - throw new Error('Lifecycle batch does not fit its raw page') - } return decodeLifecycleContinuation({ ...continuation, @@ -102,19 +109,19 @@ export function stageLifecycleBatch( versionsExamined: continuation.counters.versionsExamined + input.rawRowsExamined, versionsEligible: continuation.counters.versionsEligible + input.versions.length, }, - batch: { versions: input.versions }, + batch: { keySeparator: SEPARATOR, versions: input.versions }, }) } -export function advanceLifecycleEvaluationPage( +export function advanceEmptyLifecyclePage( continuation: LifecycleContinuation, page: LifecycleEvaluationPage ): LifecycleContinuation { if (continuation.batch !== undefined) { throw new Error('Lifecycle continuation already has a staged batch') } - if (page.rawRowsExamined > 0 && page.pageEnd === undefined) { - throw new Error('A non-empty lifecycle page requires pageEnd') + if (page.candidates.length !== 0) { + throw new Error('Lifecycle pages with eligible candidates must be staged before advancing') } return decodeLifecycleContinuation({ @@ -123,7 +130,6 @@ export function advanceLifecycleEvaluationPage( counters: { ...continuation.counters, versionsExamined: continuation.counters.versionsExamined + page.rawRowsExamined, - versionsEligible: continuation.counters.versionsEligible + page.candidates.length, }, }) } @@ -132,20 +138,8 @@ export function armLifecycleAttempt( continuation: LifecycleContinuation, attempt: LifecycleInFlightAttempt ): LifecycleContinuation { - if (!continuation.batch || continuation.batch.versions.length === 0) { - throw new Error('Lifecycle continuation has no batch to arm') - } - if (continuation.batch.inFlight) { - if ( - continuation.batch.inFlight.attemptId === attempt.attemptId && - continuation.batch.inFlight.authorizedGeneration === attempt.authorizedGeneration - ) { - return continuation - } - throw new Error('Lifecycle batch already has a different armed attempt') - } - if (attempt.authorizedGeneration !== continuation.generation) { - throw new Error('Lifecycle attempt generation does not match the staged policy') + if (continuation.batch?.inFlight) { + throw new Error('Lifecycle batch already has an armed attempt') } return decodeLifecycleContinuation({ @@ -162,15 +156,6 @@ export function recordLifecycleArtifactOutcomes( throw new Error('Lifecycle batch must be armed before recording artifact outcomes') } - const knownKeys = new Set( - continuation.batch.versions.flatMap((version) => - version.artifacts.map((artifact) => artifact.key) - ) - ) - for (const key of outcomes.keys()) { - if (!knownKeys.has(key)) throw new Error(`Lifecycle outcome references unknown artifact ${key}`) - } - const versions = continuation.batch.versions.map((version) => ({ ...version, artifacts: version.artifacts.map((artifact) => { @@ -181,148 +166,64 @@ export function recordLifecycleArtifactOutcomes( return decodeLifecycleContinuation({ ...continuation, - batch: { versions, inFlight: continuation.batch.inFlight }, - }) -} - -export function filterStagedLifecycleBatch( - continuation: LifecycleContinuation, - retainedVersions: ReadonlyArray> -): LifecycleContinuation { - if (!continuation.batch || continuation.batch.inFlight) { - throw new Error('Lifecycle batch must be staged and unarmed before filtering') - } - - const versions = selectRetainedVersions(continuation.batch.versions, retainedVersions) - if (versions.length === continuation.batch.versions.length) return continuation - if (versions.length === 0) return advanceLifecycleBatch(continuation) - - return decodeLifecycleContinuation({ - ...continuation, - batch: { versions }, - }) -} - -export function refreshStagedLifecycleBatchArtifacts( - continuation: LifecycleContinuation, - candidates: ReadonlyArray -): LifecycleContinuation { - if (!continuation.batch || continuation.batch.inFlight) { - throw new Error('Lifecycle batch must be staged and unarmed before refreshing artifacts') - } - if (candidates.length !== continuation.batch.versions.length) { - throw new Error('Lifecycle artifact refresh must cover the current batch') - } - - const currentVersions = continuation.batch.versions - const versions = candidates.map((candidate, index) => { - const current = currentVersions[index] - if (current.name !== candidate.name || current.version !== candidate.version) { - throw new Error('Lifecycle artifact refresh identities must preserve batch order') - } - return freezeLifecycleBatchVersion(candidate) - }) - - return decodeLifecycleContinuation({ - ...continuation, - batch: { versions }, + batch: { ...continuation.batch, versions }, }) } export function commitLifecycleBatchResults( continuation: LifecycleContinuation, - retainedVersions: ReadonlyArray>, - deleted: LifecycleDeletionCounts + abandonPending = false ): LifecycleContinuation { if (!continuation.batch?.inFlight) { throw new Error('Lifecycle batch must be armed before committing results') } - const versions = selectRetainedVersions(continuation.batch.versions, retainedVersions) - if (versions.length === 0) return completeLifecycleBatch(continuation, deleted) - - const retained = new Set(versions) - if ( - continuation.batch.versions.some( - (version) => !retained.has(version) && lifecycleVersionNeedsCompensation(version) - ) - ) { - throw new Error('Lifecycle batch still requires artifact compensation') - } + const versions = abandonPending + ? [] + : continuation.batch.versions.filter(lifecycleVersionNeedsCompensation) + if (versions.length === 0) return advanceLifecycleBatch(continuation) return decodeLifecycleContinuation({ ...continuation, - counters: addDeletionCounters(continuation.counters, deleted), - batch: { versions, inFlight: continuation.batch.inFlight }, + batch: { ...continuation.batch, versions }, }) } -export function completeLifecycleBatch( - continuation: LifecycleContinuation, - deleted: LifecycleDeletionCounts = { objectVersions: 0, deleteMarkers: 0, bytes: 0 } -): LifecycleContinuation { - if (!continuation.batch || !continuation.pageEnd) { - throw new Error('Lifecycle continuation has no batch to complete') - } - if (!continuation.batch.inFlight) { - throw new Error('Lifecycle batch must be armed before completing') - } - if (continuation.batch.versions.some(lifecycleVersionNeedsCompensation)) { - throw new Error('Lifecycle batch still requires artifact compensation') - } - - return advanceLifecycleBatch(continuation, deleted) +export function skipLifecycleBatch(continuation: LifecycleContinuation): LifecycleContinuation { + const { batch: _batch, pageEnd: _pageEnd, ...rest } = continuation + return decodeLifecycleContinuation({ ...rest, cursor: continuation.pageEnd }) } -function advanceLifecycleBatch( - continuation: LifecycleContinuation, - deleted: LifecycleDeletionCounts = { objectVersions: 0, deleteMarkers: 0, bytes: 0 } -): LifecycleContinuation { - if (!continuation.batch || !continuation.pageEnd) { - throw new Error('Lifecycle continuation has no batch to advance') - } - - const { batch: _batch, pageEnd: _pageEnd, ...rest } = continuation +function advanceLifecycleBatch(continuation: LifecycleContinuation): LifecycleContinuation { + const skipped = skipLifecycleBatch(continuation) return decodeLifecycleContinuation({ - ...rest, - cursor: continuation.pageEnd, + ...skipped, counters: { - ...addDeletionCounters(continuation.counters, deleted), - batchesCompleted: continuation.counters.batchesCompleted + 1, + ...skipped.counters, + batchesCompleted: skipped.counters.batchesCompleted + 1, }, }) } -export function lifecycleVersionNeedsCompensation(version: LifecycleBatchVersion): boolean { - const outcomes = new Set(version.artifacts.map((artifact) => artifact.outcome)) - return ( - outcomes.has('UNRESOLVED') || - (outcomes.has('FAILED') && (outcomes.has('DELETED') || outcomes.has('ABSENT'))) +export function pendingArtifacts( + continuation: LifecycleContinuation +): Array> { + return continuation.batch!.versions.flatMap(({ name, version, artifacts }) => + artifacts + .filter((artifact) => artifact.outcome === 'UNRESOLVED' || artifact.outcome === 'FAILED') + .map((artifact) => ({ ...artifact, name, version })) ) } -function selectRetainedVersions( - currentVersions: LifecycleBatchVersion[], - retainedVersions: ReadonlyArray> -): LifecycleBatchVersion[] { - const identities = new Set( - retainedVersions.map((version) => lifecycleVersionIdentity(version.name, version.version)) - ) - if (identities.size !== retainedVersions.length) { - throw new Error('Lifecycle retained version identities must be unique') - } - - const versions = currentVersions.filter((version) => - identities.has(lifecycleVersionIdentity(version.name, version.version)) - ) - if (versions.length !== retainedVersions.length) { - throw new Error('Lifecycle retained versions must belong to the current batch') - } - return versions +function lifecycleVersionNeedsCompensation(version: LifecycleBatchVersion): boolean { + return version.artifacts.some(({ outcome }) => outcome === 'UNRESOLVED' || outcome === 'FAILED') } -export function lifecycleVersionIdentity(name: string, version: string | null): string { - return JSON.stringify([name, version]) +export function recordLifecycleMetadataDeletions( + continuation: LifecycleContinuation, + deleted: LifecycleDeletionCounts +): LifecycleContinuation { + return { ...continuation, counters: addDeletionCounters(continuation.counters, deleted) } } function addDeletionCounters( @@ -333,7 +234,7 @@ function addDeletionCounters( ...counters, objectVersionsDeleted: counters.objectVersionsDeleted + deleted.objectVersions, deleteMarkersDeleted: counters.deleteMarkersDeleted + deleted.deleteMarkers, - bytesDeleted: counters.bytesDeleted + deleted.bytes, + bytesDeleted: Math.min(counters.bytesDeleted + deleted.bytes, Number.MAX_SAFE_INTEGER), } } @@ -347,15 +248,8 @@ export function decodeLifecycleContinuation(value: unknown): LifecycleContinuati const decoded: LifecycleContinuation = { continuationVersion: LIFECYCLE_CONTINUATION_VERSION, runId: uuid(continuation.runId, 'runId'), - trigger: oneOf( - continuation.trigger, - ['scheduled', 'configuration_change', 'manual', 'recovery'] as const, - 'trigger' - ), generation: uuid(continuation.generation, 'generation'), snapshotAt: timestamp(continuation.snapshotAt, 'snapshotAt'), - mode: oneOf(continuation.mode, ['EVALUATE', 'DELETE'] as const, 'mode'), - topology: decodeTopology(continuation.topology), counters: decodeCounters(continuation.counters), } @@ -373,20 +267,7 @@ export function decodeLifecycleContinuation(value: unknown): LifecycleContinuati return decoded } -function decodeTopology(value: unknown): LifecycleShardTopology { - const topology = record(value, 'topology') - const shardCount = safeInteger(topology.shardCount, 'topology.shardCount', 1, 1024) - const shardId = safeInteger(topology.shardId, 'topology.shardId', 0, shardCount - 1) - - return { - scanKind: oneOf(topology.scanKind, ['NONCURRENT', 'CURRENT'] as const, 'topology.scanKind'), - epoch: positiveBigint(topology.epoch, 'topology.epoch'), - shardId, - shardCount, - } -} - -function decodeCursor(value: unknown, label: string): NoncurrentLifecycleShardCursor { +export function decodeCursor(value: unknown, label: string): NoncurrentLifecycleCursor { const cursor = record(value, label) return { archivedAt: timestamp(cursor.archivedAt, `${label}.archivedAt`), @@ -414,39 +295,53 @@ function decodeCounters(value: unknown): LifecycleRunCounters { function decodeBatch(value: unknown): LifecycleContinuationBatch { const batch = record(value, 'batch') + const keySeparator = oneOf( + batch.keySeparator, + [PATH_SEPARATOR, FILE_VERSION_SEPARATOR] as const, + 'batch.keySeparator' + ) if (!Array.isArray(batch.versions) || batch.versions.length > LIFECYCLE_MAX_PAGE_SIZE) { throw decodeError(`batch.versions must contain at most ${LIFECYCLE_MAX_PAGE_SIZE} entries`) } const versions = batch.versions.map((entry, index): LifecycleBatchVersion => { const version = record(entry, `batch.versions[${index}]`) + const artifactCount = version.version ? 2 : 1 if ( !Array.isArray(version.artifacts) || - (version.artifacts.length !== 0 && version.artifacts.length !== 2) + (version.artifacts.length !== 0 && version.artifacts.length !== artifactCount) ) { - throw decodeError(`batch.versions[${index}].artifacts must contain 0 or 2 entries`) + throw decodeError( + `batch.versions[${index}].artifacts must contain 0 or ${artifactCount} entries` + ) } + const name = string(version.name, `batch.versions[${index}].name`) + const versionId = + version.version === null ? null : string(version.version, `batch.versions[${index}].version`) + const keys = lifecycleArtifactKeys(name, versionId, keySeparator) return { - name: string(version.name, `batch.versions[${index}].name`), - version: - version.version === null - ? null - : string(version.version, `batch.versions[${index}].version`), + id: uuid(version.id, `batch.versions[${index}].id`), + name, + version: versionId, artifacts: version.artifacts.map((artifact, artifactIndex) => - decodeArtifact(artifact, `batch.versions[${index}].artifacts[${artifactIndex}]`) + decodeArtifact( + artifact, + keys[artifactIndex], + `batch.versions[${index}].artifacts[${artifactIndex}]` + ) ), } }) - const decoded: LifecycleContinuationBatch = { versions } + const decoded: LifecycleContinuationBatch = { keySeparator, versions } if (batch.inFlight !== undefined) decoded.inFlight = decodeAttempt(batch.inFlight) return decoded } -function decodeArtifact(value: unknown, label: string): LifecycleArtifact { +function decodeArtifact(value: unknown, key: string, label: string): LifecycleArtifact { const artifact = record(value, label) const decoded: LifecycleArtifact = { - key: string(artifact.key, `${label}.key`), + key, outcome: oneOf( artifact.outcome, ['UNRESOLVED', 'DELETED', 'ABSENT', 'FAILED'] as const, @@ -461,7 +356,6 @@ function decodeAttempt(value: unknown): LifecycleInFlightAttempt { const attempt = record(value, 'batch.inFlight') return { attemptId: uuid(attempt.attemptId, 'batch.inFlight.attemptId'), - authorizedGeneration: uuid(attempt.authorizedGeneration, 'batch.inFlight.authorizedGeneration'), startedAt: timestamp(attempt.startedAt, 'batch.inFlight.startedAt'), } } @@ -491,29 +385,9 @@ function timestamp(value: unknown, label: string): string { return parsed } -function signedBigint(value: unknown, label: string): string { - const parsed = string(value, label) - if (!BIGINT.test(parsed)) throw decodeError(`${label} must be a bigint decimal string`) - const bigint = BigInt(parsed) - if (bigint < MIN_SIGNED_BIGINT || bigint > MAX_SIGNED_BIGINT) { - throw decodeError(`${label} is outside the signed bigint range`) - } - return parsed -} - -function positiveBigint(value: unknown, label: string): string { - const parsed = signedBigint(value, label) - if (BigInt(parsed) < 1n) throw decodeError(`${label} must be positive`) - return parsed -} - function nonnegativeInteger(value: unknown, label: string): number { - return safeInteger(value, label, 0, Number.MAX_SAFE_INTEGER) -} - -function safeInteger(value: unknown, label: string, minimum: number, maximum: number): number { - if (!Number.isSafeInteger(value) || (value as number) < minimum || (value as number) > maximum) { - throw decodeError(`${label} must be an integer between ${minimum} and ${maximum}`) + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw decodeError(`${label} must be an integer between 0 and ${Number.MAX_SAFE_INTEGER}`) } return value as number } diff --git a/src/storage/lifecycle/counters.test.ts b/src/storage/lifecycle/counters.test.ts new file mode 100644 index 000000000..79baf4db2 --- /dev/null +++ b/src/storage/lifecycle/counters.test.ts @@ -0,0 +1,81 @@ +import { logSchema } from '@internal/monitoring' +import { describe, expect, it, vi } from 'vitest' +import { lifecycleDeletionCounters } from './counters' + +const context = { tenantId: 'tenant', bucketId: 'bucket' } + +describe('lifecycle deletion counters', () => { + it.each([ + null, + {}, + { size: null }, + { size: -1 }, + { size: 1.5 }, + { size: 'invalid' }, + { size: '' }, + { size: ' ' }, + { size: true }, + { size: Number.MAX_SAFE_INTEGER + 1 }, + { size: Number.NaN }, + { size: Number.POSITIVE_INFINITY }, + ])('counts invalid metadata %j as zero bytes without losing the object count', (metadata) => { + const warning = vi.spyOn(logSchema, 'warning').mockImplementation(() => {}) + expect( + lifecycleDeletionCounters( + [ + { is_delete_marker: false, metadata }, + { is_delete_marker: false, metadata: { size: 10 } }, + ], + context + ) + ).toEqual({ objectVersions: 2, deleteMarkers: 0, bytes: 10 }) + expect(warning).toHaveBeenCalledExactlyOnceWith( + expect.anything(), + expect.stringContaining('invalid object sizes'), + expect.objectContaining({ + tenantId: context.tenantId, + metadata: JSON.stringify({ bucketId: context.bucketId, invalidSizeCount: 1 }), + }) + ) + }) + + it.each([0, 1, '1', Number.MAX_SAFE_INTEGER])('accepts size %j', (size) => { + const warning = vi.spyOn(logSchema, 'warning') + expect( + lifecycleDeletionCounters([{ is_delete_marker: false, metadata: { size } }], context) + ).toEqual({ objectVersions: 1, deleteMarkers: 0, bytes: Number(size) }) + expect(warning).not.toHaveBeenCalled() + }) + + it('counts delete markers separately without inspecting their size', () => { + const warning = vi.spyOn(logSchema, 'warning') + expect( + lifecycleDeletionCounters( + [ + { is_delete_marker: true, metadata: null }, + { is_delete_marker: true, metadata: { size: 100 } }, + ], + context + ) + ).toEqual({ objectVersions: 0, deleteMarkers: 2, bytes: 0 }) + expect(warning).not.toHaveBeenCalled() + }) + + it('counts a row that would overflow the safe integer range as invalid', () => { + const warning = vi.spyOn(logSchema, 'warning').mockImplementation(() => {}) + expect( + lifecycleDeletionCounters( + [ + { is_delete_marker: false, metadata: { size: Number.MAX_SAFE_INTEGER } }, + { is_delete_marker: false, metadata: { size: 1 } }, + ], + context + ) + ).toEqual({ objectVersions: 2, deleteMarkers: 0, bytes: Number.MAX_SAFE_INTEGER }) + expect(warning).toHaveBeenCalledWith( + expect.anything(), + expect.any(String), + expect.objectContaining({ metadata: expect.stringContaining('"invalidSizeCount":1') }) + ) + }) +}) diff --git a/src/storage/lifecycle/counters.ts b/src/storage/lifecycle/counters.ts new file mode 100644 index 000000000..76431e7a5 --- /dev/null +++ b/src/storage/lifecycle/counters.ts @@ -0,0 +1,47 @@ +import { logger, logSchema } from '@internal/monitoring' + +export interface LifecycleDeletedObjectRow { + is_delete_marker: boolean + metadata: Record | null +} + +export function lifecycleDeletionCounters( + rows: LifecycleDeletedObjectRow[], + context: { tenantId: string; bucketId: string } +): { + objectVersions: number + deleteMarkers: number + bytes: number +} { + let objectVersions = 0 + let deleteMarkers = 0 + let bytes = 0 + let invalidSizeCount = 0 + for (const row of rows) { + if (row.is_delete_marker) { + deleteMarkers++ + continue + } + objectVersions++ + const metadata = row.metadata + const rawSize = metadata && typeof metadata === 'object' ? metadata.size : undefined + const size = + typeof rawSize === 'number' || (typeof rawSize === 'string' && rawSize.trim() !== '') + ? Number(rawSize) + : NaN + if (!Number.isSafeInteger(size) || size < 0 || !Number.isSafeInteger(bytes + size)) { + invalidSizeCount++ + continue + } + bytes += size + } + if (invalidSizeCount > 0) { + logSchema.warning(logger, '[Lifecycle] Missing or invalid object sizes counted as zero bytes', { + type: 'event', + tenantId: context.tenantId, + project: context.tenantId, + metadata: JSON.stringify({ bucketId: context.bucketId, invalidSizeCount }), + }) + } + return { objectVersions, deleteMarkers, bytes } +} diff --git a/src/storage/lifecycle/evaluation.test.ts b/src/storage/lifecycle/evaluation.test.ts new file mode 100644 index 000000000..ccf5ad240 --- /dev/null +++ b/src/storage/lifecycle/evaluation.test.ts @@ -0,0 +1,71 @@ +import type { LifecycleEvaluationPage } from '../schemas/lifecycle' +import { lifecycleEvaluationBatches, validateLifecycleDeleteBatchSize } from './evaluation' + +test.each([0, -1, 1.5, 49, Number.NaN, Number.POSITIVE_INFINITY])( + 'rejects delete batch size %s', + (size) => { + expect(() => validateLifecycleDeleteBatchSize(size)).toThrow('between 1 and 48') + } +) + +const cases = [ + { length: 0, size: 48, selectivity: 'none' }, + { length: 1, size: 1, selectivity: 'all' }, + { length: 48, size: 48, selectivity: 'all' }, + { length: 49, size: 48, selectivity: 'all' }, + { length: 96, size: 48, selectivity: 'all' }, + { length: 100, size: 48, selectivity: 'all' }, + { length: 480, size: 48, selectivity: 'all' }, + { length: 100, size: 1, selectivity: 'sparse' }, + { length: 100, size: 2, selectivity: 'sparse' }, + { length: 100, size: 17, selectivity: 'sparse' }, + { length: 480, size: 48, selectivity: 'sparse' }, + { length: 480, size: 48, selectivity: 'none' }, +] + +test.each(cases)( + 'partitions $length raw rows, $selectivity eligibility, limit $size without gaps', + ({ length, size, selectivity }) => { + const rows = Array.from({ length }, (_, index) => ({ + id: String(index), + name: `name-${index}`, + version: String(index), + isDeleteMarker: index % 2 === 0, + archivedAt: `2026-08-01T00:00:00.${String(index).padStart(6, '0')}Z`, + rawRowNumber: index + 1, + })) + const candidates = rows.filter( + (_, index) => selectivity === 'all' || (selectivity === 'sparse' && index % 3 === 1) + ) + const last = rows.at(-1) + const page: LifecycleEvaluationPage = { + rawRowsExamined: length, + candidates, + ...(last ? { pageEnd: { name: last.name, archivedAt: last.archivedAt } } : {}), + exhausted: length < 480, + } + const batches = [...lifecycleEvaluationBatches(page, size)] + expect(batches.flatMap((batch) => batch.candidates)).toEqual(candidates) + if (!candidates.length) { + expect(batches).toEqual([]) + return + } + expect(batches.reduce((sum, batch) => sum + batch.rawRowsExamined, 0)).toBe(length) + expect(batches.at(-1)?.pageEnd).toEqual(page.pageEnd) + let consumed = 0 + for (const batch of batches) { + expect(batch.candidates.length).toBeGreaterThan(0) + expect(batch.candidates.length).toBeLessThanOrEqual(size) + expect(batch.rawRowsExamined).toBeGreaterThanOrEqual(batch.candidates.length) + consumed += batch.rawRowsExamined + const boundary = rows[consumed - 1] + expect(batch.pageEnd).toEqual({ name: boundary.name, archivedAt: boundary.archivedAt }) + // A restart at every boundary must see every remaining candidate exactly once. + const remainder = candidates.filter((candidate) => candidate.rawRowNumber > consumed) + const processed = batches + .slice(0, batches.indexOf(batch) + 1) + .flatMap((part) => part.candidates) + expect([...processed, ...remainder]).toEqual(candidates) + } + } +) diff --git a/src/storage/lifecycle/evaluation.ts b/src/storage/lifecycle/evaluation.ts new file mode 100644 index 000000000..4c8d3df83 --- /dev/null +++ b/src/storage/lifecycle/evaluation.ts @@ -0,0 +1,29 @@ +import { LIFECYCLE_MAX_DELETE_BATCH_SIZE, type LifecycleEvaluationPage } from '../schemas/lifecycle' + +export function validateLifecycleDeleteBatchSize(size: number): void { + if (!Number.isSafeInteger(size) || size < 1 || size > LIFECYCLE_MAX_DELETE_BATCH_SIZE) { + throw new Error( + `Lifecycle delete batch size must be between 1 and ${LIFECYCLE_MAX_DELETE_BATCH_SIZE}` + ) + } +} + +export function* lifecycleEvaluationBatches( + page: LifecycleEvaluationPage, + size: number +): Generator> { + validateLifecycleDeleteBatchSize(size) + let consumed = 0 + for (let offset = 0; offset < page.candidates.length; offset += size) { + const candidates = page.candidates.slice(offset, offset + size) + const last = candidates[candidates.length - 1] + const final = offset + size >= page.candidates.length + const end = final ? page.rawRowsExamined : last.rawRowNumber + yield { + candidates, + rawRowsExamined: end - consumed, + pageEnd: final ? page.pageEnd : { name: last.name, archivedAt: last.archivedAt }, + } + consumed = end + } +} diff --git a/src/storage/schemas/lifecycle.ts b/src/storage/schemas/lifecycle.ts index 91c7e5cc6..b7431f463 100644 --- a/src/storage/schemas/lifecycle.ts +++ b/src/storage/schemas/lifecycle.ts @@ -1,4 +1,4 @@ -import type { Bucket } from './bucket' +import type { Bucket, BucketVersioningStatus } from './bucket' export const LIFECYCLE_MAX_RULES = 1000 export const LIFECYCLE_MAX_NONCURRENT_DAYS = 2147483647 @@ -70,61 +70,45 @@ export interface LifecycleEvaluationRule { newerNoncurrentVersions?: number } -export const LIFECYCLE_MAX_PAGE_SIZE = 500 +export const LIFECYCLE_MAX_PAGE_SIZE = 480 +// Stay below PostgreSQL 64 cached subxids +export const LIFECYCLE_MAX_DELETE_BATCH_SIZE = 48 -export interface NoncurrentLifecycleShardCursor { +export interface NoncurrentLifecycleCursor { archivedAt: string name: string } -export interface LifecycleCandidate { +export interface LifecycleCandidateIdentity { id: string - bucketId: string name: string version: string | null - isVersioned: boolean isDeleteMarker: boolean - metadata: Record | null - createdAt: string - archivedAt: string } -export interface LifecycleCandidateIdentity { - name: string - version: string | null - isDeleteMarker: boolean +export interface LifecycleEvaluatedCandidate extends LifecycleCandidateIdentity { + archivedAt: string + // One-based position in the raw page including ineligible rows. + rawRowNumber: number } export interface LifecycleEvaluationPage { rawRowsExamined: number - candidates: LifecycleCandidateIdentity[] - pageEnd?: NoncurrentLifecycleShardCursor + candidates: LifecycleEvaluatedCandidate[] + pageEnd?: NoncurrentLifecycleCursor exhausted: boolean } export interface EvaluateNoncurrentLifecyclePageInput { bucketId: string snapshotAt: string - cursor?: NoncurrentLifecycleShardCursor + cursor?: NoncurrentLifecycleCursor rules: LifecycleEvaluationRule[] pageSize: number } export const LIFECYCLE_CONTINUATION_VERSION = 1 as const -export type LifecycleExecutionMode = 'EVALUATE' | 'DELETE' - -export type LifecycleScanKind = 'NONCURRENT' | 'CURRENT' - -export type LifecycleTrigger = 'scheduled' | 'configuration_change' | 'manual' | 'recovery' - -export interface LifecycleShardTopology { - scanKind: LifecycleScanKind - epoch: string - shardId: number - shardCount: number -} - export interface LifecycleRunCounters { versionsExamined: number versionsEligible: number @@ -143,6 +127,7 @@ export interface LifecycleArtifact { } export interface LifecycleBatchVersion { + id: string name: string version: string | null artifacts: LifecycleArtifact[] @@ -150,11 +135,11 @@ export interface LifecycleBatchVersion { export interface LifecycleInFlightAttempt { attemptId: string - authorizedGeneration: string startedAt: string } export interface LifecycleContinuationBatch { + keySeparator: '/' | '-$v-' versions: LifecycleBatchVersion[] inFlight?: LifecycleInFlightAttempt } @@ -162,18 +147,67 @@ export interface LifecycleContinuationBatch { export interface LifecycleContinuation { continuationVersion: typeof LIFECYCLE_CONTINUATION_VERSION runId: string - trigger: LifecycleTrigger generation: string snapshotAt: string - mode: LifecycleExecutionMode - topology: LifecycleShardTopology - cursor?: NoncurrentLifecycleShardCursor - pageEnd?: NoncurrentLifecycleShardCursor + cursor?: NoncurrentLifecycleCursor + pageEnd?: NoncurrentLifecycleCursor counters: LifecycleRunCounters batch?: LifecycleContinuationBatch } +export interface LifecycleClaimIdentity { + bucketId: string + claimId: string +} + +export interface LifecycleClaimInput extends LifecycleClaimIdentity { + leaseMs: number +} + +export interface LifecycleArmAttemptInput extends LifecycleClaimIdentity { + leaseMs: number + continuation: LifecycleContinuation +} + +export interface LifecycleCommitAttemptInput extends LifecycleClaimIdentity { + attemptId: string + continuation: LifecycleContinuation + abandonPending?: boolean +} + +export interface LifecycleReleaseClaimInput extends LifecycleClaimIdentity { + // Omit after an ambiguous write to preserve the durable journal. + continuation?: LifecycleContinuation | null + nextRunAt: string | null + // Cap a retry only when preserving a persisted journal in flight. + maxInFlightRetryAt?: string + error?: Record +} + +export interface LifecycleDispatchState { + nextDispatchAt: Date | null + maxFailureCount: number + oldestInFlightStartedAt: string | null + hasAbandonedArtifacts: boolean +} + +export interface LifecycleState { + bucketId: string + claimId: string | null + continuation: LifecycleContinuation | null + failureCount: number +} + +export type LifecycleClaimedState = LifecycleState & { + bucket: Pick< + LifecycleBucket, + 'versioning_status' | 'lifecycle_configuration' | 'lifecycle_configuration_generation' + > +} + export type LifecycleBucket = Pick & { + type: 'STANDARD' | 'ANALYTICS' + versioning_status: BucketVersioningStatus lifecycle_configuration: BucketLifecycleConfiguration | null lifecycle_configuration_generation: string | null } diff --git a/src/test/bucket-lifecycle-configuration.test.ts b/src/test/bucket-lifecycle-configuration.test.ts index 4e757a4e6..85dfb277d 100644 --- a/src/test/bucket-lifecycle-configuration.test.ts +++ b/src/test/bucket-lifecycle-configuration.test.ts @@ -1,11 +1,15 @@ import { randomUUID } from 'node:crypto' import { ROUTE_OPERATIONS } from '../http/routes/operations' import { StoragePgDB } from '../storage/database' -import { useStorage, withDeleteEnabled } from './utils/storage' +import { normalizeLifecycleConfiguration } from '../storage/lifecycle' +import { useLifecycleStorage } from './utils/lifecycle-versioning' describe('bucket lifecycle configuration persistence', () => { let currentOperation: string | undefined - const helper = useStorage({ operation: () => currentOperation }) + const fixtures = useLifecycleStorage('bucket-lifecycle-configuration', { + operation: () => currentOperation, + }) + const helper = fixtures.helper let bucketId: string async function withOperation(operation: string, fn: () => Promise): Promise { @@ -28,16 +32,8 @@ describe('bucket lifecycle configuration persistence', () => { return helper.database.deleteLifecycleConfiguration(bucketId) } - beforeEach(async () => { - bucketId = `bucket-lifecycle-configuration-${randomUUID()}` - await helper.database.createBucket({ id: bucketId, name: bucketId }) - }) - - afterEach(async () => { - await withDeleteEnabled(helper.database.connection, async (transaction) => { - await transaction.query('DELETE FROM storage.objects WHERE bucket_id = $1', [bucketId]) - await transaction.query('DELETE FROM storage.buckets WHERE id = $1', [bucketId]) - }) + beforeEach(() => { + bucketId = fixtures.bucketId }) const rules = [ @@ -58,35 +54,65 @@ describe('bucket lifecycle configuration persistence', () => { }, ] - it('stores canonical policy state and preserves generation for an equivalent PUT', async () => { - const first = await putLifecycleConfiguration({ rules }) - - expect(first.lifecycle_configuration).toEqual({ rules }) - expect(first.lifecycle_configuration_generation).toMatch(/^[0-9a-f]{8}-[0-9a-f-]{27}$/i) - const beforeRetry = await helper.database.findBucketById(bucketId, 'updated_at') - - const retry = await putLifecycleConfiguration({ rules }) - expect(retry.lifecycle_configuration_generation).toBe(first.lifecycle_configuration_generation) - expect(retry.lifecycle_configuration?.rules).toEqual(rules) - await expect(helper.database.findBucketById(bucketId, 'updated_at')).resolves.toEqual( - beforeRetry - ) - - await expect(helper.database.findLifecycleBucket(bucketId)).resolves.toEqual(retry) + it('schedules a policy written by a writer that predates the control plane', async () => { + await fixtures.setStatus(bucketId, 'ENABLED') + const hasMigration = vi + .spyOn(StoragePgDB.prototype, 'hasMigration') + .mockImplementation(async (name) => name !== 'lifecycle-writer-coordination') + try { + await putLifecycleConfiguration({ rules }) + } finally { + hasMigration.mockRestore() + } + await expect( + fixtures.readState(bucketId, 'next_run_at <= clock_timestamp() AS due, claim_id') + ).resolves.toEqual([{ due: true, claim_id: null }]) }) - it('keeps stored order, generation and updated_at when rules are reordered', async () => { - const first = await putLifecycleConfiguration({ rules }) - const beforeRetry = await helper.database.findBucketById(bucketId, 'updated_at') - const reordered = { rules: [...rules].reverse() } - - await expect(putLifecycleConfiguration(reordered)).resolves.toEqual(first) - await expect(helper.database.findLifecycleBucket(bucketId)).resolves.toEqual(first) - await expect(putLifecycleConfiguration(reordered)).resolves.toEqual(first) - await expect(helper.database.findBucketById(bucketId, 'updated_at')).resolves.toEqual( - beforeRetry - ) - }) + it.each(['equivalent', 'reordered', 'ID-less'])( + 'preserves stored policy, generation, timestamp and execution state for an %s PUT', + async (kind) => { + const configuration = + kind === 'ID-less' + ? normalizeLifecycleConfiguration({ rules: rules.map(({ id: _id, ...rule }) => rule) }) + : { rules } + const first = await putLifecycleConfiguration(configuration) + expect(first.lifecycle_configuration).toEqual(configuration) + expect(first.lifecycle_configuration_generation).toMatch(/^[0-9a-f]{8}-[0-9a-f-]{27}$/i) + if (kind === 'ID-less') { + expect(first.lifecycle_configuration?.rules[0].id).toMatch(/^rule-[0-9a-f]{64}$/) + } + const beforeRetry = await helper.database.findBucketById(bucketId, 'updated_at') + const continuation = { preserved: true } + await helper.database.connection.query( + `UPDATE storage.bucket_lifecycle_states + SET continuation = $2::jsonb, next_run_at = '2030-01-01', failure_count = 3 + WHERE bucket_id = $1`, + [bucketId, JSON.stringify(continuation)] + ) + const retry = { rules: configuration.rules.map((rule) => ({ ...rule, filter: {} })) } + if (kind === 'reordered') retry.rules.reverse() + + for (let attempt = 0; attempt < 2; attempt++) { + await expect(putLifecycleConfiguration(retry)).resolves.toEqual(first) + await expect(helper.database.findLifecycleBucket(bucketId)).resolves.toEqual(first) + } + await expect(helper.database.findBucketById(bucketId, 'updated_at')).resolves.toEqual( + beforeRetry + ) + const state = await helper.database.connection.query( + 'SELECT continuation, next_run_at, failure_count FROM storage.bucket_lifecycle_states WHERE bucket_id = $1', + [bucketId] + ) + expect(state.rows).toEqual([ + { + continuation, + next_run_at: new Date('2030-01-01T00:00:00Z'), + failure_count: 3, + }, + ]) + } + ) it('rotates generation only when canonical policy content changes', async () => { const first = await putLifecycleConfiguration({ rules }) @@ -146,6 +172,12 @@ describe('bucket lifecycle configuration persistence', () => { lifecycle_configuration_generation: null, }) + const state = await helper.database.connection.query( + 'SELECT 1 FROM storage.bucket_lifecycle_states WHERE bucket_id = $1', + [bucketId] + ) + expect(state.rowCount).toBe(0) + const beforeRetry = await helper.database.findBucketById(bucketId, 'updated_at') await expect(deleteLifecycleConfiguration()).resolves.toMatchObject({ lifecycle_configuration: null, diff --git a/src/test/bucket-lifecycle.test.ts b/src/test/bucket-lifecycle.test.ts index 48885f399..235e45958 100644 --- a/src/test/bucket-lifecycle.test.ts +++ b/src/test/bucket-lifecycle.test.ts @@ -90,6 +90,11 @@ describe('bucket lifecycle controls', () => { await expect(fixtures.readState(bucketId, 'next_run_at')).resolves.toEqual([ { next_run_at: null }, ]) + await expect( + helper.database.findLifecycleDispatchState({ runBudgetMs: 20_000 }) + ).resolves.toMatchObject({ + nextDispatchAt: null, + }) } ) @@ -100,6 +105,7 @@ describe('bucket lifecycle controls', () => { await expect( fixtures.readState(bucketId, 'next_run_at <= clock_timestamp() AS due') ).resolves.toEqual([{ due: true }]) + await expect(helper.database.listDueLifecycleBuckets(100)).resolves.toContainEqual(bucketId) const retried = await helper.database.putLifecycleConfiguration(bucketId, configuration) expect(retried.lifecycle_configuration_generation).toBe( configured.lifecycle_configuration_generation @@ -127,7 +133,7 @@ describe('bucket lifecycle controls', () => { const now = new Date().toISOString() return armedContinuation( generation, - [{ name: 'object', version: randomUUID(), isDeleteMarker: false }], + [{ id: randomUUID(), name: 'object', version: randomUUID(), isDeleteMarker: false }], { name: 'object', archivedAt: now }, { snapshotAt: now, startedAt: now } ) @@ -312,6 +318,21 @@ describe('bucket lifecycle controls', () => { } ) + it('repairs missing state on an unchanged API PUT without rotating the generation', async () => { + const bucket = await helper.database.putLifecycleConfiguration(bucketId, configuration) + await helper.database.connection.query( + 'DELETE FROM storage.bucket_lifecycle_states WHERE bucket_id = $1', + [bucketId] + ) + const retried = await helper.database.putLifecycleConfiguration(bucketId, configuration) + expect(retried.lifecycle_configuration_generation).toBe( + bucket.lifecycle_configuration_generation + ) + await expect(fixtures.readState(bucketId, 'next_run_at')).resolves.toEqual([ + { next_run_at: null }, + ]) + }) + it('preserves healthy state on a SQL update that does not change the generation', async () => { await fixtures.setStatus(bucketId, 'ENABLED') await helper.database.putLifecycleConfiguration(bucketId, configuration) diff --git a/src/test/noncurrent-lifecycle-database.test.ts b/src/test/noncurrent-lifecycle-database.test.ts new file mode 100644 index 000000000..f67eb3e68 --- /dev/null +++ b/src/test/noncurrent-lifecycle-database.test.ts @@ -0,0 +1,1902 @@ +import { randomUUID } from 'node:crypto' +import { + buildEvaluateNoncurrentLifecyclePageStatement, + mapLifecycleEvaluationPage, +} from '@storage/database/lifecycle' +import { compileLifecycleEvaluationRules } from '@storage/lifecycle/configuration' +import { + armLifecycleAttempt, + encodeLifecycleContinuation, + recordLifecycleArtifactOutcomes, +} from '@storage/lifecycle/continuation' +import { lifecycleEvaluationBatches } from '@storage/lifecycle/evaluation' +import type { NoncurrentLifecycleCursor } from '@storage/schemas' +import { Client } from 'pg' +import { getConfig } from '../config' +import { armedContinuation, stagedContinuation } from './utils/lifecycle-continuation' +import { useLifecycleStorage } from './utils/lifecycle-versioning' +import { withDeleteEnabled } from './utils/storage' + +describe('noncurrent lifecycle database API', () => { + const fixtures = useLifecycleStorage('lifecycle-db') + const tHelper = fixtures.helper + const { withServiceOperation } = fixtures + let bucketId: string + beforeEach(() => { + bucketId = fixtures.bucketId + }) + + async function insertArchivedObjects( + rows: Array<{ id?: string; name: string; archivedAt: string; version?: string }> + ) { + await tHelper.database.connection.query( + `INSERT INTO storage.objects (id, bucket_id, name, version, archived_at, is_versioned, metadata) + SELECT COALESCE(r.id::uuid, gen_random_uuid()), $1, r.name, COALESCE(r.version, gen_random_uuid()::text), + r.archived_at::timestamptz, true, '{"size": 10}'::jsonb + FROM unnest($2::text[], $3::text[], $4::text[], $5::text[]) AS r(name, archived_at, version, id)`, + [ + bucketId, + rows.map((row) => row.name), + rows.map((row) => row.archivedAt), + rows.map((row) => row.version ?? null), + rows.map((row) => row.id ?? null), + ] + ) + } + + async function insertOtherBucketSibling(name: string, archivedAt: string) { + const otherBucketId = await fixtures.createBucket() + await tHelper.database.connection.query( + `INSERT INTO storage.objects (bucket_id, name, version, archived_at, is_versioned, metadata) + VALUES ($1, $2, gen_random_uuid()::text, $3::timestamptz, true, '{"size": 10}'::jsonb)`, + [otherBucketId, name, archivedAt] + ) + } + + async function putConfiguration( + newerNoncurrentVersions?: number, + beforeWrite?: () => Promise + ) { + await fixtures.setStatus(bucketId, 'ENABLED') + await beforeWrite?.() + const bucket = await tHelper.database.putLifecycleConfiguration(bucketId, { + rules: [ + { + status: 'Enabled', + filter: {}, + noncurrentVersionExpiration: { + noncurrentDays: 30, + ...(newerNoncurrentVersions === undefined ? {} : { newerNoncurrentVersions }), + }, + }, + ], + }) + return bucket.lifecycle_configuration_generation! + } + + const SNAPSHOT_AT = '2026-08-15T12:00:00.000Z' + + const evaluatePage = ( + rules: Array<{ cutoffAt: string; newerNoncurrentVersions?: number }>, + extra: { pageSize?: number; cursor?: NoncurrentLifecycleCursor } = {} + ) => + tHelper.database.evaluateNoncurrentLifecyclePage({ + bucketId, + snapshotAt: SNAPSHOT_AT, + pageSize: 480, + rules, + ...extra, + }) + + const configuration = (noncurrentDays: number) => ({ + rules: [ + { + status: 'Enabled' as const, + filter: {}, + noncurrentVersionExpiration: { noncurrentDays }, + }, + ], + }) + const dueNow = async () => + (await fixtures.readState(bucketId, 'next_run_at <= clock_timestamp() AS due'))[0].due + + test('evaluates a sparse page once and preserves each chunk boundary and the ineligible tail', async () => { + const archivedAt = '2026-07-01T00:00:00.000123Z' + await insertArchivedObjects([ + ...['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'].map((name) => ({ name, archivedAt })), + ...['b', 'd', 'f'].map((name) => ({ name, archivedAt: '2026-08-10T00:00:00Z' })), + ]) + const rules = [{ cutoffAt: '2026-08-01T00:00:00Z', newerNoncurrentVersions: 1 }] + const page = await evaluatePage(rules, { pageSize: 100 }) + expect(page.candidates.map(({ name, rawRowNumber }) => ({ name, rawRowNumber }))).toEqual([ + { name: 'b', rawRowNumber: 2 }, + { name: 'd', rawRowNumber: 4 }, + { name: 'f', rawRowNumber: 6 }, + ]) + expect(page).toMatchObject({ + rawRowsExamined: 8, + pageEnd: { name: 'h', archivedAt }, + exhausted: true, + }) + expect([...lifecycleEvaluationBatches(page, 2)]).toMatchObject([ + { + rawRowsExamined: 4, + pageEnd: { name: 'd', archivedAt }, + candidates: [{ name: 'b' }, { name: 'd' }], + }, + { rawRowsExamined: 4, pageEnd: { name: 'h', archivedAt }, candidates: [{ name: 'f' }] }, + ]) + const empty = await evaluatePage(rules, { + pageSize: 100, + cursor: page.pageEnd, + }) + expect(empty).toEqual({ rawRowsExamined: 0, candidates: [], exhausted: true }) + }) + + test('loads typed Standard-bucket authority', async () => { + const analyticsBucketId = await fixtures.createBucket('ANALYTICS') + const generation = await putConfiguration() + const bucket = await tHelper.database.findLifecycleBucket(bucketId) + + expect(bucket).toMatchObject({ + id: bucketId, + type: 'STANDARD', + lifecycle_configuration_generation: generation, + lifecycle_configuration: { + rules: [ + { + status: 'Enabled', + filter: {}, + noncurrentVersionExpiration: { noncurrentDays: 30 }, + }, + ], + }, + }) + await expect(tHelper.database.findLifecycleBucket(analyticsBucketId)).rejects.toMatchObject({ + code: 'InvalidRequest', + }) + await expect( + tHelper.database.putLifecycleConfiguration(analyticsBucketId, bucket.lifecycle_configuration!) + ).rejects.toMatchObject({ code: 'InvalidRequest', httpStatusCode: 400 }) + }) + + test('lists, claims, and revalidates only Standard-bucket state', async () => { + const analyticsBucketId = await fixtures.createBucket('ANALYTICS') + await putConfiguration() + + await tHelper.database.connection.query( + `INSERT INTO storage.bucket_lifecycle_states ( + bucket_id, next_run_at + ) VALUES ($1, now())`, + [analyticsBucketId] + ) + + const due = await tHelper.database.listDueLifecycleBuckets(100) + expect(due).toContainEqual(bucketId) + expect(due.some((id) => id === analyticsBucketId)).toBe(false) + + const claimId = randomUUID() + await expect( + tHelper.database.claimLifecycleBucket({ + bucketId: analyticsBucketId, + claimId: randomUUID(), + leaseMs: 60_000, + }) + ).resolves.toBeUndefined() + + await expect( + tHelper.database.claimLifecycleBucket({ bucketId, claimId, leaseMs: 60_000 }) + ).resolves.toMatchObject({ claimId }) + + await expect( + tHelper.database.revalidateLifecycleClaim({ bucketId, claimId }) + ).resolves.toMatchObject({ claimId }) + + await expect( + tHelper.database.revalidateLifecycleClaim({ bucketId, claimId: randomUUID() }) + ).resolves.toBeUndefined() + }) + + test.each([Number.MAX_SAFE_INTEGER, 800_000, 2_460_000])( + 'evaluates a policy with %i days without timestamp overflow', + async (noncurrentDays) => { + await insertArchivedObjects([{ name: 'overflow', archivedAt: '2026-01-01T00:00:00.000Z' }]) + const snapshotAt = new Date('2026-08-15T00:00:00Z') + const rules = compileLifecycleEvaluationRules( + { + rules: [ + { + status: 'Enabled', + filter: {}, + noncurrentVersionExpiration: { noncurrentDays }, + }, + ], + }, + snapshotAt + ) + await expect( + tHelper.database.evaluateNoncurrentLifecyclePage({ + bucketId, + snapshotAt: snapshotAt.toISOString(), + pageSize: 10, + rules, + }) + ).resolves.toEqual({ rawRowsExamined: 0, candidates: [], exhausted: true }) + } + ) + + test('evaluates all rules in the database while raw-page progress stays selectivity-independent', async () => { + await insertArchivedObjects([ + { name: 'old-a', archivedAt: '2026-07-01T00:00:00.000Z' }, + { name: 'old-b', archivedAt: '2026-07-02T00:00:00.000Z' }, + { name: 'young', archivedAt: '2026-08-10T00:00:00.000Z' }, + ]) + + const page = await evaluatePage([ + { cutoffAt: '2026-08-01T00:00:00.000Z' }, + { cutoffAt: '2026-07-15T00:00:00.000Z' }, + ]) + + expect(page.rawRowsExamined).toBe(2) + expect(page.exhausted).toBe(true) + expect(page.pageEnd).toBeDefined() + expect(page.candidates.map((candidate) => candidate.name).sort()).toEqual(['old-a', 'old-b']) + expect(new Set(page.candidates.map((candidate) => candidate.version)).size).toBe(2) + }) + + test('excludes rows archived exactly at each rule cutoff', async () => { + await insertArchivedObjects([ + { name: 'at-first-cutoff', archivedAt: '2026-07-15T00:00:00.000Z' }, + { name: 'at-latest-cutoff', archivedAt: '2026-08-01T00:00:00.000Z' }, + ]) + + await expect( + evaluatePage([ + { cutoffAt: '2026-07-15T00:00:00.000Z' }, + { cutoffAt: '2026-08-01T00:00:00.000Z', newerNoncurrentVersions: 1 }, + ]) + ).resolves.toMatchObject({ rawRowsExamined: 1, candidates: [] }) + }) + + test('skips an archived row with an empty version instead of failing evaluation', async () => { + await insertArchivedObjects([ + { name: 'blank-version', archivedAt: '2026-07-01T00:00:00.000Z', version: '' }, + { name: 'valid', archivedAt: '2026-07-02T00:00:00.000Z' }, + ]) + + const page = await evaluatePage([{ cutoffAt: '2026-08-01T00:00:00.000Z' }]) + + expect(page.rawRowsExamined).toBe(2) + expect(page.exhausted).toBe(true) + expect(page.candidates.map((candidate) => candidate.name)).toEqual(['valid']) + }) + + test.each([ + { label: 'an empty name', name: '', archivedAt: '2026-07-01T00:00:00.000Z' }, + { label: 'a NULL name', name: null, archivedAt: '2026-07-01T00:00:00.000Z' }, + { label: 'a BC archived_at', name: 'poison', archivedAt: '0044-03-15T00:00:00Z BC' }, + { label: 'a -infinity archived_at', name: 'poison', archivedAt: '-infinity' }, + ])('skips an archived row with $label instead of failing evaluation', async (poison) => { + await insertArchivedObjects([ + { name: poison.name as string, archivedAt: poison.archivedAt }, + { name: 'valid', archivedAt: '2026-07-02T00:00:00.000Z' }, + ]) + + const page = await evaluatePage([{ cutoffAt: '2026-08-01T00:00:00.000Z' }]) + + expect(page.rawRowsExamined).toBe(1) + expect(page.exhausted).toBe(true) + expect(page.candidates.map((candidate) => candidate.name)).toEqual(['valid']) + expect(page.pageEnd?.name).toBe('valid') + }) + + test('renders the page cursor in UTC under a session zone with a seconds offset', async () => { + await insertArchivedObjects([{ name: 'old', archivedAt: '1970-06-01T00:00:00Z' }]) + const statement = buildEvaluateNoncurrentLifecyclePageStatement({ + bucketId, + snapshotAt: SNAPSHOT_AT, + pageSize: 480, + rules: [{ cutoffAt: '2026-08-01T00:00:00.000Z' }], + }) + + const client = new Client({ connectionString: getConfig().databaseURL }) + await client.connect() + try { + await client.query("SET TIME ZONE 'Africa/Monrovia'") + const result = await client.query(statement) + const page = mapLifecycleEvaluationPage(result.rows[0], 480) + expect(page.pageEnd).toEqual({ name: 'old', archivedAt: '1970-06-01T00:00:00.000000Z' }) + } finally { + await client.end() + } + }) + + test('reconsiders an age-due row when a newer sibling later satisfies the retention count', async () => { + const name = 'history/count-protected' + const versions = [randomUUID(), randomUUID(), randomUUID()] + const insertArchivedVersion = (version: string, archivedAt: string) => + insertArchivedObjects([{ name, version, archivedAt }]) + + await insertArchivedVersion(versions[0], '2026-06-01T00:00:00.000Z') + await insertArchivedVersion(versions[1], '2026-06-02T00:00:00.000Z') + + const evaluateFromRangeStart = () => + evaluatePage([{ cutoffAt: '2026-08-01T00:00:00.000Z', newerNoncurrentVersions: 2 }]) + + await expect(evaluateFromRangeStart()).resolves.toMatchObject({ + rawRowsExamined: 2, + candidates: [], + }) + + await insertArchivedVersion(versions[2], '2026-06-03T00:00:00.000Z') + + const reconsidered = await evaluateFromRangeStart() + expect(reconsidered.rawRowsExamined).toBe(3) + expect(reconsidered.candidates).toEqual([ + expect.objectContaining({ name, version: versions[0] }), + ]) + }) + + test.each([ + { label: 'in another bucket', sibling: 'other' }, + { label: 'archived after the snapshot', sibling: 'late' }, + ])('does not count a newer sibling $label', async ({ sibling }) => { + const name = 'history/count-scope' + await insertArchivedObjects([ + { name, archivedAt: '2026-06-01T00:00:00.000Z' }, + ...(sibling === 'late' ? [{ name, archivedAt: '2026-09-01T00:00:00.000Z' }] : []), + ]) + if (sibling === 'other') await insertOtherBucketSibling(name, '2026-07-01T00:00:00.000Z') + + await expect( + evaluatePage([{ cutoffAt: '2026-08-01T00:00:00.000Z', newerNoncurrentVersions: 1 }]) + ).resolves.toMatchObject({ candidates: [] }) + }) + + test.each([1, 17, 480])( + 'produces the same candidates and counters with page size %i', + async (pageSize) => { + await insertArchivedObjects( + Array.from({ length: 17 }, (_, index) => + ['2026-06-01T00:00:00.000Z', '2026-06-02T00:00:00.000Z'].map((archivedAt) => ({ + name: `history/page-${index.toString().padStart(2, '0')}`, + archivedAt, + })) + ).flat() + ) + + let cursor: NoncurrentLifecycleCursor | undefined + let rawRowsExamined = 0 + const candidates: string[] = [] + for (;;) { + const page = await evaluatePage( + [{ cutoffAt: '2026-08-01T00:00:00.000Z', newerNoncurrentVersions: 1 }], + { pageSize, cursor } + ) + rawRowsExamined += page.rawRowsExamined + candidates.push(...page.candidates.map((candidate) => candidate.name)) + if (page.exhausted) break + cursor = page.pageEnd + } + + expect(rawRowsExamined).toBe(34) + expect(candidates).toEqual( + [...Array(17).keys()].map((index) => `history/page-${index.toString().padStart(2, '0')}`) + ) + } + ) + + describe('retained-version checks against explicit history fixtures', () => { + async function stageCountProtectedVersion( + options: { + legacy?: boolean + marker?: boolean + metadata?: Record | null + suspended?: boolean + } = {} + ) { + const generation = await putConfiguration(1) + if (options.suspended) { + await fixtures.setStatus(bucketId, 'SUSPENDED') + } + const name = 'history/retention-count-race' + const olderId = randomUUID() + const olderVersion = options.legacy ? null : randomUUID() + const newerVersion = randomUUID() + const writtenAt = '2000-01-01T00:00:00.000Z' + await withServiceOperation('storage.object.upload', async (db) => { + await db.query( + `INSERT INTO storage.objects ( + id, bucket_id, name, version, created_at, archived_at, is_versioned, metadata, is_delete_marker + ) VALUES + ($1, $2, $3, $4, $6::timestamptz, '2001-01-01', $8, $7::jsonb, $9), + (gen_random_uuid(), $2, $3, $5, $6::timestamptz, '2001-02-01', true, $10::jsonb, false)`, + [ + olderId, + bucketId, + name, + olderVersion, + newerVersion, + writtenAt, + options.metadata === null ? null : JSON.stringify(options.metadata ?? { size: 10 }), + !options.legacy, + options.marker ?? false, + JSON.stringify({ size: 10 }), + ] + ) + }) + const identity = { bucketId, claimId: randomUUID() } + await tHelper.database.claimLifecycleBucket({ ...identity, leaseMs: 60_000 }) + const snapshotAt = '2001-06-01T00:00:00.000Z' + const evaluation = { + bucketId, + snapshotAt, + rules: [{ cutoffAt: '2001-05-01T00:00:00.000Z', newerNoncurrentVersions: 1 }], + pageSize: 10, + } + const page = await tHelper.database.evaluateNoncurrentLifecyclePage(evaluation) + expect(page.candidates).toEqual([ + expect.objectContaining({ + id: olderId, + name, + version: olderVersion, + isDeleteMarker: options.marker ?? false, + }), + ]) + const armed = armedContinuation(generation, page.candidates, page.pageEnd!, { + snapshotAt, + }) + const arm = () => + tHelper.database.armLifecycleAttempt({ + ...identity, + leaseMs: 60_000, + continuation: armed, + }) + return { + identity, + olderId, + armed, + arm, + evaluation, + name, + olderVersion, + newerVersion, + } + } + + test.each([ + { label: 'legacy null version', legacy: true, metadata: { size: 10 } }, + { label: 'version without extra metadata', metadata: { size: 10 } }, + { label: 'metadata-null marker', marker: true, metadata: null }, + { label: 'suspended history', suspended: true, metadata: { size: 10 } }, + ])( + 'arms $label using the compatible row identity and policy', + async ({ label: _label, ...options }) => { + const fixture = await stageCountProtectedVersion(options) + await expect(fixture.arm()).resolves.toMatchObject({ + continuation: { + batch: { inFlight: { attemptId: fixture.armed.batch!.inFlight!.attemptId } }, + }, + }) + await expect( + tHelper.database.connection.query('SELECT 1 FROM storage.objects WHERE id = $1', [ + fixture.olderId, + ]) + ).resolves.toMatchObject({ rowCount: 0 }) + } + ) + + test('rechecks retention at arm time after a raw fixture history change', async () => { + const fixture = await stageCountProtectedVersion() + await withServiceOperation('storage.lifecycle.test_fixture', async (transaction) => { + await transaction.query( + 'DELETE FROM storage.objects WHERE bucket_id = $1 AND name = $2 AND version = $3', + [bucketId, fixture.name, fixture.newerVersion] + ) + }) + await expect(fixture.arm()).resolves.toBeUndefined() + await expect( + tHelper.database.revalidateLifecycleClaim(fixture.identity) + ).resolves.toBeUndefined() + await expect( + tHelper.database.claimLifecycleBucket({ + ...fixture.identity, + claimId: randomUUID(), + leaseMs: 60_000, + }) + ).resolves.toMatchObject({ continuation: null }) + await expect( + tHelper.database.evaluateNoncurrentLifecyclePage(fixture.evaluation) + ).resolves.toMatchObject({ candidates: [] }) + }) + }) + it('records artifact outcomes and advances only once every artifact is resolved', async () => { + const generation = await putConfiguration() + const name = 'history/a-expired.txt' + const version = randomUUID() + const id = randomUUID() + const completedName = 'history/b-completed.txt' + const completedVersion = randomUUID() + const completedId = randomUUID() + await fixtures.withServiceOperation('storage.s3.bucket.put_lifecycle', async (transaction) => { + for (const [objectId, objectName, objectVersion, size] of [ + [id, name, version, 42], + [completedId, completedName, completedVersion, 8], + ]) { + await transaction.query( + `INSERT INTO storage.objects (id, bucket_id, name, version, metadata, archived_at, is_versioned) + VALUES ($1, $2, $3, $4, $5::jsonb, '2026-07-02', true)`, + [objectId, bucketId, objectName, objectVersion, JSON.stringify({ size })] + ) + } + }) + + const claimId = randomUUID() + const claimed = await tHelper.database.claimLifecycleBucket({ + bucketId, + claimId, + leaseMs: 120_000, + }) + expect(claimed).toMatchObject({ claimId }) + const candidate = { + id, + name, + version, + isDeleteMarker: false, + archivedAt: '2026-07-02T00:00:00.000Z', + } + const completedCandidate = { + ...candidate, + id: completedId, + name: completedName, + version: completedVersion, + } + const armed = armedContinuation( + generation, + [candidate, completedCandidate], + { name: completedName, archivedAt: completedCandidate.archivedAt }, + { snapshotAt: '2026-09-01T00:00:00.000Z' } + ) + const armedState = await tHelper.database.armLifecycleAttempt({ + bucketId, + claimId, + leaseMs: 120_000, + continuation: armed, + }) + expect(armedState?.continuation).toMatchObject({ + batch: { inFlight: { attemptId: armed.batch!.inFlight!.attemptId } }, + }) + + const partiallyRecorded = recordLifecycleArtifactOutcomes( + { ...armed, counters: armedState!.continuation!.counters }, + new Map([ + [ + armed.batch!.versions[0].artifacts[0].key, + { outcome: 'UNRESOLVED' as const, error: 'retryable backend failure' }, + ], + [armed.batch!.versions[0].artifacts[1].key, { outcome: 'DELETED' as const }], + ...armed.batch!.versions[1].artifacts.map( + (artifact) => [artifact.key, { outcome: 'DELETED' as const }] as const + ), + ]) + ) + const partial = await tHelper.database.commitLifecycleAttempt({ + bucketId, + claimId, + attemptId: armed.batch!.inFlight!.attemptId, + continuation: partiallyRecorded, + }) + expect(partial?.continuation).toMatchObject({ + batch: { + inFlight: { attemptId: armed.batch!.inFlight!.attemptId }, + versions: [ + { + artifacts: [ + expect.objectContaining({ outcome: 'UNRESOLVED' }), + expect.objectContaining({ outcome: 'DELETED' }), + ], + }, + ], + }, + counters: { objectVersionsDeleted: 2, bytesDeleted: 50, batchesCompleted: 0 }, + }) + + const recorded = recordLifecycleArtifactOutcomes( + partial!.continuation!, + new Map([[armed.batch!.versions[0].artifacts[0].key, { outcome: 'DELETED' as const }]]) + ) + const committed = await tHelper.database.commitLifecycleAttempt({ + bucketId, + claimId, + attemptId: armed.batch!.inFlight!.attemptId, + continuation: recorded, + }) + expect(committed?.continuation).toMatchObject({ + cursor: { name: completedName, archivedAt: candidate.archivedAt }, + counters: { objectVersionsDeleted: 2, bytesDeleted: 50, batchesCompleted: 1 }, + }) + }) + describe('bucket queries', () => { + const claim = (id = bucketId, leaseMs = 60_000) => + tHelper.database.claimLifecycleBucket({ bucketId: id, claimId: randomUUID(), leaseMs }) + const setState = (sets: string, id = bucketId, values: unknown[] = []) => + tHelper.database.connection.query( + `UPDATE storage.bucket_lifecycle_states SET ${sets} WHERE bucket_id = $1`, + [id, ...values] + ) + const dispatchState = (options: { inFlightOnly?: boolean } = {}) => + tHelper.database.findLifecycleDispatchState({ runBudgetMs: 20_000, ...options }) + const listed = async (id: string) => + (await tHelper.database.listDueLifecycleBuckets(1000)).some((bucketId) => bucketId === id) + + test('excludes a live claim from the due list and from the next dispatch time', async () => { + await putConfiguration() + expect(await listed(bucketId)).toBe(true) + await expect(claim()).resolves.toBeDefined() + + expect(await listed(bucketId)).toBe(false) + const { nextDispatchAt: next } = await dispatchState() + expect(next!.getTime()).toBeGreaterThan(Date.now()) + + await setState("claim_until = now() - interval '1 second'") + expect(await listed(bucketId)).toBe(true) + }) + + test.each([ + { + name: 'the run budget when it ends before the lease', + leaseMs: 3_600_000, + expectedMs: 20_000, + }, + { + name: 'the lease expiry when it ends before the run budget', + leaseMs: 5_000, + expectedMs: 5_000, + }, + ])('reports a claimed bucket at $name rather than as due', async ({ leaseMs, expectedMs }) => { + await putConfiguration() + await setState("next_run_at = '2000-01-01'") + const startedAt = new Date() + await setState( + `claim_id = gen_random_uuid(), last_started_at = $2, claim_until = $3`, + bucketId, + [startedAt, new Date(startedAt.getTime() + leaseMs)] + ) + + const { nextDispatchAt: next } = await dispatchState() + expect(next).toEqual(new Date(startedAt.getTime() + expectedMs)) + }) + + test('cannot claim a bucket that is not yet due', async () => { + await putConfiguration() + await setState("next_run_at = now() + interval '1 hour'") + expect(await listed(bucketId)).toBe(false) + await expect(claim()).resolves.toBeUndefined() + }) + + test('stops revalidating a claim once its lease has passed', async () => { + await putConfiguration() + const claimId = randomUUID() + await tHelper.database.claimLifecycleBucket({ bucketId, claimId, leaseMs: 60_000 }) + await expect( + tHelper.database.revalidateLifecycleClaim({ bucketId, claimId }) + ).resolves.toMatchObject({ claimId }) + + await setState("claim_until = now() - interval '1 second'") + await expect( + tHelper.database.revalidateLifecycleClaim({ bucketId, claimId }) + ).resolves.toBeUndefined() + }) + + test('release keeps the last error on a clean release and stamps completion', async () => { + await putConfiguration() + const claimId = randomUUID() + await tHelper.database.claimLifecycleBucket({ bucketId, claimId, leaseMs: 60_000 }) + await tHelper.database.releaseLifecycleClaim({ + bucketId, + claimId, + nextRunAt: new Date().toISOString(), + error: { message: 'boom' }, + }) + await expect( + fixtures.readState(bucketId, 'failure_count, last_error, last_completed_at') + ).resolves.toMatchObject([ + { failure_count: 1, last_error: { message: 'boom' }, last_completed_at: expect.any(Date) }, + ]) + + const second = randomUUID() + await expect( + tHelper.database.claimLifecycleBucket({ bucketId, claimId: second, leaseMs: 60_000 }) + ).resolves.toBeDefined() + await setState('last_completed_at = NULL') + await tHelper.database.releaseLifecycleClaim({ + bucketId, + claimId: second, + nextRunAt: new Date().toISOString(), + }) + await expect( + fixtures.readState(bucketId, 'failure_count, last_error, last_completed_at') + ).resolves.toMatchObject([ + { failure_count: 0, last_error: { message: 'boom' }, last_completed_at: expect.any(Date) }, + ]) + }) + + test('completes a run only for the claim that holds the bucket', async () => { + await putConfiguration() + const claimId = randomUUID() + await tHelper.database.claimLifecycleBucket({ bucketId, claimId, leaseMs: 60_000 }) + await setState(`continuation = '{"continuationVersion": 1}'::jsonb`) + + await expect( + tHelper.database.completeLifecycleRun( + { bucketId, claimId: randomUUID() }, + {}, + new Date().toISOString() + ) + ).resolves.toBe(false) + await expect(fixtures.readState(bucketId, 'continuation, claim_id')).resolves.toEqual([ + { continuation: { continuationVersion: 1 }, claim_id: claimId }, + ]) + }) + + test('keeps a policy written under a racing claim due after the claimed run completes', async () => { + await putConfiguration() + await setState(`next_run_at = '2000-01-01'`) + const stale = (await tHelper.database.findLifecycleBucket(bucketId)) + .lifecycle_configuration_generation + const writer = new Client({ connectionString: getConfig().databaseURL }) + await writer.connect() + try { + await writer.query('BEGIN') + const pid = (await writer.query('SELECT pg_backend_pid() AS pid')).rows[0].pid + await writer.query('SET LOCAL ROLE service_role') + await writer.query( + `UPDATE storage.buckets SET lifecycle_configuration = $2::jsonb, + lifecycle_configuration_generation = gen_random_uuid() WHERE id = $1`, + [bucketId, JSON.stringify(configuration(1))] + ) + const identity = { bucketId, claimId: randomUUID() } + const claim = tHelper.database.claimLifecycleBucket({ ...identity, leaseMs: 60_000 }) + await expect + .poll( + async () => + ( + await tHelper.database.connection.query( + 'SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE $1 = ANY(pg_blocking_pids(pid))) AS blocked', + [pid] + ) + ).rows[0].blocked + ) + .toBe(true) + await writer.query('COMMIT') + expect((await claim)?.bucket.lifecycle_configuration_generation).toBe(stale) + + await expect( + tHelper.database.completeLifecycleRun( + identity, + { generation: stale }, + new Date(Date.now() + 86_400_000).toISOString() + ) + ).resolves.toBe(true) + await expect(dueNow()).resolves.toBe(true) + } finally { + await writer.end() + } + }) + + test('does not report a claimed bucket that has no scheduled run', async () => { + await putConfiguration() + await expect(claim()).resolves.toBeDefined() + await setState('next_run_at = NULL') + + await expect(dispatchState()).resolves.toMatchObject({ nextDispatchAt: null }) + }) + + test('restricts due buckets and the next dispatch time to journals in flight on request', async () => { + const other = await fixtures.createBucket() + for (const id of [bucketId, other]) { + await fixtures.setStatus(id, 'ENABLED') + await tHelper.database.putLifecycleConfiguration(id, { + rules: [ + { status: 'Enabled', filter: {}, noncurrentVersionExpiration: { noncurrentDays: 30 } }, + ], + }) + } + await setState("next_run_at = '2000-01-01'", other) + await setState("next_run_at = '2000-01-02'", bucketId) + const generation = (await tHelper.database.findLifecycleBucket(bucketId)) + .lifecycle_configuration_generation! + const continuation = armedContinuation( + generation, + [{ id: randomUUID(), name: 'a', version: randomUUID(), isDeleteMarker: false }], + { name: 'a', archivedAt: new Date().toISOString() } + ) + await setState('continuation = $2::jsonb', bucketId, [JSON.stringify(continuation)]) + + const all = await tHelper.database.listDueLifecycleBuckets(1000) + expect(all).toEqual(expect.arrayContaining([bucketId, other])) + expect(all.indexOf(other)).toBeLessThan(all.indexOf(bucketId)) + const filtered = await tHelper.database.listDueLifecycleBuckets(1000, { inFlightOnly: true }) + expect(filtered).toContain(bucketId) + expect(filtered).not.toContain(other) + + const { nextDispatchAt: next } = await dispatchState({ inFlightOnly: true }) + expect(next).toEqual(new Date('2000-01-02')) + await expect(dispatchState()).resolves.toMatchObject({ + nextDispatchAt: new Date('2000-01-01'), + }) + }) + }) + + test('reports parked failures and in-flight age without decoding journals', async () => { + await putConfiguration() + const other = await fixtures.createBucket() + await tHelper.database.putLifecycleConfiguration(other, { + rules: [ + { status: 'Enabled', filter: {}, noncurrentVersionExpiration: { noncurrentDays: 1 } }, + ], + }) + await tHelper.database.connection.query( + `UPDATE storage.bucket_lifecycle_states SET next_run_at = NULL, failure_count = 9 WHERE bucket_id = $1`, + [bucketId] + ) + await tHelper.database.connection.query( + `UPDATE storage.bucket_lifecycle_states SET next_run_at = '2026-01-01', failure_count = 3, + continuation = '{"continuationVersion":999,"batch":{"inFlight":{"startedAt":"2025-12-01T00:00:00.000Z"}}}' + WHERE bucket_id = $1`, + [other] + ) + await expect( + tHelper.database.findLifecycleDispatchState({ inFlightOnly: true, runBudgetMs: 20_000 }) + ).resolves.toEqual({ + nextDispatchAt: new Date('2026-01-01'), + maxFailureCount: 9, + oldestInFlightStartedAt: '2025-12-01T00:00:00.000Z', + hasAbandonedArtifacts: false, + }) + await tHelper.database.connection.query( + `UPDATE storage.bucket_lifecycle_states SET continuation = jsonb_set(continuation, '{batch,inFlight,startedAt}', '"invalid"') WHERE bucket_id = $1`, + [other] + ) + await expect( + tHelper.database.findLifecycleDispatchState({ runBudgetMs: 20_000 }) + ).resolves.toMatchObject({ + nextDispatchAt: new Date('2026-01-01'), + maxFailureCount: 9, + oldestInFlightStartedAt: null, + }) + }) + + describe('arm and commit coordination', () => { + async function armBatch( + names: string[], + options: { + archivedAt?: string + snapshotAt?: string + newerNoncurrentVersions?: number + beforeConfiguration?: () => Promise + versions?: string[] + } = {} + ) { + const rows = names.map((name, index) => ({ + id: randomUUID(), + name, + version: options.versions?.[index] ?? randomUUID(), + archivedAt: + options.archivedAt ?? + (options.versions ? `2026-06-01T00:00:00.00${index}Z` : '2026-06-01T00:00:00.000Z'), + })) + await insertArchivedObjects(rows) + const generation = await putConfiguration( + options.newerNoncurrentVersions, + options.beforeConfiguration + ) + const identity = { bucketId, claimId: randomUUID() } + await tHelper.database.claimLifecycleBucket({ ...identity, leaseMs: 60_000 }) + const staged = stagedContinuation( + generation, + rows.map(({ id, name, version }) => ({ id, name, version, isDeleteMarker: false })), + { name: rows.at(-1)!.name, archivedAt: rows.at(-1)!.archivedAt }, + options + ) + const armed = armLifecycleAttempt(staged, { + attemptId: randomUUID(), + startedAt: new Date().toISOString(), + }) + const input = { + ...identity, + leaseMs: 60_000, + continuation: armed, + } + const recorded = () => + recordLifecycleArtifactOutcomes( + armed, + new Map( + armed.batch!.versions.flatMap((version) => + version.artifacts.map((artifact) => [artifact.key, { outcome: 'DELETED' as const }]) + ) + ) + ) + return { + identity, + rows, + generation, + staged, + armed, + input, + arm: () => tHelper.database.armLifecycleAttempt(input), + commit: (continuation = recorded()) => + tHelper.database.commitLifecycleAttempt({ + ...identity, + attemptId: armed.batch!.inFlight!.attemptId, + continuation, + }), + recorded, + } + } + + test('rejects oversized new arms before deleting metadata or persisting a journal', async () => { + const fixture = await armBatch(Array.from({ length: 49 }, (_, index) => `oversized/${index}`)) + + await expect(fixture.arm()).rejects.toMatchObject({ code: 'InvalidParameter' }) + const remaining = await tHelper.database.connection.query( + 'SELECT count(*)::integer AS count FROM storage.objects WHERE bucket_id = $1', + [bucketId] + ) + expect(remaining.rows).toEqual([{ count: 49 }]) + await expect(fixtures.readState(bucketId, 'continuation')).resolves.toEqual([ + { continuation: null }, + ]) + }) + + test('rejects arm and commit inputs whose journal does not describe an armed attempt', async () => { + const fixture = await armBatch(['invalid-input']) + + await expect( + tHelper.database.armLifecycleAttempt({ ...fixture.input, continuation: fixture.staged }) + ).rejects.toMatchObject({ code: 'InvalidParameter' }) + await expect( + tHelper.database.commitLifecycleAttempt({ + ...fixture.identity, + attemptId: randomUUID(), + continuation: fixture.recorded(), + }) + ).rejects.toMatchObject({ code: 'InvalidParameter' }) + }) + + test.each([ + { state: 'armed', explicit: false, capped: true }, + { state: 'armed', explicit: true, capped: false }, + { state: 'unreadable armed', explicit: false, capped: true }, + { state: 'unreadable idle', explicit: false, capped: false }, + { state: 'empty', explicit: false, capped: false }, + ])( + 'caps only preserved journals on release: $state, explicit=$explicit', + async ({ state, explicit, capped }) => { + const fixture = await armBatch(['retry-cap']) + await expect(fixture.arm()).resolves.toBeDefined() + const continuation = + state === 'armed' + ? encodeLifecycleContinuation(fixture.armed) + : state === 'empty' + ? null + : { + continuationVersion: 999, + ...(state === 'unreadable armed' && { + batch: { inFlight: { attemptId: randomUUID() } }, + }), + } + await tHelper.database.connection.query( + `UPDATE storage.bucket_lifecycle_states + SET continuation = $2::jsonb, claim_until = '2000-01-01' WHERE bucket_id = $1`, + [bucketId, continuation === null ? null : JSON.stringify(continuation)] + ) + const nextRunAt = '2099-01-01T01:00:00.000Z' + const maxInFlightRetryAt = '2099-01-01T00:15:00.000Z' + await expect( + tHelper.database.releaseLifecycleClaim({ + ...fixture.identity, + ...(explicit && { continuation: fixture.armed }), + nextRunAt, + maxInFlightRetryAt, + error: { message: 'lost acknowledgement' }, + }) + ).resolves.toBe(true) + await expect( + fixtures.readState(bucketId, 'continuation, claim_id, next_run_at') + ).resolves.toEqual([ + { + continuation, + claim_id: null, + next_run_at: new Date(capped ? maxInFlightRetryAt : nextRunAt), + }, + ]) + } + ) + + test.each([ + { continuation: 'idle', error: false, expected: 0 }, + { continuation: 'armed', error: false, expected: 7 }, + { continuation: 'idle', error: true, expected: 8 }, + ])( + 'release with $continuation journal, error=$error leaves failure_count $expected', + async ({ continuation, error, expected }) => { + const fixture = await armBatch(['failure-count']) + await expect(fixture.arm()).resolves.toBeDefined() + await tHelper.database.connection.query( + 'UPDATE storage.bucket_lifecycle_states SET failure_count = 7 WHERE bucket_id = $1', + [bucketId] + ) + await expect( + tHelper.database.releaseLifecycleClaim({ + ...fixture.identity, + continuation: continuation === 'armed' ? fixture.armed : fixture.staged, + nextRunAt: '2099-01-01T01:00:00.000Z', + ...(error && { error: { message: 'boom' } }), + }) + ).resolves.toBe(true) + await expect(fixtures.readState(bucketId, 'failure_count')).resolves.toEqual([ + { failure_count: expected }, + ]) + } + ) + + test.each([ + { journal: 'preserved', outcome: 'InvalidParameter' }, + { journal: 'armed', outcome: 'InvalidParameter' }, + { journal: 'staged', outcome: true }, + { journal: 'empty', outcome: true }, + ] as const)( + 'release without a schedule with $journal journal: $outcome', + async ({ journal, outcome }) => { + const fixture = await armBatch(['release/unscheduled']) + await expect(fixture.arm()).resolves.toBeDefined() + const continuation = { + preserved: undefined, + armed: fixture.armed, + staged: fixture.staged, + empty: null, + }[journal] + + await expect( + tHelper.database + .releaseLifecycleClaim({ ...fixture.identity, continuation, nextRunAt: null }) + .catch((error: { code: string }) => error.code) + ).resolves.toBe(outcome) + } + ) + + const expireClaim = () => + tHelper.database.connection.query( + "UPDATE storage.bucket_lifecycle_states SET claim_until = now() - interval '1 second' WHERE bucket_id = $1", + [bucketId] + ) + + test('rejects arming once the claim lease has expired', async () => { + const fixture = await armBatch(['arm/expired']) + await expireClaim() + + await expect(fixture.arm()).resolves.toBeUndefined() + await expect(fixtures.readState(bucketId, 'continuation')).resolves.toEqual([ + { continuation: null }, + ]) + }) + + test('rejects arming over a journal in flight even when its rows are eligible again', async () => { + const fixture = await armBatch(['arm/rearmed']) + await expect(fixture.arm()).resolves.toBeDefined() + await insertArchivedObjects(fixture.rows) + const before = await fixtures.readState(bucketId, 'continuation') + + await expect(fixture.arm()).resolves.toBeUndefined() + await expect(fixtures.readState(bucketId, 'continuation')).resolves.toEqual(before) + }) + + test.each([ + { + change: 'versioning is disabled', + sql: "UPDATE storage.buckets SET versioning_status = 'DISABLED' WHERE id = $1", + claimKept: true, + }, + { + change: 'the version became a delete marker', + sql: 'UPDATE storage.objects SET is_delete_marker = true WHERE bucket_id = $1', + claimKept: false, + }, + { + change: 'a policy replacement that keeps the row eligible', + sql: `UPDATE storage.buckets SET lifecycle_configuration_generation = gen_random_uuid(), + lifecycle_configuration = jsonb_set(lifecycle_configuration, + '{rules,0,noncurrentVersionExpiration,noncurrentDays}', '1') + WHERE id = $1`, + claimKept: true, + }, + ])('does not arm a staged batch after $change', async ({ sql, claimKept }) => { + const fixture = await armBatch(['arm/recheck']) + const writer = await holdWriter( + sql, + [bucketId], + 'SET LOCAL session_replication_role = replica' + ) + await writer.finish('COMMIT') + + await expect(fixture.arm()).resolves.toBeUndefined() + await expect(fixtures.readState(bucketId, 'continuation, claim_id')).resolves.toEqual([ + { continuation: null, claim_id: claimKept ? fixture.identity.claimId : null }, + ]) + }) + + test('does not arm a batch when only one of its rows lost eligibility', async () => { + const fixture = await armBatch(['arm/partial-a', 'arm/partial-b']) + const writer = await holdWriter( + 'UPDATE storage.objects SET is_delete_marker = true WHERE bucket_id = $1 AND name = $2', + [bucketId, 'arm/partial-b'], + 'SET LOCAL session_replication_role = replica' + ) + await writer.finish('COMMIT') + + await expect(fixture.arm()).resolves.toBeUndefined() + await expect(fixtures.readState(bucketId, 'continuation, claim_id')).resolves.toEqual([ + { continuation: null, claim_id: null }, + ]) + }) + + test('persists the armed journal without artifact keys', async () => { + const fixture = await armBatch(['arm/no-keys']) + await expect(fixture.arm()).resolves.toBeDefined() + + await expect( + fixtures.readState( + bucketId, + "jsonb_path_exists(continuation, '$.batch.versions[*].artifacts[*].key') AS has_key, jsonb_array_length(continuation #> '{batch,versions,0,artifacts}') AS artifacts" + ) + ).resolves.toEqual([{ has_key: false, artifacts: 2 }]) + }) + + async function reference(id: string) { + await tHelper.database.connection.query( + 'CREATE TABLE public.lifecycle_arm_refs (object_id uuid REFERENCES storage.objects(id))' + ) + await tHelper.database.connection.query('INSERT INTO public.lifecycle_arm_refs VALUES ($1)', [ + id, + ]) + } + const dropReferences = () => + tHelper.database.connection.query('DROP TABLE IF EXISTS public.lifecycle_arm_refs') + + test('arms only the versions whose metadata delete is not vetoed', async () => { + const fixture = await armBatch(['arm/vetoed', 'arm/kept']) + try { + await reference(fixture.rows[0].id) + const state = await fixture.arm() + + expect(state?.claimId).toBe(fixture.identity.claimId) + expect(state?.continuation?.batch?.versions.map(({ id }) => id)).toEqual([ + fixture.rows[1].id, + ]) + expect(state?.continuation?.counters).toMatchObject({ objectVersionsDeleted: 1 }) + await expect( + fixtures.readState( + bucketId, + "jsonb_path_exists(continuation, '$.batch.versions[*].artifacts[*].key') AS has_key" + ) + ).resolves.toEqual([{ has_key: false }]) + expect(await rowExists(fixture.rows[0])).toBe(true) + expect(await rowExists(fixture.rows[1])).toBe(false) + } finally { + await dropReferences() + } + }) + + test('advances the cursor past a page whose every version is vetoed without arming', async () => { + const fixture = await armBatch(['arm/all-vetoed']) + const { batch: _batch, pageEnd, ...rest } = fixture.staged + const skipped = { ...rest, cursor: pageEnd } + try { + await reference(fixture.rows[0].id) + const state = await fixture.arm() + + expect(state?.claimId).toBe(fixture.identity.claimId) + expect(state?.continuation).toEqual(skipped) + await expect( + fixtures.readState( + bucketId, + 'continuation, claim_id, claim_until > now() AS claimed, next_run_at <= now() AS due' + ) + ).resolves.toEqual([ + { continuation: skipped, claim_id: fixture.identity.claimId, claimed: true, due: true }, + ]) + } finally { + await dropReferences() + } + }) + + test('does not skip a vetoed page once the claim has expired', async () => { + const fixture = await armBatch(['arm/vetoed-guard']) + await expireClaim() + try { + await reference(fixture.rows[0].id) + + await expect(fixture.arm()).resolves.toBeUndefined() + await expect(fixtures.readState(bucketId, 'continuation')).resolves.toEqual([ + { continuation: null }, + ]) + } finally { + await dropReferences() + } + }) + + test('persists the committed results without artifact keys', async () => { + const fixture = await armBatch(['commit/no-keys']) + await expect(fixture.arm()).resolves.toBeDefined() + await expect(fixture.commit(fixture.armed)).resolves.toBeDefined() + + await expect( + fixtures.readState( + bucketId, + "jsonb_path_exists(continuation, '$.batch.versions[*].artifacts[*].key') AS has_key, jsonb_array_length(continuation #> '{batch,versions}') AS versions" + ) + ).resolves.toEqual([{ has_key: false, versions: 1 }]) + }) + + test('revalidates a recovery attempt only while the claim lease is live', async () => { + const fixture = await armBatch(['arm/recovery']) + await expect(fixture.arm()).resolves.toBeDefined() + const attemptId = fixture.armed.batch!.inFlight!.attemptId + + await expect( + tHelper.database.revalidateLifecycleRecoveryAttempt(fixture.identity, attemptId, 60_000) + ).resolves.toBeDefined() + await expireClaim() + await expect( + tHelper.database.revalidateLifecycleRecoveryAttempt(fixture.identity, attemptId, 60_000) + ).resolves.toBeUndefined() + }) + + const SNAPSHOT_AT = '2026-08-15T00:00:00.000Z' + test.each([ + { label: 'exactly at the rule cutoff', archivedAt: '2026-07-16T00:00:00.000Z', arms: false }, + { label: 'just before the rule cutoff', archivedAt: '2026-07-15T23:59:59.999Z', arms: true }, + ])('arms a row archived $label: $arms', async ({ archivedAt, arms }) => { + const fixture = await armBatch(['arm/cutoff'], { archivedAt, snapshotAt: SNAPSHOT_AT }) + + const armed = await fixture.arm() + expect(armed !== undefined).toBe(arms) + if (!arms) { + await expect(fixtures.readState(bucketId, 'continuation, claim_id')).resolves.toEqual([ + { continuation: null, claim_id: null }, + ]) + } + }) + + test.each([ + { + label: 'a sibling archived after the snapshot', + newer: 1, + siblings: ['2026-09-01'], + arms: false, + }, + { + label: 'a sibling archived before the snapshot', + newer: 1, + siblings: ['2026-07-01'], + arms: true, + }, + { label: 'too few newer siblings', newer: 2, siblings: ['2026-07-01'], arms: false }, + { + label: 'more newer siblings than the count', + newer: 2, + siblings: ['2026-07-01', '2026-07-02', '2026-07-03'], + arms: true, + }, + ])( + 'applies newerNoncurrentVersions at arm time with $label', + async ({ newer, siblings, arms }) => { + const fixture = await armBatch(['arm/newer'], { + newerNoncurrentVersions: newer, + snapshotAt: SNAPSHOT_AT, + }) + await insertArchivedObjects( + siblings.map((archivedAt) => ({ name: 'arm/newer', archivedAt })) + ) + + expect((await fixture.arm()) !== undefined).toBe(arms) + } + ) + + test('does not count a newer sibling in another bucket at arm time', async () => { + const fixture = await armBatch(['arm/newer-scope'], { + newerNoncurrentVersions: 1, + snapshotAt: SNAPSHOT_AT, + }) + await insertOtherBucketSibling('arm/newer-scope', '2026-07-01T00:00:00.000Z') + + await expect(fixture.arm()).resolves.toBeUndefined() + }) + + const rowExists = async ({ name, version }: { name: string; version: string }) => + ( + await tHelper.database.connection.query( + 'SELECT 1 FROM storage.objects WHERE bucket_id = $1 AND name = $2 AND version = $3', + [bucketId, name, version] + ) + ).rowCount === 1 + + async function holdWriter(sql: string, values: unknown[], setup?: string) { + const writer = new Client({ connectionString: getConfig().databaseURL }) + await writer.connect() + await writer.query('BEGIN') + const pid = (await writer.query('SELECT pg_backend_pid() AS pid')).rows[0].pid + if (setup) await writer.query(setup) + await writer.query(sql, values) + return { + writer, + pid, + async finish(action: 'COMMIT' | 'ROLLBACK') { + await writer.query(action) + await writer.end() + }, + blocked: () => + expect + .poll( + async () => + ( + await tHelper.database.connection.query( + 'SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE $1 = ANY(pg_blocking_pids(pid))) AS blocked', + [pid] + ) + ).rows[0].blocked + ) + .toBe(true), + } + } + + test('keeps a policy change made during an armed batch due after the stale run completes', async () => { + const fixture = await armBatch(['policy/armed']) + await expect(fixture.arm()).resolves.toBeDefined() + await tHelper.database.putLifecycleConfiguration(bucketId, configuration(2)) + + await expect( + tHelper.database.completeLifecycleRun( + fixture.identity, + { generation: fixture.generation }, + new Date(Date.now() + 86_400_000).toISOString() + ) + ).resolves.toBe(true) + await expect(dueNow()).resolves.toBe(true) + }) + + test('rejects a commit for a stale attempt', async () => { + const fixture = await armBatch(['commit/stale-attempt']) + await expect(fixture.arm()).resolves.toBeDefined() + const continuation = fixture.recorded() + const attemptId = randomUUID() + continuation.batch!.inFlight!.attemptId = attemptId + await expect( + tHelper.database.commitLifecycleAttempt({ + ...fixture.identity, + attemptId, + continuation, + }) + ).resolves.toBeUndefined() + await expect( + fixtures.readState(bucketId, "continuation #>> '{batch,inFlight,attemptId}' AS attempt") + ).resolves.toEqual([{ attempt: fixture.armed.batch!.inFlight!.attemptId }]) + }) + + test('abandons pending artifacts only for the claim holder and advances the cursor', async () => { + const fixture = await armBatch(['abandon/persist']) + await expect(fixture.arm()).resolves.toBeDefined() + const keys = fixture.armed.batch!.versions[0].artifacts.map((artifact) => artifact.key) + const failed = recordLifecycleArtifactOutcomes( + fixture.armed, + new Map([ + [keys[0], { outcome: 'FAILED' as const, error: 'denied' }], + [keys[1], { outcome: 'DELETED' as const }], + ]) + ) + const commit = (claimId: string, abandonPending = false) => + tHelper.database.commitLifecycleAttempt({ + ...fixture.identity, + claimId, + attemptId: fixture.armed.batch!.inFlight!.attemptId, + continuation: failed, + abandonPending, + }) + const abandoned = async () => + ( + await tHelper.database.connection.query( + 'SELECT key, name, version FROM storage.lifecycle_abandoned_artifacts WHERE bucket_id = $1', + [bucketId] + ) + ).rows + + const retained = await commit(fixture.identity.claimId) + expect(retained?.continuation?.batch?.versions).toHaveLength(1) + await expect(abandoned()).resolves.toEqual([]) + + await expect(commit(randomUUID(), true)).resolves.toBeUndefined() + expect( + (await fixtures.readState(bucketId, 'continuation'))[0].continuation.batch + ).toBeDefined() + await expect(abandoned()).resolves.toEqual([]) + + const result = await commit(fixture.identity.claimId, true) + expect(result?.continuation?.batch).toBeUndefined() + expect(result?.continuation).toMatchObject({ + cursor: fixture.staged.pageEnd, + counters: { batchesCompleted: 1 }, + }) + await expect(abandoned()).resolves.toEqual([ + { key: keys[0], name: 'abandon/persist', version: fixture.rows[0].version }, + ]) + }) + + const mutations = { + renamed: () => ['UPDATE storage.objects SET name = name || $2 WHERE id = $1', '-moved'], + reversioned: () => ['UPDATE storage.objects SET version = $2 WHERE id = $1', randomUUID()], + replaced: () => [ + `WITH old AS (DELETE FROM storage.objects WHERE id = $1 RETURNING *) + INSERT INTO storage.objects (id, bucket_id, name, version, archived_at, is_versioned, metadata) + SELECT $2, bucket_id, name, version, archived_at, is_versioned, metadata FROM old`, + randomUUID(), + ], + } as const + + async function mutateRow(kind: keyof typeof mutations, id: string) { + const [sql, param] = mutations[kind]() + await withServiceOperation('storage.lifecycle.test_fixture', (transaction) => + transaction.query(sql, [id, param]) + ) + return kind === 'replaced' ? param : id + } + + test.each(Object.keys(mutations) as Array)( + 'does not arm a row that was %s after staging', + async (kind) => { + const fixture = await armBatch(['identity/pre-arm']) + await mutateRow(kind, fixture.rows[0].id) + + await expect(fixture.arm()).resolves.toBeUndefined() + } + ) + + test('rejects a commit whose version or artifact set differs from the armed batch', async () => { + const fixture = await armBatch(['commit/a', 'commit/b']) + await expect(fixture.arm()).resolves.toBeDefined() + const recorded = fixture.recorded() + + const dropped = { + ...recorded, + batch: { ...recorded.batch!, versions: recorded.batch!.versions.slice(0, 1) }, + } + const withoutInfo = { + ...recorded, + batch: { + ...recorded.batch!, + versions: recorded.batch!.versions.map((version, index) => + index === 0 ? { ...version, artifacts: version.artifacts.slice(0, 1) } : version + ), + }, + } + const retargeted = (changes: Record) => ({ + ...recorded, + batch: { + ...recorded.batch!, + versions: recorded.batch!.versions.map((version, index) => + index === 0 ? { ...version, ...changes } : version + ), + }, + }) + for (const proposed of [ + dropped, + withoutInfo, + { ...recorded, batch: { ...recorded.batch!, keySeparator: '-$v-' as const } }, + retargeted({ id: randomUUID() }), + retargeted({ name: 'commit/other' }), + retargeted({ version: randomUUID() }), + ]) { + await expect(fixture.commit(proposed)).rejects.toMatchObject({ + code: 'InvalidRequest', + message: expect.stringContaining('artifact set changed'), + }) + } + await expect(fixture.commit()).resolves.toBeDefined() + }) + + test.each(['name', 'version', 'bucket_id'] as const)( + 'does not arm a row whose %s changes while arm waits for its row lock', + async (column) => { + const fixture = await armBatch(['lock/changed']) + const replacement = column === 'bucket_id' ? await fixtures.createBucket() : randomUUID() + const held = await holdWriter( + `UPDATE storage.objects SET ${column} = $4 WHERE bucket_id = $1 AND name = $2 AND version = $3`, + [bucketId, fixture.rows[0].name, fixture.rows[0].version, replacement] + ) + try { + const arming = fixture.arm() + await held.blocked() + await held.finish('COMMIT') + await expect(arming).resolves.toBeUndefined() + const rows = await tHelper.database.connection.query( + `SELECT 1 FROM storage.objects WHERE ${column} = $1`, + [replacement] + ) + expect(rows.rowCount).toBe(1) + } finally { + await held.writer.end().catch(() => undefined) + } + } + ) + + test.each([ + { isolation: 'READ COMMITTED', code: undefined }, + { isolation: 'REPEATABLE READ', code: '40001' }, + { isolation: 'SERIALIZABLE', code: '40001' }, + ])('a $isolation promoter racing arm finds the row deleted', async ({ isolation, code }) => { + const fixture = await armBatch(['arm/promoter']) + const promoter = new Client({ connectionString: getConfig().databaseURL }) + await promoter.connect() + try { + await promoter.query(`BEGIN ISOLATION LEVEL ${isolation}`) + await promoter.query('SELECT 1') + await expect(fixture.arm()).resolves.toBeDefined() + const promotion = promoter.query( + 'UPDATE storage.objects SET archived_at = NULL WHERE id = $1', + [fixture.rows[0].id] + ) + if (code) await expect(promotion).rejects.toMatchObject({ code }) + else await expect(promotion).resolves.toMatchObject({ rowCount: 0 }) + } finally { + await promoter.query('ROLLBACK').catch(() => undefined) + await promoter.end() + } + expect(await rowExists(fixture.rows[0])).toBe(false) + }) + + test.each([ + { holder: 'object advisory lock' }, + { holder: 'bucket row lock' }, + { holder: 'object row lock' }, + ])( + 'fails arm with LockTimeout after about 5s behind a $holder', + async ({ holder }) => { + const fixture = await armBatch(['lock/a']) + const release = Promise.withResolvers() + let holding: Promise | undefined + let writer: Awaited> | undefined + if (holder === 'object advisory lock') { + const acquired = Promise.withResolvers() + holding = tHelper.database.withTransaction(async (db) => { + await db.waitObjectLocks([{ bucketId, objectName: fixture.rows[0].name }]) + acquired.resolve() + await release.promise + }) + await acquired.promise + } else if (holder === 'bucket row lock') { + writer = await holdWriter('SELECT 1 FROM storage.buckets WHERE id = $1 FOR UPDATE', [ + bucketId, + ]) + } else { + writer = await holdWriter( + 'SELECT 1 FROM storage.objects WHERE bucket_id = $1 AND name = $2 FOR UPDATE', + [bucketId, fixture.rows[0].name] + ) + } + const startedAt = Date.now() + try { + await expect(fixture.arm()).rejects.toMatchObject({ + code: 'LockTimeout', + httpStatusCode: 503, + }) + expect(Date.now() - startedAt).toBeGreaterThan(4_500) + } finally { + release.resolve() + await holding + await writer?.finish('ROLLBACK') + } + }, + 30_000 + ) + + test('arm waits for a writer holding the object advisory lock, then proceeds', async () => { + const fixture = await armBatch(['lock/wait']) + const acquired = Promise.withResolvers() + const release = Promise.withResolvers() + const holding = tHelper.database.withTransaction(async (db) => { + await db.waitObjectLocks([{ bucketId, objectName: fixture.rows[0].name }]) + acquired.resolve() + await release.promise + }) + await acquired.promise + const arming = fixture.arm() + await expect + .poll( + async () => + ( + await tHelper.database.connection.query( + `SELECT count(*)::int AS waiting FROM pg_locks WHERE locktype = 'advisory' AND NOT granted` + ) + ).rows[0].waiting + ) + .toBeGreaterThan(0) + release.resolve() + await holding + await expect(arming).resolves.toBeDefined() + }) + + test.each([ + { + by: 'name', + names: ['lock/order/c', 'lock/order/a', 'lock/order/B'], + versions: undefined, + }, + { + by: 'version', + names: ['lock/order/x', 'lock/order/x', 'lock/order/x'], + versions: ['c', 'a', 'B'], + }, + ])('locks staged rows in $by order regardless of batch order', async ({ names, versions }) => { + const fixture = await armBatch(names, { versions }) + await tHelper.database.connection.query( + `INSERT INTO storage.objects (bucket_id, name, version, is_versioned, metadata) + SELECT $1, 'filler/' || g, gen_random_uuid()::text, true, '{"size":1}' + FROM generate_series(1, 1000) AS g`, + [bucketId] + ) + await tHelper.database.connection.query('ANALYZE storage.objects') + const held = await holdWriter('SELECT 1 FROM storage.objects WHERE id = $1 FOR UPDATE', [ + fixture.rows[1].id, + ]) + try { + const arming = fixture.arm() + await held.blocked() + const free = await tHelper.database.connection.query( + `SELECT name, version FROM storage.objects + WHERE bucket_id = $1 AND archived_at IS NOT NULL + FOR UPDATE SKIP LOCKED`, + [bucketId] + ) + expect(free.rows).toEqual([{ name: names[0], version: fixture.rows[0].version }]) + await held.finish('ROLLBACK') + await expect(arming).resolves.toBeDefined() + } finally { + await held.writer.end().catch(() => undefined) + } + }) + + test('holds a bucket share lock during arming that blocks bucket deletion', async () => { + const fixture = await armBatch(['lock/share']) + const deleter = new Client({ connectionString: getConfig().databaseURL }) + await deleter.connect() + let deleting: Promise | undefined + try { + await tHelper.database.withTransaction(async (db) => { + await expect(db.armLifecycleAttempt(fixture.input)).resolves.toBeDefined() + await deleter.query('BEGIN') + await deleter.query("SELECT set_config('storage.allow_delete_query', 'true', true)") + const pid = (await deleter.query('SELECT pg_backend_pid() AS pid')).rows[0].pid + deleting = deleter + .query('DELETE FROM storage.buckets WHERE id = $1', [bucketId]) + .catch((error: unknown) => error) + await expect + .poll( + async () => + ( + await tHelper.database.connection.query( + 'SELECT cardinality(pg_blocking_pids($1)) > 0 AS blocked', + [pid] + ) + ).rows[0].blocked + ) + .toBe(true) + }) + await deleting + await deleter.query('ROLLBACK') + } finally { + await deleter.end() + } + }) + }) + + test.each([ + { + by: 'name', + names: ['order/a', 'order/B', 'order/c'], + versions: [randomUUID(), randomUUID(), randomUUID()], + }, + { by: 'version', names: ['order/x', 'order/x', 'order/x'], versions: ['a', 'B', 'c'] }, + ])('locks id-batch deletions in the same $by order as arming', async ({ names, versions }) => { + const ids = [ + 'ffffffff-0000-4000-8000-000000000000', + '88888888-0000-4000-8000-000000000000', + '00000000-0000-4000-8000-000000000000', + ] + await tHelper.database.connection.query( + `INSERT INTO storage.objects (id, bucket_id, name, version, is_versioned, metadata, archived_at) + SELECT id::uuid, $1, name, v, true, '{"size":1}', now() - i * interval '1 second' + FROM unnest($2::text[], $3::text[], $4::text[]) WITH ORDINALITY AS t(name, v, id, i) + ORDER BY i DESC`, + [bucketId, names, versions, ids] + ) + // Enough rows for the planner to scan by id instead of by name. + await tHelper.database.connection.query( + `INSERT INTO storage.objects (bucket_id, name, version, is_versioned, metadata) + SELECT $1, 'filler/' || g, gen_random_uuid()::text, true, '{"size":1}' + FROM generate_series(1, 20000) AS g`, + [bucketId] + ) + await tHelper.database.connection.query('ANALYZE storage.objects') + const holder = new Client({ connectionString: getConfig().databaseURL }) + const observer = new Client({ connectionString: getConfig().databaseURL }) + await holder.connect() + await observer.connect() + let deleting: Promise | undefined + try { + await holder.query('BEGIN') + await holder.query('SELECT 1 FROM storage.objects WHERE id = $1 FOR UPDATE', [ids[1]]) + deleting = tHelper.database.withTransaction((db) => db.deleteObjects(bucketId, ids, 'id')) + await expect + .poll( + async () => + ( + await observer.query( + 'SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE $1 = ANY(pg_blocking_pids(pid))) AS blocked', + [(await holder.query('SELECT pg_backend_pid() AS pid')).rows[0].pid] + ) + ).rows[0].blocked + ) + .toBe(true) + const free = ( + await observer.query( + 'SELECT id FROM storage.objects WHERE bucket_id = $1 FOR UPDATE SKIP LOCKED', + [bucketId] + ) + ).rows.map((row) => row.id) + expect(free).toEqual(expect.arrayContaining([ids[0], ids[2]])) + } finally { + await holder.query('ROLLBACK') + await deleting + await holder.end() + await observer.end() + } + }) + + describe('abandoned artifacts', () => { + const seed = (bucket: string, rows: Array<[string, string]>) => + tHelper.database.connection.query( + `INSERT INTO storage.lifecycle_abandoned_artifacts (bucket_id, key, name, attempted_at) + SELECT $1, r.key, r.key, r.attempted_at::timestamptz + FROM unnest($2::text[], $3::text[]) AS r(key, attempted_at)`, + [bucket, rows.map(([key]) => key), rows.map(([, attemptedAt]) => attemptedAt)] + ) + + test('lists tenant-wide by attempt age and resolves bucket and key pairs', async () => { + await seed(bucketId, [ + ['a', '2026-01-01T00:00:01Z'], + ['b', '2026-01-01T00:00:03Z'], + ['c', '2026-01-01T00:00:05Z'], + ]) + const other = await fixtures.createBucket() + await seed(other, [ + ['a', '2026-01-01T00:00:02Z'], + ['z', '2026-01-01T00:00:04Z'], + ]) + const list = (attemptedBefore: string, limit = 10) => + tHelper.database.listLifecycleAbandonedArtifacts(attemptedBefore, limit) + + await expect(list('2026-01-01T00:00:04Z')).resolves.toEqual([ + { bucketId, key: 'a', live: false }, + { bucketId: other, key: 'a', live: false }, + { bucketId, key: 'b', live: false }, + ]) + await expect(list('2026-01-02T00:00:00Z', 2)).resolves.toEqual([ + { bucketId, key: 'a', live: false }, + { bucketId: other, key: 'a', live: false }, + ]) + + await tHelper.database.resolveLifecycleAbandonedArtifacts( + [{ bucketId: other, key: 'a' }], + [{ bucketId, key: 'a' }] + ) + + await expect(list('2026-01-02T00:00:00Z')).resolves.toEqual([ + { bucketId, key: 'b', live: false }, + { bucketId: other, key: 'z', live: false }, + { bucketId, key: 'c', live: false }, + ]) + await expect(list(new Date(Date.now() + 60_000).toISOString())).resolves.toEqual([ + { bucketId, key: 'b', live: false }, + { bucketId: other, key: 'z', live: false }, + { bucketId, key: 'c', live: false }, + { bucketId, key: 'a', live: false }, + ]) + }) + + describe('live object backstop', () => { + const future = () => new Date(Date.now() + 60_000).toISOString() + const remaining = async () => + ( + await tHelper.database.connection.query( + 'SELECT key FROM storage.lifecycle_abandoned_artifacts WHERE bucket_id = $1 ORDER BY key', + [bucketId] + ) + ).rows.map((row) => row.key) + const record = (key: string, name: string, version: string | null) => + tHelper.database.connection.query( + `INSERT INTO storage.lifecycle_abandoned_artifacts (bucket_id, key, name, version) + VALUES ($1, $2, $3, $4)`, + [bucketId, key, name, version] + ) + const liveObject = (name: string, version: string | null) => + tHelper.database.connection.query( + `INSERT INTO storage.objects (bucket_id, name, version, is_versioned, archived_at, metadata) + VALUES ($1, $2, $3::text, $3::text IS NOT NULL, CASE WHEN $3::text IS NULL THEN NULL ELSE now() END, '{"size":1}')`, + [bucketId, name, version] + ) + + test('drops records owned by a live object with the same name and version', async () => { + const owned = randomUUID() + const orphan = randomUUID() + const reversioned = randomUUID() + await liveObject('guard/a', owned) + await liveObject('guard/c', randomUUID()) + await record(`guard/a/${owned}`, 'guard/a', owned) + await record(`guard/a/${owned}.info`, 'guard/a', owned) + await record(`guard/b/${orphan}`, 'guard/b', orphan) + await record(`guard/c/${reversioned}`, 'guard/c', reversioned) + + await expect( + tHelper.database.listLifecycleAbandonedArtifacts(future(), 10) + ).resolves.toEqual([ + { bucketId, key: `guard/a/${owned}`, live: true }, + { bucketId, key: `guard/a/${owned}.info`, live: true }, + { bucketId, key: `guard/b/${orphan}`, live: false }, + { bucketId, key: `guard/c/${reversioned}`, live: false }, + ]) + await expect(remaining()).resolves.toEqual([`guard/b/${orphan}`, `guard/c/${reversioned}`]) + }) + + test('matches a null-version record only to a live null-version row', async () => { + const other = randomUUID() + await liveObject('guard/legacy', null) + await liveObject('guard/other', other) + await record('guard/legacy', 'guard/legacy', null) + await record('guard/other', 'guard/other', null) + + await expect( + tHelper.database.listLifecycleAbandonedArtifacts(future(), 10) + ).resolves.toEqual([ + { bucketId, key: 'guard/legacy', live: true }, + { bucketId, key: 'guard/other', live: false }, + ]) + await expect(remaining()).resolves.toEqual(['guard/other']) + }) + }) + + test('survives bucket deletion', async () => { + const other = await fixtures.createBucket() + await seed(other, [['a', '2026-01-01T00:00:00Z']]) + + await withDeleteEnabled(tHelper.database.connection, (transaction) => + transaction.query('DELETE FROM storage.buckets WHERE id = $1', [other]) + ) + + await expect( + tHelper.database.listLifecycleAbandonedArtifacts('2026-01-02T00:00:00Z', 10) + ).resolves.toEqual([{ bucketId: other, key: 'a', live: false }]) + }) + + test('reports whether any abandoned artifact exists', async () => { + const hasAbandoned = async () => + (await tHelper.database.findLifecycleDispatchState({ runBudgetMs: 20_000 })) + .hasAbandonedArtifacts + + await expect(hasAbandoned()).resolves.toBe(false) + await seed(bucketId, [['a', '2026-01-01T00:00:00Z']]) + await expect(hasAbandoned()).resolves.toBe(true) + await tHelper.database.resolveLifecycleAbandonedArtifacts([{ bucketId, key: 'a' }], []) + await expect(hasAbandoned()).resolves.toBe(false) + }) + }) +}) diff --git a/src/test/noncurrent-lifecycle-schema.test.ts b/src/test/noncurrent-lifecycle-schema.test.ts index 873c43830..f4be8805d 100644 --- a/src/test/noncurrent-lifecycle-schema.test.ts +++ b/src/test/noncurrent-lifecycle-schema.test.ts @@ -141,7 +141,7 @@ describe('noncurrent lifecycle schema foundation', () => { newerNoncurrentVersions: 1, }, ], - pageSize: 500, + pageSize: 480, }) const plan = await transaction.query<{ 'QUERY PLAN': unknown }>({ text: `EXPLAIN (FORMAT JSON) ${evaluation.text}`, diff --git a/src/test/utils/lifecycle-continuation.ts b/src/test/utils/lifecycle-continuation.ts index c53b621b0..6e0e86404 100644 --- a/src/test/utils/lifecycle-continuation.ts +++ b/src/test/utils/lifecycle-continuation.ts @@ -5,7 +5,7 @@ import { freezeLifecycleBatchVersion, stageLifecycleBatch, } from '@storage/lifecycle/continuation' -import type { LifecycleCandidateIdentity, NoncurrentLifecycleShardCursor } from '@storage/schemas' +import type { LifecycleCandidateIdentity, NoncurrentLifecycleCursor } from '@storage/schemas' export function newContinuation( generation: string, @@ -15,9 +15,6 @@ export function newContinuation( runId: randomUUID(), generation, snapshotAt: new Date().toISOString(), - trigger: 'scheduled', - mode: 'DELETE', - topology: { scanKind: 'NONCURRENT', epoch: '1', shardId: 0, shardCount: 1 }, ...overrides, }) } @@ -25,7 +22,7 @@ export function newContinuation( export function stagedContinuation( generation: string, versions: LifecycleCandidateIdentity[], - pageEnd: NoncurrentLifecycleShardCursor, + pageEnd: NoncurrentLifecycleCursor, options: { snapshotAt?: string } = {} ) { return stageLifecycleBatch( @@ -41,12 +38,11 @@ export function stagedContinuation( export function armedContinuation( generation: string, versions: LifecycleCandidateIdentity[], - pageEnd: NoncurrentLifecycleShardCursor, + pageEnd: NoncurrentLifecycleCursor, options: { snapshotAt?: string; startedAt?: string } = {} ) { return armLifecycleAttempt(stagedContinuation(generation, versions, pageEnd, options), { attemptId: randomUUID(), - authorizedGeneration: generation, startedAt: options.startedAt ?? new Date().toISOString(), }) } From dd7a4bce6b7cafb2bab6b34e93937058506f6983 Mon Sep 17 00:00:00 2001 From: Ferhat Elmas Date: Thu, 29 Oct 2026 21:41:39 +0100 Subject: [PATCH 2/9] fix: require generation in run completion Signed-off-by: Ferhat Elmas --- src/storage/database/adapter.ts | 3 +- src/storage/database/pg.ts | 16 ++++-- src/storage/schemas/lifecycle.ts | 4 ++ .../noncurrent-lifecycle-database.test.ts | 57 ++++++++++++++++--- 4 files changed, 68 insertions(+), 12 deletions(-) diff --git a/src/storage/database/adapter.ts b/src/storage/database/adapter.ts index 3ae7818bf..0554b7bde 100644 --- a/src/storage/database/adapter.ts +++ b/src/storage/database/adapter.ts @@ -13,6 +13,7 @@ import { LifecycleClaimIdentity, LifecycleClaimInput, LifecycleCommitAttemptInput, + LifecycleCompleteRunInput, LifecycleDispatchState, LifecycleEvaluationPage, LifecycleReleaseClaimInput, @@ -255,7 +256,7 @@ export interface Database { releaseLifecycleClaim(input: LifecycleReleaseClaimInput): Promise completeLifecycleRun( - input: LifecycleClaimIdentity, + input: LifecycleCompleteRunInput, lastResult: Record, nextRunAt: string ): Promise diff --git a/src/storage/database/pg.ts b/src/storage/database/pg.ts index 32dc16eaf..07f094523 100644 --- a/src/storage/database/pg.ts +++ b/src/storage/database/pg.ts @@ -46,6 +46,7 @@ import { LifecycleClaimIdentity, LifecycleClaimInput, LifecycleCommitAttemptInput, + LifecycleCompleteRunInput, LifecycleContinuation, LifecycleDispatchState, LifecycleEvaluationPage, @@ -3927,10 +3928,13 @@ export class StoragePgDB implements Database { } async completeLifecycleRun( - input: LifecycleClaimIdentity, + input: LifecycleCompleteRunInput, lastResult: Record, nextRunAt: string ): Promise { + if (typeof input.generation !== 'string' || !input.generation) { + throw ERRORS.InvalidParameter('lifecycle run generation') + } const result = await this.runQuery('CompleteLifecycleRun', (db, signal) => this.query( db, @@ -3939,8 +3943,7 @@ export class StoragePgDB implements Database { UPDATE storage.bucket_lifecycle_states AS state SET continuation = NULL, next_run_at = CASE - WHEN bucket.lifecycle_configuration_generation::text - IS NOT DISTINCT FROM $4::jsonb ->> 'generation' + WHEN bucket.lifecycle_configuration_generation = $5::uuid THEN $3::timestamptz ELSE clock_timestamp() END, @@ -3956,7 +3959,12 @@ export class StoragePgDB implements Database { WHERE ${LIFECYCLE_CLAIM_IDENTITY} AND bucket.id = state.bucket_id `, - values: [...lifecycleClaimIdentityValues(input), nextRunAt, JSON.stringify(lastResult)], + values: [ + ...lifecycleClaimIdentityValues(input), + nextRunAt, + JSON.stringify(lastResult), + input.generation, + ], }, signal ) diff --git a/src/storage/schemas/lifecycle.ts b/src/storage/schemas/lifecycle.ts index b7431f463..bd4d3a108 100644 --- a/src/storage/schemas/lifecycle.ts +++ b/src/storage/schemas/lifecycle.ts @@ -160,6 +160,10 @@ export interface LifecycleClaimIdentity { claimId: string } +export interface LifecycleCompleteRunInput extends LifecycleClaimIdentity { + generation: string +} + export interface LifecycleClaimInput extends LifecycleClaimIdentity { leaseMs: number } diff --git a/src/test/noncurrent-lifecycle-database.test.ts b/src/test/noncurrent-lifecycle-database.test.ts index f67eb3e68..b27e661c1 100644 --- a/src/test/noncurrent-lifecycle-database.test.ts +++ b/src/test/noncurrent-lifecycle-database.test.ts @@ -722,14 +722,14 @@ describe('noncurrent lifecycle database API', () => { }) test('completes a run only for the claim that holds the bucket', async () => { - await putConfiguration() + const generation = await putConfiguration() const claimId = randomUUID() await tHelper.database.claimLifecycleBucket({ bucketId, claimId, leaseMs: 60_000 }) await setState(`continuation = '{"continuationVersion": 1}'::jsonb`) await expect( tHelper.database.completeLifecycleRun( - { bucketId, claimId: randomUUID() }, + { bucketId, claimId: randomUUID(), generation }, {}, new Date().toISOString() ) @@ -739,11 +739,54 @@ describe('noncurrent lifecycle database API', () => { ]) }) + test.each(['omitted', 'different'] as const)( + 'uses the run generation even when the result generation is %s', + async (resultGeneration) => { + const generation = await putConfiguration() + const claimId = randomUUID() + await expect( + tHelper.database.claimLifecycleBucket({ bucketId, claimId, leaseMs: 60_000 }) + ).resolves.toBeDefined() + const lastResult = resultGeneration === 'omitted' ? {} : { generation: randomUUID() } + const nextRunAt = '2099-01-01T00:00:00.000Z' + + await expect( + tHelper.database.completeLifecycleRun( + { bucketId, claimId, generation }, + lastResult, + nextRunAt + ) + ).resolves.toBe(true) + await expect(fixtures.readState(bucketId, 'next_run_at, last_result')).resolves.toEqual([ + { next_run_at: new Date(nextRunAt), last_result: lastResult }, + ]) + await expect(claim()).resolves.toBeUndefined() + } + ) + + test('rejects completion without a run generation before changing state', async () => { + await putConfiguration() + const claimId = randomUUID() + await expect( + tHelper.database.claimLifecycleBucket({ bucketId, claimId, leaseMs: 60_000 }) + ).resolves.toBeDefined() + await expect( + tHelper.database.completeLifecycleRun( + { bucketId, claimId, generation: undefined as unknown as string }, + {}, + '2099-01-01T00:00:00.000Z' + ) + ).rejects.toMatchObject({ code: 'InvalidParameter' }) + await expect(fixtures.readState(bucketId, 'claim_id, last_result')).resolves.toEqual([ + { claim_id: claimId, last_result: null }, + ]) + }) + test('keeps a policy written under a racing claim due after the claimed run completes', async () => { await putConfiguration() await setState(`next_run_at = '2000-01-01'`) const stale = (await tHelper.database.findLifecycleBucket(bucketId)) - .lifecycle_configuration_generation + .lifecycle_configuration_generation! const writer = new Client({ connectionString: getConfig().databaseURL }) await writer.connect() try { @@ -773,8 +816,8 @@ describe('noncurrent lifecycle database API', () => { await expect( tHelper.database.completeLifecycleRun( - identity, - { generation: stale }, + { ...identity, generation: stale }, + {}, new Date(Date.now() + 86_400_000).toISOString() ) ).resolves.toBe(true) @@ -1358,8 +1401,8 @@ describe('noncurrent lifecycle database API', () => { await expect( tHelper.database.completeLifecycleRun( - fixture.identity, - { generation: fixture.generation }, + { ...fixture.identity, generation: fixture.generation }, + {}, new Date(Date.now() + 86_400_000).toISOString() ) ).resolves.toBe(true) From 38d856424f98b489e6c2899465dba82b172dbf40 Mon Sep 17 00:00:00 2001 From: Ferhat Elmas Date: Thu, 29 Oct 2026 21:50:46 +0100 Subject: [PATCH 3/9] fix: keep active journals on completion and release Signed-off-by: Ferhat Elmas --- src/storage/database/pg.ts | 15 ++ src/storage/schemas/lifecycle.ts | 3 +- .../noncurrent-lifecycle-database.test.ts | 226 ++++++++++++++---- 3 files changed, 194 insertions(+), 50 deletions(-) diff --git a/src/storage/database/pg.ts b/src/storage/database/pg.ts index 07f094523..274f6a39e 100644 --- a/src/storage/database/pg.ts +++ b/src/storage/database/pg.ts @@ -3911,6 +3911,14 @@ export class StoragePgDB implements Database { last_error = COALESCE($5::jsonb, last_error), updated_at = clock_timestamp() WHERE ${LIFECYCLE_CLAIM_IDENTITY} + AND ( + NOT $6::boolean + OR state.continuation IS NULL + OR ( + state.continuation @> '{"continuationVersion":1}'::jsonb + AND state.continuation #> '{batch,inFlight}' IS NULL + ) + ) `, values: [ ...lifecycleClaimIdentityValues(input), @@ -3958,6 +3966,13 @@ export class StoragePgDB implements Database { FROM storage.buckets AS bucket WHERE ${LIFECYCLE_CLAIM_IDENTITY} AND bucket.id = state.bucket_id + AND ( + state.continuation IS NULL + OR ( + state.continuation @> '{"continuationVersion":1}'::jsonb + AND state.continuation #> '{batch,inFlight}' IS NULL + ) + ) `, values: [ ...lifecycleClaimIdentityValues(input), diff --git a/src/storage/schemas/lifecycle.ts b/src/storage/schemas/lifecycle.ts index bd4d3a108..f1f42bd00 100644 --- a/src/storage/schemas/lifecycle.ts +++ b/src/storage/schemas/lifecycle.ts @@ -180,7 +180,8 @@ export interface LifecycleCommitAttemptInput extends LifecycleClaimIdentity { } export interface LifecycleReleaseClaimInput extends LifecycleClaimIdentity { - // Omit after an ambiguous write to preserve the durable journal. + // Omit after an ambiguous write. + // Omit when an armed or unknown journal version must be preserved for recovery. continuation?: LifecycleContinuation | null nextRunAt: string | null // Cap a retry only when preserving a persisted journal in flight. diff --git a/src/test/noncurrent-lifecycle-database.test.ts b/src/test/noncurrent-lifecycle-database.test.ts index b27e661c1..e6c33ca15 100644 --- a/src/test/noncurrent-lifecycle-database.test.ts +++ b/src/test/noncurrent-lifecycle-database.test.ts @@ -1,4 +1,5 @@ import { randomUUID } from 'node:crypto' +import { Readable } from 'node:stream' import { buildEvaluateNoncurrentLifecyclePageStatement, mapLifecycleEvaluationPage, @@ -1008,54 +1009,49 @@ describe('noncurrent lifecycle database API', () => { }) test.each([ - { state: 'armed', explicit: false, capped: true }, - { state: 'armed', explicit: true, capped: false }, - { state: 'unreadable armed', explicit: false, capped: true }, - { state: 'unreadable idle', explicit: false, capped: false }, - { state: 'empty', explicit: false, capped: false }, - ])( - 'caps only preserved journals on release: $state, explicit=$explicit', - async ({ state, explicit, capped }) => { - const fixture = await armBatch(['retry-cap']) - await expect(fixture.arm()).resolves.toBeDefined() - const continuation = - state === 'armed' - ? encodeLifecycleContinuation(fixture.armed) - : state === 'empty' - ? null - : { - continuationVersion: 999, - ...(state === 'unreadable armed' && { - batch: { inFlight: { attemptId: randomUUID() } }, - }), - } - await tHelper.database.connection.query( - `UPDATE storage.bucket_lifecycle_states + { state: 'armed', capped: true }, + { state: 'unreadable armed', capped: true }, + { state: 'unreadable idle', capped: false }, + { state: 'empty', capped: false }, + ])('caps only preserved journals on release: $state', async ({ state, capped }) => { + const fixture = await armBatch(['retry-cap']) + await expect(fixture.arm()).resolves.toBeDefined() + const continuation = + state === 'armed' + ? encodeLifecycleContinuation(fixture.armed) + : state === 'empty' + ? null + : { + continuationVersion: 999, + ...(state === 'unreadable armed' && { + batch: { inFlight: { attemptId: randomUUID() } }, + }), + } + await tHelper.database.connection.query( + `UPDATE storage.bucket_lifecycle_states SET continuation = $2::jsonb, claim_until = '2000-01-01' WHERE bucket_id = $1`, - [bucketId, continuation === null ? null : JSON.stringify(continuation)] - ) - const nextRunAt = '2099-01-01T01:00:00.000Z' - const maxInFlightRetryAt = '2099-01-01T00:15:00.000Z' - await expect( - tHelper.database.releaseLifecycleClaim({ - ...fixture.identity, - ...(explicit && { continuation: fixture.armed }), - nextRunAt, - maxInFlightRetryAt, - error: { message: 'lost acknowledgement' }, - }) - ).resolves.toBe(true) - await expect( - fixtures.readState(bucketId, 'continuation, claim_id, next_run_at') - ).resolves.toEqual([ - { - continuation, - claim_id: null, - next_run_at: new Date(capped ? maxInFlightRetryAt : nextRunAt), - }, - ]) - } - ) + [bucketId, continuation === null ? null : JSON.stringify(continuation)] + ) + const nextRunAt = '2099-01-01T01:00:00.000Z' + const maxInFlightRetryAt = '2099-01-01T00:15:00.000Z' + await expect( + tHelper.database.releaseLifecycleClaim({ + ...fixture.identity, + nextRunAt, + maxInFlightRetryAt, + error: { message: 'lost acknowledgement' }, + }) + ).resolves.toBe(true) + await expect( + fixtures.readState(bucketId, 'continuation, claim_id, next_run_at') + ).resolves.toEqual([ + { + continuation, + claim_id: null, + next_run_at: new Date(capped ? maxInFlightRetryAt : nextRunAt), + }, + ]) + }) test.each([ { continuation: 'idle', error: false, expected: 0 }, @@ -1065,7 +1061,9 @@ describe('noncurrent lifecycle database API', () => { 'release with $continuation journal, error=$error leaves failure_count $expected', async ({ continuation, error, expected }) => { const fixture = await armBatch(['failure-count']) - await expect(fixture.arm()).resolves.toBeDefined() + const armed = await fixture.arm() + expect(armed).toBeDefined() + if (continuation === 'idle') await expect(fixture.commit()).resolves.toBeDefined() await tHelper.database.connection.query( 'UPDATE storage.bucket_lifecycle_states SET failure_count = 7 WHERE bucket_id = $1', [bucketId] @@ -1073,7 +1071,7 @@ describe('noncurrent lifecycle database API', () => { await expect( tHelper.database.releaseLifecycleClaim({ ...fixture.identity, - continuation: continuation === 'armed' ? fixture.armed : fixture.staged, + ...(continuation === 'idle' && { continuation: fixture.staged }), nextRunAt: '2099-01-01T01:00:00.000Z', ...(error && { error: { message: 'boom' } }), }) @@ -1094,6 +1092,9 @@ describe('noncurrent lifecycle database API', () => { async ({ journal, outcome }) => { const fixture = await armBatch(['release/unscheduled']) await expect(fixture.arm()).resolves.toBeDefined() + if (journal === 'staged' || journal === 'empty') { + await expect(fixture.commit()).resolves.toBeDefined() + } const continuation = { preserved: undefined, armed: fixture.armed, @@ -1394,10 +1395,137 @@ describe('noncurrent lifecycle database API', () => { } } + test('preserves an armed journal until its backend artifacts are resolved', async () => { + const fixture = await armBatch(['completion/pending']) + const row = fixture.rows[0] + const { adapter, storage } = tHelper + const root = storage.location.getRootLocation() + const key = storage.location.getKeyLocation({ + tenantId: getConfig().tenantId, + bucketId, + objectName: row.name, + }) + await adapter.uploadObject( + root, + key, + row.version, + Readable.from(['pending bytes']), + 'text/plain', + 'no-cache' + ) + + const armed = await fixture.arm() + expect(armed?.continuation?.batch?.inFlight).toBeDefined() + expect(await rowExists(row)).toBe(false) + const before = await fixtures.readState(bucketId, 'continuation, claim_id, next_run_at') + const nextRunAt = '2099-01-01T00:00:00.000Z' + await expect( + tHelper.database.completeLifecycleRun( + { ...fixture.identity, generation: fixture.generation }, + {}, + nextRunAt + ) + ).resolves.toBe(false) + const changed = recordLifecycleArtifactOutcomes( + armed!.continuation!, + new Map([ + [armed!.continuation!.batch!.versions[0].artifacts[0].key, { outcome: 'DELETED' }], + ]) + ) + for (const continuation of [null, fixture.staged, changed, armed!.continuation!]) { + await expect( + tHelper.database.releaseLifecycleClaim({ ...fixture.identity, continuation, nextRunAt }) + ).resolves.toBe(false) + } + await expect( + fixtures.readState(bucketId, 'continuation, claim_id, next_run_at') + ).resolves.toEqual(before) + await expect(adapter.headObject(root, key, row.version)).resolves.toMatchObject({ size: 13 }) + + await adapter.deleteObject(root, key, row.version) + const [data, info] = armed!.continuation!.batch!.versions[0].artifacts + const recorded = recordLifecycleArtifactOutcomes( + armed!.continuation!, + new Map([ + [data.key, { outcome: 'DELETED' as const }], + [info.key, { outcome: 'ABSENT' as const }], + ]) + ) + await expect(fixture.commit(recorded)).resolves.toBeDefined() + await expect( + tHelper.database.completeLifecycleRun( + { ...fixture.identity, generation: fixture.generation }, + {}, + nextRunAt + ) + ).resolves.toBe(true) + await expect(adapter.headObject(root, key, row.version)).rejects.toThrow() + }) + + test('preserves an unknown journal on completion or explicit release', async () => { + const fixture = await armBatch(['completion/unknown']) + const continuation = { continuationVersion: 999 } + await tHelper.database.connection.query( + 'UPDATE storage.bucket_lifecycle_states SET continuation = $2::jsonb WHERE bucket_id = $1', + [bucketId, JSON.stringify(continuation)] + ) + const nextRunAt = '2099-01-01T00:00:00.000Z' + await expect( + tHelper.database.completeLifecycleRun( + { ...fixture.identity, generation: fixture.generation }, + {}, + nextRunAt + ) + ).resolves.toBe(false) + await expect( + tHelper.database.releaseLifecycleClaim({ + ...fixture.identity, + continuation: null, + nextRunAt, + }) + ).resolves.toBe(false) + await expect( + tHelper.database.releaseLifecycleClaim({ ...fixture.identity, nextRunAt }) + ).resolves.toBe(true) + await expect(fixtures.readState(bucketId, 'continuation, claim_id')).resolves.toEqual([ + { continuation, claim_id: null }, + ]) + }) + + test('preserves an in-flight journal with newer fields when releasing the claim', async () => { + const fixture = await armBatch(['release/future-fields']) + await expect(fixture.arm()).resolves.toBeDefined() + const nextRunAt = '2099-01-01T01:00:00.000Z' + await tHelper.database.connection.query( + `UPDATE storage.bucket_lifecycle_states + SET continuation = jsonb_set(continuation, '{futureField}', '{"enabled":true}') + WHERE bucket_id = $1`, + [bucketId] + ) + const stored = (await fixtures.readState(bucketId, 'continuation'))[0].continuation + expect(stored.futureField).toEqual({ enabled: true }) + const decoded = await tHelper.database.revalidateLifecycleClaim(fixture.identity) + expect(decoded?.continuation).not.toHaveProperty('futureField') + + await expect( + tHelper.database.releaseLifecycleClaim({ ...fixture.identity, nextRunAt }) + ).resolves.toBe(true) + await expect( + fixtures.readState(bucketId, 'continuation, claim_id, next_run_at') + ).resolves.toEqual([ + { + continuation: stored, + claim_id: null, + next_run_at: new Date(nextRunAt), + }, + ]) + }) + test('keeps a policy change made during an armed batch due after the stale run completes', async () => { const fixture = await armBatch(['policy/armed']) await expect(fixture.arm()).resolves.toBeDefined() await tHelper.database.putLifecycleConfiguration(bucketId, configuration(2)) + await expect(fixture.commit()).resolves.toBeDefined() await expect( tHelper.database.completeLifecycleRun( From f5d1affd1fcec82d31d974cb6a03dff6daed2ef0 Mon Sep 17 00:00:00 2001 From: Ferhat Elmas Date: Fri, 30 Oct 2026 00:32:39 +0100 Subject: [PATCH 4/9] fix: reject armed journals when releasing claims Signed-off-by: Ferhat Elmas --- src/storage/database/pg.ts | 5 ++++- .../noncurrent-lifecycle-database.test.ts | 22 ++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/storage/database/pg.ts b/src/storage/database/pg.ts index 274f6a39e..f380ebec3 100644 --- a/src/storage/database/pg.ts +++ b/src/storage/database/pg.ts @@ -3885,7 +3885,10 @@ export class StoragePgDB implements Database { async releaseLifecycleClaim(input: LifecycleReleaseClaimInput): Promise { const { continuation } = input - if ((continuation === undefined || continuation?.batch?.inFlight) && input.nextRunAt === null) { + if (continuation?.batch?.inFlight) { + throw ERRORS.InvalidParameter('lifecycle armed journal') + } + if (continuation === undefined && input.nextRunAt === null) { throw ERRORS.InvalidParameter('lifecycle recovery schedule') } const result = await this.runQuery('ReleaseLifecycleClaim', (db, signal) => diff --git a/src/test/noncurrent-lifecycle-database.test.ts b/src/test/noncurrent-lifecycle-database.test.ts index e6c33ca15..2be11b2c6 100644 --- a/src/test/noncurrent-lifecycle-database.test.ts +++ b/src/test/noncurrent-lifecycle-database.test.ts @@ -1110,6 +1110,21 @@ describe('noncurrent lifecycle database API', () => { } ) + test('rejects an explicit armed journal that was never armed', async () => { + const fixture = await armBatch(['release/unarmed']) + await expect( + tHelper.database.releaseLifecycleClaim({ + ...fixture.identity, + continuation: fixture.armed, + nextRunAt: '2099-01-01T00:00:00.000Z', + }) + ).rejects.toMatchObject({ code: 'InvalidParameter' }) + await expect(fixtures.readState(bucketId, 'continuation, claim_id')).resolves.toEqual([ + { continuation: null, claim_id: fixture.identity.claimId }, + ]) + expect(await rowExists(fixture.rows[0])).toBe(true) + }) + const expireClaim = () => tHelper.database.connection.query( "UPDATE storage.bucket_lifecycle_states SET claim_until = now() - interval '1 second' WHERE bucket_id = $1", @@ -1432,11 +1447,16 @@ describe('noncurrent lifecycle database API', () => { [armed!.continuation!.batch!.versions[0].artifacts[0].key, { outcome: 'DELETED' }], ]) ) - for (const continuation of [null, fixture.staged, changed, armed!.continuation!]) { + for (const continuation of [null, fixture.staged]) { await expect( tHelper.database.releaseLifecycleClaim({ ...fixture.identity, continuation, nextRunAt }) ).resolves.toBe(false) } + for (const continuation of [changed, armed!.continuation!]) { + await expect( + tHelper.database.releaseLifecycleClaim({ ...fixture.identity, continuation, nextRunAt }) + ).rejects.toMatchObject({ code: 'InvalidParameter' }) + } await expect( fixtures.readState(bucketId, 'continuation, claim_id, next_run_at') ).resolves.toEqual(before) From 9ccdf6aedbca05998ef662229067de6335673072 Mon Sep 17 00:00:00 2001 From: Ferhat Elmas Date: Fri, 30 Oct 2026 00:59:49 +0100 Subject: [PATCH 5/9] fix: preserve persisted state when committing outcomes Signed-off-by: Ferhat Elmas --- src/storage/database/pg.ts | 11 +++++++++-- src/test/noncurrent-lifecycle-database.test.ts | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/storage/database/pg.ts b/src/storage/database/pg.ts index f380ebec3..f799e45e0 100644 --- a/src/storage/database/pg.ts +++ b/src/storage/database/pg.ts @@ -23,6 +23,7 @@ import { encodeLifecycleContinuation, lifecycleConfigurationsEqual, pendingArtifacts, + recordLifecycleArtifactOutcomes, recordLifecycleMetadataDeletions, skipLifecycleBatch, } from '@storage/lifecycle' @@ -3746,7 +3747,13 @@ export class StoragePgDB implements Database { throw ERRORS.InvalidRequest('Lifecycle attempt artifact set changed before completion') } - const persisted = commitLifecycleBatchResults(continuation, input.abandonPending) + const outcomes = new Map( + continuation.batch!.versions.flatMap(({ artifacts }) => + artifacts.map(({ key, ...outcome }) => [key, outcome] as const) + ) + ) + const merged = recordLifecycleArtifactOutcomes(state.continuation, outcomes) + const persisted = commitLifecycleBatchResults(merged, input.abandonPending) const result = await serviceDatabase.runQuery('CommitLifecycleAttemptState', (db, signal) => serviceDatabase.query( @@ -3772,7 +3779,7 @@ export class StoragePgDB implements Database { ) if (!result.rows[0]) return undefined if (input.abandonPending) { - const pending = pendingArtifacts(continuation) + const pending = pendingArtifacts(merged) if (pending.length > 0) { await serviceDatabase.runQuery('PersistLifecycleAbandonedArtifacts', (db, signal) => serviceDatabase.query( diff --git a/src/test/noncurrent-lifecycle-database.test.ts b/src/test/noncurrent-lifecycle-database.test.ts index 2be11b2c6..a802c05eb 100644 --- a/src/test/noncurrent-lifecycle-database.test.ts +++ b/src/test/noncurrent-lifecycle-database.test.ts @@ -1298,6 +1298,22 @@ describe('noncurrent lifecycle database API', () => { ).resolves.toEqual([{ has_key: false, versions: 1 }]) }) + test('keeps the arm counters when the commit proposal predates the arm', async () => { + const fixture = await armBatch(['commit/stale-counters']) + const armed = await fixture.arm() + expect(armed?.continuation?.counters).toMatchObject({ + objectVersionsDeleted: 1, + bytesDeleted: 10, + batchesCompleted: 0, + }) + const committed = await fixture.commit() + expect(committed?.continuation?.counters).toMatchObject({ + objectVersionsDeleted: 1, + bytesDeleted: 10, + batchesCompleted: 1, + }) + }) + test('revalidates a recovery attempt only while the claim lease is live', async () => { const fixture = await armBatch(['arm/recovery']) await expect(fixture.arm()).resolves.toBeDefined() From 53c09d6b68e08a7401eb0f4be409b35ffe5a5e88 Mon Sep 17 00:00:00 2001 From: Ferhat Elmas Date: Fri, 30 Oct 2026 12:05:34 +0100 Subject: [PATCH 6/9] fix: distinguish arm rejection outcomes Signed-off-by: Ferhat Elmas --- src/storage/database/adapter.ts | 3 +- src/storage/database/pg.ts | 8 +- src/storage/schemas/lifecycle.ts | 5 + .../noncurrent-lifecycle-database.test.ts | 125 ++++++++++++------ 4 files changed, 93 insertions(+), 48 deletions(-) diff --git a/src/storage/database/adapter.ts b/src/storage/database/adapter.ts index 0554b7bde..67a7e9c26 100644 --- a/src/storage/database/adapter.ts +++ b/src/storage/database/adapter.ts @@ -8,6 +8,7 @@ import { EvaluateNoncurrentLifecyclePageInput, IcebergCatalog, LifecycleArmAttemptInput, + LifecycleArmAttemptResult, LifecycleBucket, LifecycleClaimedState, LifecycleClaimIdentity, @@ -233,7 +234,7 @@ export interface Database { input: EvaluateNoncurrentLifecyclePageInput ): Promise - armLifecycleAttempt(input: LifecycleArmAttemptInput): Promise + armLifecycleAttempt(input: LifecycleArmAttemptInput): Promise revalidateLifecycleRecoveryAttempt( input: LifecycleClaimIdentity, diff --git a/src/storage/database/pg.ts b/src/storage/database/pg.ts index f799e45e0..4cdd3cbc6 100644 --- a/src/storage/database/pg.ts +++ b/src/storage/database/pg.ts @@ -42,6 +42,7 @@ import { IcebergCatalog, LIFECYCLE_MAX_DELETE_BATCH_SIZE, LifecycleArmAttemptInput, + LifecycleArmAttemptResult, LifecycleBucket, LifecycleClaimedState, LifecycleClaimIdentity, @@ -3529,7 +3530,7 @@ export class StoragePgDB implements Database { return mapLifecycleEvaluationPage(row, input.pageSize) } - async armLifecycleAttempt(input: LifecycleArmAttemptInput): Promise { + async armLifecycleAttempt(input: LifecycleArmAttemptInput): Promise { if (!this.options.tnx) { return this.withTransaction((db) => db.armLifecycleAttempt(input)) } @@ -3641,7 +3642,8 @@ export class StoragePgDB implements Database { ) ) - if (!result.rows[0]?.claim_id) return undefined + if (!result.rows[0]) return { status: 'fenced' } + if (!result.rows[0].claim_id) return { status: 'ineligible' } // The metadata delete and armed journal commit together. // No new foreign-key reference can be added after this transaction commits. @@ -3691,7 +3693,7 @@ export class StoragePgDB implements Database { ) if (!saved.rows[0]) throw new Error('Lifecycle claim disappeared while arming metadata deletions') - return mapLifecycleState(saved.rows[0]) + return { status: 'armed', state: mapLifecycleState(saved.rows[0]) } } async revalidateLifecycleRecoveryAttempt( diff --git a/src/storage/schemas/lifecycle.ts b/src/storage/schemas/lifecycle.ts index f1f42bd00..1eaa2fc8e 100644 --- a/src/storage/schemas/lifecycle.ts +++ b/src/storage/schemas/lifecycle.ts @@ -173,6 +173,11 @@ export interface LifecycleArmAttemptInput extends LifecycleClaimIdentity { continuation: LifecycleContinuation } +export type LifecycleArmAttemptResult = + | { status: 'armed'; state: LifecycleState } + | { status: 'ineligible' } + | { status: 'fenced' } + export interface LifecycleCommitAttemptInput extends LifecycleClaimIdentity { attemptId: string continuation: LifecycleContinuation diff --git a/src/test/noncurrent-lifecycle-database.test.ts b/src/test/noncurrent-lifecycle-database.test.ts index a802c05eb..73dd6867a 100644 --- a/src/test/noncurrent-lifecycle-database.test.ts +++ b/src/test/noncurrent-lifecycle-database.test.ts @@ -475,8 +475,11 @@ describe('noncurrent lifecycle database API', () => { async ({ label: _label, ...options }) => { const fixture = await stageCountProtectedVersion(options) await expect(fixture.arm()).resolves.toMatchObject({ - continuation: { - batch: { inFlight: { attemptId: fixture.armed.batch!.inFlight!.attemptId } }, + status: 'armed', + state: { + continuation: { + batch: { inFlight: { attemptId: fixture.armed.batch!.inFlight!.attemptId } }, + }, }, }) await expect( @@ -495,7 +498,7 @@ describe('noncurrent lifecycle database API', () => { [bucketId, fixture.name, fixture.newerVersion] ) }) - await expect(fixture.arm()).resolves.toBeUndefined() + await expect(fixture.arm()).resolves.toEqual({ status: 'ineligible' }) await expect( tHelper.database.revalidateLifecycleClaim(fixture.identity) ).resolves.toBeUndefined() @@ -564,12 +567,14 @@ describe('noncurrent lifecycle database API', () => { leaseMs: 120_000, continuation: armed, }) - expect(armedState?.continuation).toMatchObject({ + expect(armedState.status).toBe('armed') + if (armedState.status !== 'armed') throw new Error('Expected armed result') + expect(armedState.state.continuation).toMatchObject({ batch: { inFlight: { attemptId: armed.batch!.inFlight!.attemptId } }, }) const partiallyRecorded = recordLifecycleArtifactOutcomes( - { ...armed, counters: armedState!.continuation!.counters }, + { ...armed, counters: armedState.state.continuation!.counters }, new Map([ [ armed.batch!.versions[0].artifacts[0].key, @@ -788,6 +793,13 @@ describe('noncurrent lifecycle database API', () => { await setState(`next_run_at = '2000-01-01'`) const stale = (await tHelper.database.findLifecycleBucket(bucketId)) .lifecycle_configuration_generation! + const row = { + id: randomUUID(), + name: 'policy/claim-race', + version: randomUUID(), + archivedAt: '2000-01-01T00:00:00.000Z', + } + await insertArchivedObjects([row]) const writer = new Client({ connectionString: getConfig().databaseURL }) await writer.connect() try { @@ -815,6 +827,17 @@ describe('noncurrent lifecycle database API', () => { await writer.query('COMMIT') expect((await claim)?.bucket.lifecycle_configuration_generation).toBe(stale) + const continuation = armedContinuation(stale!, [{ ...row, isDeleteMarker: false }], { + name: row.name, + archivedAt: row.archivedAt, + }) + await expect( + tHelper.database.armLifecycleAttempt({ ...identity, leaseMs: 60_000, continuation }) + ).resolves.toEqual({ status: 'fenced' }) + await expect(tHelper.database.revalidateLifecycleClaim(identity)).resolves.toMatchObject({ + claimId: identity.claimId, + }) + await expect( tHelper.database.completeLifecycleRun( { ...identity, generation: stale }, @@ -1015,7 +1038,7 @@ describe('noncurrent lifecycle database API', () => { { state: 'empty', capped: false }, ])('caps only preserved journals on release: $state', async ({ state, capped }) => { const fixture = await armBatch(['retry-cap']) - await expect(fixture.arm()).resolves.toBeDefined() + await expect(fixture.arm()).resolves.toMatchObject({ status: 'armed' }) const continuation = state === 'armed' ? encodeLifecycleContinuation(fixture.armed) @@ -1091,7 +1114,7 @@ describe('noncurrent lifecycle database API', () => { 'release without a schedule with $journal journal: $outcome', async ({ journal, outcome }) => { const fixture = await armBatch(['release/unscheduled']) - await expect(fixture.arm()).resolves.toBeDefined() + await expect(fixture.arm()).resolves.toMatchObject({ status: 'armed' }) if (journal === 'staged' || journal === 'empty') { await expect(fixture.commit()).resolves.toBeDefined() } @@ -1135,7 +1158,7 @@ describe('noncurrent lifecycle database API', () => { const fixture = await armBatch(['arm/expired']) await expireClaim() - await expect(fixture.arm()).resolves.toBeUndefined() + await expect(fixture.arm()).resolves.toEqual({ status: 'fenced' }) await expect(fixtures.readState(bucketId, 'continuation')).resolves.toEqual([ { continuation: null }, ]) @@ -1143,11 +1166,11 @@ describe('noncurrent lifecycle database API', () => { test('rejects arming over a journal in flight even when its rows are eligible again', async () => { const fixture = await armBatch(['arm/rearmed']) - await expect(fixture.arm()).resolves.toBeDefined() + await expect(fixture.arm()).resolves.toMatchObject({ status: 'armed' }) await insertArchivedObjects(fixture.rows) const before = await fixtures.readState(bucketId, 'continuation') - await expect(fixture.arm()).resolves.toBeUndefined() + await expect(fixture.arm()).resolves.toEqual({ status: 'fenced' }) await expect(fixtures.readState(bucketId, 'continuation')).resolves.toEqual(before) }) @@ -1179,7 +1202,9 @@ describe('noncurrent lifecycle database API', () => { ) await writer.finish('COMMIT') - await expect(fixture.arm()).resolves.toBeUndefined() + await expect(fixture.arm()).resolves.toEqual({ + status: claimKept ? 'fenced' : 'ineligible', + }) await expect(fixtures.readState(bucketId, 'continuation, claim_id')).resolves.toEqual([ { continuation: null, claim_id: claimKept ? fixture.identity.claimId : null }, ]) @@ -1194,7 +1219,7 @@ describe('noncurrent lifecycle database API', () => { ) await writer.finish('COMMIT') - await expect(fixture.arm()).resolves.toBeUndefined() + await expect(fixture.arm()).resolves.toEqual({ status: 'ineligible' }) await expect(fixtures.readState(bucketId, 'continuation, claim_id')).resolves.toEqual([ { continuation: null, claim_id: null }, ]) @@ -1202,7 +1227,7 @@ describe('noncurrent lifecycle database API', () => { test('persists the armed journal without artifact keys', async () => { const fixture = await armBatch(['arm/no-keys']) - await expect(fixture.arm()).resolves.toBeDefined() + await expect(fixture.arm()).resolves.toMatchObject({ status: 'armed' }) await expect( fixtures.readState( @@ -1227,13 +1252,16 @@ describe('noncurrent lifecycle database API', () => { const fixture = await armBatch(['arm/vetoed', 'arm/kept']) try { await reference(fixture.rows[0].id) - const state = await fixture.arm() + const result = await fixture.arm() + expect(result.status).toBe('armed') + if (result.status !== 'armed') throw new Error('Expected armed result') + const state = result.state - expect(state?.claimId).toBe(fixture.identity.claimId) - expect(state?.continuation?.batch?.versions.map(({ id }) => id)).toEqual([ + expect(state.claimId).toBe(fixture.identity.claimId) + expect(state.continuation?.batch?.versions.map(({ id }) => id)).toEqual([ fixture.rows[1].id, ]) - expect(state?.continuation?.counters).toMatchObject({ objectVersionsDeleted: 1 }) + expect(state.continuation?.counters).toMatchObject({ objectVersionsDeleted: 1 }) await expect( fixtures.readState( bucketId, @@ -1253,10 +1281,13 @@ describe('noncurrent lifecycle database API', () => { const skipped = { ...rest, cursor: pageEnd } try { await reference(fixture.rows[0].id) - const state = await fixture.arm() + const result = await fixture.arm() + expect(result.status).toBe('armed') + if (result.status !== 'armed') throw new Error('Expected armed result') + const state = result.state - expect(state?.claimId).toBe(fixture.identity.claimId) - expect(state?.continuation).toEqual(skipped) + expect(state.claimId).toBe(fixture.identity.claimId) + expect(state.continuation).toEqual(skipped) await expect( fixtures.readState( bucketId, @@ -1276,7 +1307,7 @@ describe('noncurrent lifecycle database API', () => { try { await reference(fixture.rows[0].id) - await expect(fixture.arm()).resolves.toBeUndefined() + await expect(fixture.arm()).resolves.toEqual({ status: 'fenced' }) await expect(fixtures.readState(bucketId, 'continuation')).resolves.toEqual([ { continuation: null }, ]) @@ -1287,7 +1318,7 @@ describe('noncurrent lifecycle database API', () => { test('persists the committed results without artifact keys', async () => { const fixture = await armBatch(['commit/no-keys']) - await expect(fixture.arm()).resolves.toBeDefined() + await expect(fixture.arm()).resolves.toMatchObject({ status: 'armed' }) await expect(fixture.commit(fixture.armed)).resolves.toBeDefined() await expect( @@ -1301,7 +1332,9 @@ describe('noncurrent lifecycle database API', () => { test('keeps the arm counters when the commit proposal predates the arm', async () => { const fixture = await armBatch(['commit/stale-counters']) const armed = await fixture.arm() - expect(armed?.continuation?.counters).toMatchObject({ + expect(armed.status).toBe('armed') + if (armed.status !== 'armed') throw new Error('Expected armed result') + expect(armed.state.continuation?.counters).toMatchObject({ objectVersionsDeleted: 1, bytesDeleted: 10, batchesCompleted: 0, @@ -1316,7 +1349,7 @@ describe('noncurrent lifecycle database API', () => { test('revalidates a recovery attempt only while the claim lease is live', async () => { const fixture = await armBatch(['arm/recovery']) - await expect(fixture.arm()).resolves.toBeDefined() + await expect(fixture.arm()).resolves.toMatchObject({ status: 'armed' }) const attemptId = fixture.armed.batch!.inFlight!.attemptId await expect( @@ -1336,7 +1369,7 @@ describe('noncurrent lifecycle database API', () => { const fixture = await armBatch(['arm/cutoff'], { archivedAt, snapshotAt: SNAPSHOT_AT }) const armed = await fixture.arm() - expect(armed !== undefined).toBe(arms) + expect(armed.status).toBe(arms ? 'armed' : 'ineligible') if (!arms) { await expect(fixtures.readState(bucketId, 'continuation, claim_id')).resolves.toEqual([ { continuation: null, claim_id: null }, @@ -1375,7 +1408,7 @@ describe('noncurrent lifecycle database API', () => { siblings.map((archivedAt) => ({ name: 'arm/newer', archivedAt })) ) - expect((await fixture.arm()) !== undefined).toBe(arms) + expect((await fixture.arm()).status).toBe(arms ? 'armed' : 'ineligible') } ) @@ -1386,7 +1419,7 @@ describe('noncurrent lifecycle database API', () => { }) await insertOtherBucketSibling('arm/newer-scope', '2026-07-01T00:00:00.000Z') - await expect(fixture.arm()).resolves.toBeUndefined() + await expect(fixture.arm()).resolves.toEqual({ status: 'ineligible' }) }) const rowExists = async ({ name, version }: { name: string; version: string }) => @@ -1446,7 +1479,9 @@ describe('noncurrent lifecycle database API', () => { ) const armed = await fixture.arm() - expect(armed?.continuation?.batch?.inFlight).toBeDefined() + expect(armed.status).toBe('armed') + if (armed.status !== 'armed') throw new Error('Expected armed result') + expect(armed.state.continuation?.batch?.inFlight).toBeDefined() expect(await rowExists(row)).toBe(false) const before = await fixtures.readState(bucketId, 'continuation, claim_id, next_run_at') const nextRunAt = '2099-01-01T00:00:00.000Z' @@ -1458,9 +1493,9 @@ describe('noncurrent lifecycle database API', () => { ) ).resolves.toBe(false) const changed = recordLifecycleArtifactOutcomes( - armed!.continuation!, + armed.state.continuation!, new Map([ - [armed!.continuation!.batch!.versions[0].artifacts[0].key, { outcome: 'DELETED' }], + [armed.state.continuation!.batch!.versions[0].artifacts[0].key, { outcome: 'DELETED' }], ]) ) for (const continuation of [null, fixture.staged]) { @@ -1468,7 +1503,7 @@ describe('noncurrent lifecycle database API', () => { tHelper.database.releaseLifecycleClaim({ ...fixture.identity, continuation, nextRunAt }) ).resolves.toBe(false) } - for (const continuation of [changed, armed!.continuation!]) { + for (const continuation of [changed, armed.state.continuation!]) { await expect( tHelper.database.releaseLifecycleClaim({ ...fixture.identity, continuation, nextRunAt }) ).rejects.toMatchObject({ code: 'InvalidParameter' }) @@ -1479,9 +1514,9 @@ describe('noncurrent lifecycle database API', () => { await expect(adapter.headObject(root, key, row.version)).resolves.toMatchObject({ size: 13 }) await adapter.deleteObject(root, key, row.version) - const [data, info] = armed!.continuation!.batch!.versions[0].artifacts + const [data, info] = armed.state.continuation!.batch!.versions[0].artifacts const recorded = recordLifecycleArtifactOutcomes( - armed!.continuation!, + armed.state.continuation!, new Map([ [data.key, { outcome: 'DELETED' as const }], [info.key, { outcome: 'ABSENT' as const }], @@ -1530,7 +1565,7 @@ describe('noncurrent lifecycle database API', () => { test('preserves an in-flight journal with newer fields when releasing the claim', async () => { const fixture = await armBatch(['release/future-fields']) - await expect(fixture.arm()).resolves.toBeDefined() + await expect(fixture.arm()).resolves.toMatchObject({ status: 'armed' }) const nextRunAt = '2099-01-01T01:00:00.000Z' await tHelper.database.connection.query( `UPDATE storage.bucket_lifecycle_states @@ -1559,7 +1594,7 @@ describe('noncurrent lifecycle database API', () => { test('keeps a policy change made during an armed batch due after the stale run completes', async () => { const fixture = await armBatch(['policy/armed']) - await expect(fixture.arm()).resolves.toBeDefined() + await expect(fixture.arm()).resolves.toMatchObject({ status: 'armed' }) await tHelper.database.putLifecycleConfiguration(bucketId, configuration(2)) await expect(fixture.commit()).resolves.toBeDefined() @@ -1575,7 +1610,7 @@ describe('noncurrent lifecycle database API', () => { test('rejects a commit for a stale attempt', async () => { const fixture = await armBatch(['commit/stale-attempt']) - await expect(fixture.arm()).resolves.toBeDefined() + await expect(fixture.arm()).resolves.toMatchObject({ status: 'armed' }) const continuation = fixture.recorded() const attemptId = randomUUID() continuation.batch!.inFlight!.attemptId = attemptId @@ -1593,7 +1628,7 @@ describe('noncurrent lifecycle database API', () => { test('abandons pending artifacts only for the claim holder and advances the cursor', async () => { const fixture = await armBatch(['abandon/persist']) - await expect(fixture.arm()).resolves.toBeDefined() + await expect(fixture.arm()).resolves.toMatchObject({ status: 'armed' }) const keys = fixture.armed.batch!.versions[0].artifacts.map((artifact) => artifact.key) const failed = recordLifecycleArtifactOutcomes( fixture.armed, @@ -1664,13 +1699,13 @@ describe('noncurrent lifecycle database API', () => { const fixture = await armBatch(['identity/pre-arm']) await mutateRow(kind, fixture.rows[0].id) - await expect(fixture.arm()).resolves.toBeUndefined() + await expect(fixture.arm()).resolves.toEqual({ status: 'ineligible' }) } ) test('rejects a commit whose version or artifact set differs from the armed batch', async () => { const fixture = await armBatch(['commit/a', 'commit/b']) - await expect(fixture.arm()).resolves.toBeDefined() + await expect(fixture.arm()).resolves.toMatchObject({ status: 'armed' }) const recorded = fixture.recorded() const dropped = { @@ -1724,7 +1759,7 @@ describe('noncurrent lifecycle database API', () => { const arming = fixture.arm() await held.blocked() await held.finish('COMMIT') - await expect(arming).resolves.toBeUndefined() + await expect(arming).resolves.toEqual({ status: 'ineligible' }) const rows = await tHelper.database.connection.query( `SELECT 1 FROM storage.objects WHERE ${column} = $1`, [replacement] @@ -1747,7 +1782,7 @@ describe('noncurrent lifecycle database API', () => { try { await promoter.query(`BEGIN ISOLATION LEVEL ${isolation}`) await promoter.query('SELECT 1') - await expect(fixture.arm()).resolves.toBeDefined() + await expect(fixture.arm()).resolves.toMatchObject({ status: 'armed' }) const promotion = promoter.query( 'UPDATE storage.objects SET archived_at = NULL WHERE id = $1', [fixture.rows[0].id] @@ -1829,7 +1864,7 @@ describe('noncurrent lifecycle database API', () => { .toBeGreaterThan(0) release.resolve() await holding - await expect(arming).resolves.toBeDefined() + await expect(arming).resolves.toMatchObject({ status: 'armed' }) }) test.each([ @@ -1866,7 +1901,7 @@ describe('noncurrent lifecycle database API', () => { ) expect(free.rows).toEqual([{ name: names[0], version: fixture.rows[0].version }]) await held.finish('ROLLBACK') - await expect(arming).resolves.toBeDefined() + await expect(arming).resolves.toMatchObject({ status: 'armed' }) } finally { await held.writer.end().catch(() => undefined) } @@ -1879,7 +1914,9 @@ describe('noncurrent lifecycle database API', () => { let deleting: Promise | undefined try { await tHelper.database.withTransaction(async (db) => { - await expect(db.armLifecycleAttempt(fixture.input)).resolves.toBeDefined() + await expect(db.armLifecycleAttempt(fixture.input)).resolves.toMatchObject({ + status: 'armed', + }) await deleter.query('BEGIN') await deleter.query("SELECT set_config('storage.allow_delete_query', 'true', true)") const pid = (await deleter.query('SELECT pg_backend_pid() AS pid')).rows[0].pid From c3fd38971b0c87bbb6704f72c1e88771df0830b8 Mon Sep 17 00:00:00 2001 From: Ferhat Elmas Date: Fri, 30 Oct 2026 21:15:51 +0100 Subject: [PATCH 7/9] fix: remove explicit postgres grants Signed-off-by: Ferhat Elmas --- migrations/tenant/0076-noncurrent-lifecycle.sql | 2 +- migrations/tenant/0079-lifecycle-writer-coordination.sql | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/migrations/tenant/0076-noncurrent-lifecycle.sql b/migrations/tenant/0076-noncurrent-lifecycle.sql index 3d105303a..9d32437d4 100644 --- a/migrations/tenant/0076-noncurrent-lifecycle.sql +++ b/migrations/tenant/0076-noncurrent-lifecycle.sql @@ -76,7 +76,7 @@ BEGIN END IF; END LOOP; - FOREACH role_name IN ARRAY ARRAY['postgres', super_user, service_role] + FOREACH role_name IN ARRAY ARRAY[super_user, service_role] LOOP IF EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = role_name) THEN EXECUTE format('GRANT ALL ON TABLE storage.bucket_lifecycle_states TO %I', role_name); diff --git a/migrations/tenant/0079-lifecycle-writer-coordination.sql b/migrations/tenant/0079-lifecycle-writer-coordination.sql index ed4364a91..949a4ced9 100644 --- a/migrations/tenant/0079-lifecycle-writer-coordination.sql +++ b/migrations/tenant/0079-lifecycle-writer-coordination.sql @@ -172,7 +172,6 @@ BEGIN END LOOP; FOREACH role_name IN ARRAY ARRAY[ - 'postgres', COALESCE(current_setting('storage.super_user', true), 'postgres'), COALESCE(current_setting('storage.service_role', true), 'service_role') ] LOOP @@ -306,7 +305,6 @@ BEGIN END LOOP; FOREACH role_name IN ARRAY ARRAY[ - 'postgres', COALESCE(current_setting('storage.super_user', true), 'postgres'), COALESCE(current_setting('storage.service_role', true), 'service_role') ] LOOP @@ -356,7 +354,7 @@ BEGIN END IF; END LOOP; - FOREACH role_name IN ARRAY ARRAY['postgres', super_user, service_role] + FOREACH role_name IN ARRAY ARRAY[super_user, service_role] LOOP IF EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = role_name) THEN EXECUTE format('GRANT ALL ON TABLE storage.lifecycle_abandoned_artifacts TO %I', role_name); From dc0fdb4e9b7856b47b34d532b2307bcec6594597 Mon Sep 17 00:00:00 2001 From: Ferhat Elmas Date: Fri, 30 Oct 2026 21:38:49 +0100 Subject: [PATCH 8/9] chore: rebase fix Signed-off-by: Ferhat Elmas --- .../bucket-lifecycle-configuration.test.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/test/bucket-lifecycle-configuration.test.ts b/src/test/bucket-lifecycle-configuration.test.ts index 85dfb277d..a4e84155a 100644 --- a/src/test/bucket-lifecycle-configuration.test.ts +++ b/src/test/bucket-lifecycle-configuration.test.ts @@ -56,13 +56,20 @@ describe('bucket lifecycle configuration persistence', () => { it('schedules a policy written by a writer that predates the control plane', async () => { await fixtures.setStatus(bucketId, 'ENABLED') - const hasMigration = vi - .spyOn(StoragePgDB.prototype, 'hasMigration') - .mockImplementation(async (name) => name !== 'lifecycle-writer-coordination') + const transaction = await helper.database.connection.transaction() try { - await putLifecycleConfiguration({ rules }) - } finally { - hasMigration.mockRestore() + await helper.database.connection.setScope(transaction) + await transaction.query( + `UPDATE storage.buckets + SET lifecycle_configuration = $2::jsonb, + lifecycle_configuration_generation = $3::uuid + WHERE id = $1`, + [bucketId, JSON.stringify({ rules }), randomUUID()] + ) + await transaction.commit() + } catch (error) { + await transaction.rollback() + throw error } await expect( fixtures.readState(bucketId, 'next_run_at <= clock_timestamp() AS due, claim_id') From a2af09d3a9447a0fad10be89eb5b64096754c140 Mon Sep 17 00:00:00 2001 From: Ferhat Elmas Date: Tue, 3 Nov 2026 21:53:51 +0100 Subject: [PATCH 9/9] chore: revert postgres grant since migration out Signed-off-by: Ferhat Elmas --- migrations/tenant/0076-noncurrent-lifecycle.sql | 2 +- migrations/tenant/0079-lifecycle-writer-coordination.sql | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/migrations/tenant/0076-noncurrent-lifecycle.sql b/migrations/tenant/0076-noncurrent-lifecycle.sql index 9d32437d4..3d105303a 100644 --- a/migrations/tenant/0076-noncurrent-lifecycle.sql +++ b/migrations/tenant/0076-noncurrent-lifecycle.sql @@ -76,7 +76,7 @@ BEGIN END IF; END LOOP; - FOREACH role_name IN ARRAY ARRAY[super_user, service_role] + FOREACH role_name IN ARRAY ARRAY['postgres', super_user, service_role] LOOP IF EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = role_name) THEN EXECUTE format('GRANT ALL ON TABLE storage.bucket_lifecycle_states TO %I', role_name); diff --git a/migrations/tenant/0079-lifecycle-writer-coordination.sql b/migrations/tenant/0079-lifecycle-writer-coordination.sql index 949a4ced9..ed4364a91 100644 --- a/migrations/tenant/0079-lifecycle-writer-coordination.sql +++ b/migrations/tenant/0079-lifecycle-writer-coordination.sql @@ -172,6 +172,7 @@ BEGIN END LOOP; FOREACH role_name IN ARRAY ARRAY[ + 'postgres', COALESCE(current_setting('storage.super_user', true), 'postgres'), COALESCE(current_setting('storage.service_role', true), 'service_role') ] LOOP @@ -305,6 +306,7 @@ BEGIN END LOOP; FOREACH role_name IN ARRAY ARRAY[ + 'postgres', COALESCE(current_setting('storage.super_user', true), 'postgres'), COALESCE(current_setting('storage.service_role', true), 'service_role') ] LOOP @@ -354,7 +356,7 @@ BEGIN END IF; END LOOP; - FOREACH role_name IN ARRAY ARRAY[super_user, service_role] + FOREACH role_name IN ARRAY ARRAY['postgres', super_user, service_role] LOOP IF EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = role_name) THEN EXECUTE format('GRANT ALL ON TABLE storage.lifecycle_abandoned_artifacts TO %I', role_name);