From 7fe58f7854183a7d3fd72cfd3ece2f173ffd50d7 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 4 Aug 2026 15:00:34 +0200 Subject: [PATCH 01/38] V3 storage: Lite compact after intial replication. --- .../src/replication/ConvexStream.ts | 2 +- .../test/src/ConvexStream.test.ts | 2 +- .../storage/implementation/MongoCompactor.ts | 53 ++--- .../implementation/MongoSyncBucketStorage.ts | 29 +-- .../implementation/v1/MongoCompactorV1.ts | 37 ++- .../v1/MongoSyncBucketStorageV1.ts | 27 ++- .../implementation/v3/MongoCompactorV3.ts | 212 ++++++++++++++++-- .../v3/MongoSyncBucketStorageV3.ts | 21 ++ .../test/src/storage_compacting.test.ts | 32 ++- .../src/replication/MongoSnapshotter.ts | 4 +- .../src/replication/BinLogStream.ts | 4 +- .../src/storage/PostgresSyncRulesStorage.ts | 8 +- .../src/replication/WalStream.ts | 4 +- .../src/storage/SyncRulesBucketStorage.ts | 14 +- 14 files changed, 347 insertions(+), 102 deletions(-) diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index 83477cb97..57e2a938b 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -140,7 +140,7 @@ export class ConvexStream { const { lastOpId } = await this.initialReplication(status.snapshotLsn); if (lastOpId != null) { - await this.storage.populatePersistentChecksumCache({ + await this.storage.compactInitialReplication({ signal: this.abortSignal, maxOpId: lastOpId }); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index 269512af6..9e8aa0b27 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -173,7 +173,7 @@ function createFakeStorage(options?: { }; Object.assign(storage, { clear: vi.fn(async () => undefined), - populatePersistentChecksumCache: vi.fn(async () => ({ buckets: 0 })), + compactInitialReplication: vi.fn(async () => ({ buckets: 0 })), createWriter: vi.fn(async (_options: any) => batch), startBatch: vi.fn(async (_options: any, callback: (batch: any) => Promise) => { await callback(batch); diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index 34eeaa95c..519f109aa 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -7,7 +7,7 @@ import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; -import { InternalOpId, isPartialChecksum, PopulateChecksumCacheResults, storage } from '@powersync/service-core'; +import { InternalOpId, isPartialChecksum, storage } from '@powersync/service-core'; import { BucketDefinitionId } from '@powersync/service-sync-rules'; import { BucketKey } from './common/BucketDataDoc.js'; @@ -50,7 +50,14 @@ export interface CurrentBucketState { opBytes: number; } -export interface MongoCompactOptions extends storage.CompactOptions {} +export interface MongoCompactOptions extends storage.CompactOptions { + /** + * Only merge adjacent V3 bucket-data chunks. This is used after initial + * replication, where reading every operation would defeat the purpose of + * the lightweight pass. + */ + compactChunksOnly?: boolean; +} const DEFAULT_CLEAR_BATCH_LIMIT = 5000; const DEFAULT_MOVE_BATCH_LIMIT = 2000; @@ -93,6 +100,8 @@ export abstract class MongoCompactor { protected readonly deleteCheckpointRequestsBefore: Date | undefined; protected readonly signal?: AbortSignal; protected readonly group_id: number; + protected readonly compactChunksOnly: boolean; + protected compactedBucketCount = 0; protected readonly logger: Logger; @@ -116,6 +125,7 @@ export abstract class MongoCompactor { this.buckets = options.compactBuckets; this.deleteCheckpointRequestsBefore = options.deleteCheckpointRequestsBefore; this.signal = options.signal; + this.compactChunksOnly = options.compactChunksOnly ?? false; this.logger = options.logger ?? defaultLogger; } @@ -124,7 +134,7 @@ export abstract class MongoCompactor { * * See /docs/storage/compacting-operations.md for details. */ - async compact() { + async compact(): Promise { await this.deleteOldCheckpointRequests(); if (this.buckets) { @@ -136,6 +146,8 @@ export abstract class MongoCompactor { } else { await this.compactDirtyBuckets(); } + + return this.compactedBucketCount; } private async deleteOldCheckpointRequests() { @@ -157,41 +169,6 @@ export abstract class MongoCompactor { }); } - /** - * Subset of compact, only populating checksums where relevant. - */ - async populateChecksums(options: { minBucketChanges: number }): Promise { - let count = 0; - // Paginate through dirty buckets in batches until no more buckets meet the criteria. - while (true) { - this.signal?.throwIfAborted(); - const buckets = await this.dirtyBucketBatchForChecksums(options); - if (buckets.length == 0) { - break; - } - this.signal?.throwIfAborted(); - - const start = Date.now(); - // Filter batch by estimated bucket size, to reduce possibility of timeouts. - const checkBuckets: typeof buckets = []; - let totalCountEstimate = 0; - for (const bucket of buckets) { - checkBuckets.push(bucket); - totalCountEstimate += bucket.estimatedCount; - if (totalCountEstimate > 50_000) { - break; - } - } - this.logger.info( - `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}` - ); - await this.updateChecksumsBatch(checkBuckets); - this.logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`); - count += checkBuckets.length; - } - return { buckets: count }; - } - protected async *dirtyBucketBatchesForCollection( collection: mongo.Collection, lastId: TCollectionBucketState['_id'], diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 9183b7814..8a926a7c1 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -11,11 +11,11 @@ import { BroadcastIterable, CHECKPOINT_INVALIDATE_ALL, CheckpointChanges, + CompactInitialReplicationOptions, + CompactInitialReplicationResults, GetCheckpointChangesOptions, InternalOpId, mergeAsyncIterables, - PopulateChecksumCacheOptions, - PopulateChecksumCacheResults, ReplicationCheckpoint, ReplicationStreamStorageIds, storage, @@ -377,33 +377,18 @@ export abstract class MongoSyncBucketStorage } } + abstract compactInitialReplication( + options: CompactInitialReplicationOptions + ): Promise; + /** * The highest op id persisted for this stream, whether or not covered by a checkpoint. * - * Used as the default `maxOpId` for {@link populatePersistentChecksumCache}, which runs after + * Used as the default `maxOpId` for {@link compactInitialReplication}, which runs after * initial replication but before the first checkpoint exists. */ protected abstract fetchPersistedOpHead(): Promise; - async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise { - this.logger.info(`Populating persistent checksum cache...`); - const start = Date.now(); - const maxOpId = options.maxOpId ?? (await this.fetchPersistedOpHead()) ?? undefined; - const compactor = this.createMongoCompactor({ - ...options, - maxOpId, - memoryLimitMB: 0, - logger: this.logger - }); - - const result = await compactor.populateChecksums({ - minBucketChanges: options.minBucketChanges ?? 10 - }); - const duration = Date.now() - start; - this.logger.info(`Populated persistent checksum cache in ${(duration / 1000).toFixed(1)}s`); - return result; - } - private async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable { if (signal.aborted) { return; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index 360424b9b..9a3be6eac 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -1,6 +1,6 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { ReplicationAssertionError } from '@powersync/lib-services-framework'; -import { addChecksums, storage, utils } from '@powersync/service-core'; +import { addChecksums, CompactInitialReplicationResults, storage, utils } from '@powersync/service-core'; import { BucketDefinitionId } from '@powersync/service-sync-rules'; import { BucketDataDoc } from '../common/BucketDataDoc.js'; import { BucketStateDocumentBase, LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js'; @@ -55,6 +55,41 @@ export class MongoCompactorV1 extends MongoCompactor { ); } + /** + * Subset of compact, only populating checksums where relevant. + */ + async populateChecksums(options: { minBucketChanges: number }): Promise { + let count = 0; + // Paginate through dirty buckets in batches until no more buckets meet the criteria. + while (true) { + this.signal?.throwIfAborted(); + const buckets = await this.dirtyBucketBatchForChecksums(options); + if (buckets.length == 0) { + break; + } + this.signal?.throwIfAborted(); + + const start = Date.now(); + // Filter batch by estimated bucket size, to reduce possibility of timeouts. + const checkBuckets: typeof buckets = []; + let totalCountEstimate = 0; + for (const bucket of buckets) { + checkBuckets.push(bucket); + totalCountEstimate += bucket.estimatedCount; + if (totalCountEstimate > 50_000) { + break; + } + } + this.logger.info( + `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}` + ); + await this.updateChecksumsBatch(checkBuckets); + this.logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`); + count += checkBuckets.length; + } + return { buckets: count }; + } + protected async writeBucketStateUpdates(): Promise { await this.db.bucketStateV1.bulkWrite( this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 89fec0c47..06e412b2c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -3,6 +3,8 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { ServiceAssertionError } from '@powersync/lib-services-framework'; import { CheckpointChanges, + CompactInitialReplicationOptions, + CompactInitialReplicationResults, deserializeParameterLookup, GetCheckpointChangesOptions, InternalOpId, @@ -29,7 +31,7 @@ import { MongoBucketStorage } from '../../MongoBucketStorage.js'; import { MongoSyncBucketStorageCheckpoint } from '../common/MongoSyncBucketStorageCheckpoint.js'; import { SourceKey } from '../models.js'; import { MongoChecksums } from '../MongoChecksums.js'; -import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; +import { MongoCompactOptions } from '../MongoCompactor.js'; import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js'; import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js'; @@ -186,10 +188,31 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { }); } - createMongoCompactor(options: MongoCompactOptions): MongoCompactor { + createMongoCompactor(options: MongoCompactOptions): MongoCompactorV1 { return new MongoCompactorV1(this, this.db, options); } + async compactInitialReplication( + options: CompactInitialReplicationOptions + ): Promise { + this.logger.info(`Compacting after initial replication...`); + const start = Date.now(); + const maxOpId = options.maxOpId ?? (await this.fetchPersistedOpHead()) ?? undefined; + const compactor = this.createMongoCompactor({ + ...options, + maxOpId, + memoryLimitMB: 0, + logger: this.logger + }); + + const result = await compactor.populateChecksums({ + minBucketChanges: options.minBucketChanges ?? 10 + }); + const duration = Date.now() - start; + this.logger.info(`Compacted after initial replication in ${(duration / 1000).toFixed(1)}s`); + return result; + } + protected createMongoParameterCompactor( checkpoint: InternalOpId, options: storage.CompactOptions diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index cd4434968..2f56a7c48 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -4,7 +4,7 @@ import { addChecksums, InternalOpId, storage, utils } from '@powersync/service-c import { BucketDefinitionId } from '@powersync/service-sync-rules'; import { BucketDataDoc } from '../common/BucketDataDoc.js'; import { BucketDataKey, BucketStateDocumentBase } from '../models.js'; -import { ConcurrentCompactionError, DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; +import { ConcurrentCompactionError, CurrentBucketState, DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; import { cacheKey } from '../OperationBatch.js'; import { loadBucketDataDocument, maxOpId, serializeBucketData } from './bucket-format.js'; import { BucketDataContextV3 } from './BucketDataContextV3.js'; @@ -28,12 +28,12 @@ interface PendingCompactionGroup { } /** - * Read one bounded prefix from a descending compaction cursor. + * Read one bounded prefix from a compaction cursor. * * The document that would cross the byte limit is deliberately not returned: - * pagination resumes below the last returned `_id`, so that document remains - * eligible for the next query. The first document is always accepted to ensure - * progress when a single document exceeds the configured byte limit. + * pagination resumes past the last returned `_id`, so that document remains + * eligible for the next query. The first document is always accepted to + * ensure progress when a single document exceeds the configured byte limit. * * `hasMore` is conservative when the document limit is reached. An extra empty * query is preferable to exhausting the cursor just to determine whether the @@ -69,7 +69,7 @@ export class MongoCompactorV3 extends MongoCompactor { declare protected readonly db: VersionedPowerSyncMongoV3; declare protected readonly storage: MongoSyncBucketStorageV3; - override async compact(): Promise { + override async compact(): Promise { if (this.storage.objectStorage) { // Clean these before compacting - should be quick in most cases. try { @@ -79,13 +79,14 @@ export class MongoCompactorV3 extends MongoCompactor { this.logger.error(`Failed to clean up object storage deletion markers before compaction`, e); } } - await super.compact(); + const compactedBuckets = await super.compact(); if (this.storage.objectStorage) { // Cleanup for any produced during compacting. // Note that markers only expire after a delay, so this may skip many produced during this compact // run. However, during long compact runs, this may also have many ones it can clean up. await this.objectStorageLifecycle.cleanup(this.logger); } + return compactedBuckets; } private get objectStorageLifecycle(): ObjectStorageLifecycle { @@ -141,6 +142,189 @@ export class MongoCompactorV3 extends MongoCompactor { return this.storage.checksums as MongoChecksumsV3; } + protected override async compactSingleBucket(bucket: string, definitionId: BucketDefinitionId | null = null) { + if (this.compactChunksOnly) { + return this.compactSingleBucketChunks(bucket, definitionId); + } + + return this.compactSingleBucketFully(bucket, definitionId); + } + + /** + * Merge adjacent bucket-data chunks without inspecting their operations + * unless a merge is possible. The metadata contains enough information to + * update the persisted checksum state and to decide whether a group can fit + * in one chunk. + */ + private async compactSingleBucketChunks(bucket: string, definitionId: BucketDefinitionId | null) { + const bucketContext = await this.getBucketDataContext(bucket, definitionId); + if (bucketContext == null) { + return; + } + + const resolvedDefinitionId = bucketContext.key.definitionId; + const collection = this.db.bucketData(this.group_id, resolvedDefinitionId); + const context = { replicationStreamId: this.group_id, definitionId: resolvedDefinitionId }; + let lowerBound = bucketContext.minId; + const upperBound = bucketContext.docId(this.maxOpId + 1n); + + let compactedOpId: bigint | null = null; + let totalChecksum = 0; + let totalOpCount = 0; + let totalOpBytes = 0; + let pendingChunks: BucketDataDocumentV3[] = []; + let pendingSize = 0; + + while (true) { + this.signal?.throwIfAborted(); + + const batch = await readCompactionBatch( + collection.aggregate( + [ + { + $match: { + _id: { + $gt: lowerBound, + $lt: upperBound + } + } + }, + { $sort: { _id: 1 } }, + { $limit: this.moveBatchQueryLimit }, + { + $project: { + _id: 1, + min_op: 1, + checksum: 1, + count: 1, + size: 1, + target_op: 1, + storage_ref: 1, + has_clear_op: 1 + } + } + ], + { batchSize: this.moveBatchQueryLimit + 1 } + ), + { + byteLimit: this.moveBatchByteLimit, + documentLimit: this.moveBatchQueryLimit + } + ); + + if (batch.documents.length == 0) { + break; + } + + for (const doc of batch.documents) { + compactedOpId = maxOpId(compactedOpId, doc._id.o); + totalChecksum = addChecksums(totalChecksum, Number(doc.checksum)); + totalOpCount += doc.count; + totalOpBytes += doc.size; + + const nextSize = pendingSize + doc.size; + if (pendingChunks.length > 0 && nextSize > DEFAULT_MAX_DOC_SIZE_BYTES) { + await this.flushChunkMerge(bucket, pendingChunks, collection, context, bucketContext); + pendingChunks = []; + pendingSize = 0; + } + + pendingChunks.push(doc); + pendingSize += doc.size; + } + + lowerBound = batch.documents[batch.documents.length - 1]._id; + if (!batch.hasMore) { + break; + } + } + + if (pendingChunks.length > 1) { + await this.flushChunkMerge(bucket, pendingChunks, collection, context, bucketContext); + } + + if (compactedOpId == null) { + return; + } + + await this.finalizeCompactedBucket( + { + bucket, + definitionId: resolvedDefinitionId, + lastNotPut: null, + opsSincePut: 0, + checksum: totalChecksum, + opCount: totalOpCount, + opBytes: totalOpBytes + }, + compactedOpId + ); + this.compactedBucketCount++; + this.logger.info(`Lightly compacted bucket ${bucket}: ${totalOpCount} ops`); + } + + private async flushChunkMerge( + bucket: string, + inputs: BucketDataDocumentV3[], + collection: mongo.Collection, + context: { replicationStreamId: number; definitionId: string }, + bucketContext: BucketDataContextV3 + ) { + if (inputs.length < 2) { + return; + } + + // The metadata scan deliberately excluded ops. Read inline payloads only + // for this merge group; object-storage payloads are fetched below using + // the same rule. + const inlineInputs = inputs.filter((input) => input.storage_ref == null); + if (inlineInputs.length > 0) { + const inlineDocuments = await collection + .find({ _id: { $in: inlineInputs.map((input) => input._id) } }, { projection: { _id: 1, ops: 1 } }) + .toArray(); + const opsById = new Map(inlineDocuments.map((document) => [document._id.o.toString(), document.ops])); + for (const input of inlineInputs) { + input.ops = opsById.get(input._id.o.toString()); + } + } + await hydrateBucketDataDocuments(inputs, this.storage.objectStorage, { signal: this.signal }); + + const operations = inputs.flatMap((input) => Array.from(loadBucketDataDocument(context, input))); + const targetOp = inputs.reduce( + (maxTarget, input) => maxOpId(maxTarget, input.target_op), + null + ); + await this.flushCompactionGroup( + bucket, + { + inputs, + ops: operations, + changed: true, + targetOp + }, + bucketContext, + context + ); + } + + private async finalizeCompactedBucket( + state: Pick< + CurrentBucketState, + 'bucket' | 'definitionId' | 'lastNotPut' | 'opsSincePut' | 'checksum' | 'opCount' | 'opBytes' + >, + compactedOpId: InternalOpId + ) { + this.updateBucketChecksums( + { + ...state, + seen: new Map(), + trackingSize: 0 + }, + compactedOpId + ); + await this.flushBucketStateUpdates(); + } + protected async computeChecksumsForBuckets( buckets: Pick[] ): Promise { @@ -203,7 +387,7 @@ export class MongoCompactorV3 extends MongoCompactor { }); } - protected override async compactSingleBucket(bucket: string, definitionId: BucketDefinitionId | null = null) { + private async compactSingleBucketFully(bucket: string, definitionId: BucketDefinitionId | null = null) { const bucketContext = await this.getBucketDataContext(bucket, definitionId); if (bucketContext == null) { return; @@ -414,24 +598,18 @@ export class MongoCompactorV3 extends MongoCompactor { } // --- Finalize: update bucket checksums and state --- - this.updateBucketChecksums( + await this.finalizeCompactedBucket( { bucket, definitionId: resolvedDefinitionId, - seen: new Map(), - trackingSize: 0, - lastNotPut: lastNotPut, - opsSincePut: opsSincePut, + lastNotPut, + opsSincePut, checksum: totalChecksum, opCount: totalOpCount, opBytes: totalOpBytes }, compactedOpId ); - if (this.bucketStateUpdates.length > 0) { - await this.writeBucketStateUpdates(); - this.bucketStateUpdates = []; - } logger.info(`Compacted bucket ${bucket}: ${totalOpCount} surviving ops`); } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index f67dff166..b71f7761c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -3,6 +3,8 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { ServiceAssertionError } from '@powersync/lib-services-framework'; import { CheckpointChanges, + CompactInitialReplicationOptions, + CompactInitialReplicationResults, GetCheckpointChangesOptions, InternalOpId, internalToExternalOpId, @@ -192,6 +194,25 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { return new MongoCompactorV3(this, this.db, options); } + override async compactInitialReplication( + options: CompactInitialReplicationOptions + ): Promise { + this.logger.info(`Compacting chunks after initial replication...`); + const start = Date.now(); + const maxOpId = options.maxOpId ?? (await this.fetchPersistedOpHead()) ?? undefined; + const compactedBuckets = await this.createMongoCompactor({ + ...options, + maxOpId, + // A metadata-only scan is cheap, so include buckets with any changes. + minBucketChanges: 1, + minChangeRatio: 0, + compactChunksOnly: true, + logger: this.logger + }).compact(); + this.logger.info(`Compacted chunks after initial replication in ${(Date.now() - start) / 1000}s`); + return { buckets: compactedBuckets }; + } + protected createMongoParameterCompactor( checkpoint: InternalOpId, options: storage.CompactOptions diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index a91edc32f..504c0f61c 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -113,7 +113,7 @@ bucket_definitions: }); }); - test('populatePersistentChecksumCache', async () => { + test('compactInitialReplication', async () => { // Populate old replication stream const { factory } = await setup(); @@ -133,20 +133,20 @@ bucket_definitions: const { checkpoint } = await bucketStorage.getCheckpoint(); // Default is to small small numbers - should be a no-op - const result0 = await bucketStorage.populatePersistentChecksumCache({ + const result0 = await bucketStorage.compactInitialReplication({ maxOpId: checkpoint }); expect(result0.buckets).toEqual(0); // This should cache the checksums for the two buckets - const result1 = await bucketStorage.populatePersistentChecksumCache({ + const result1 = await bucketStorage.compactInitialReplication({ maxOpId: checkpoint, minBucketChanges: 1 }); expect(result1.buckets).toEqual(2); // This should be a no-op, as the checksums are already cached - const result2 = await bucketStorage.populatePersistentChecksumCache({ + const result2 = await bucketStorage.compactInitialReplication({ maxOpId: checkpoint, minBucketChanges: 1 }); @@ -1501,6 +1501,30 @@ bucket_definitions: ); } + test('initial compaction merges small chunks and refreshes bucket metadata', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); + await insertDocs(collection, [ + serializeBucketData(BUCKET, [makeOp(1, 'A', 'a', ctx, sourceTableId), makeOp(2, 'B', 'b', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(3, 'C', 'c', ctx, sourceTableId), makeOp(4, 'D', 'd', ctx, sourceTableId)]) + ]); + await insertBucketState(bucketStateCollection, ctx.definitionId, 4n); + + const result = await bucketStorage.compactInitialReplication({ maxOpId: 4n }); + + expect(result).toEqual({ buckets: 1 }); + const documents = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray(); + expect(documents).toHaveLength(1); + expect(documents[0].ops!.map((op) => op.o)).toEqual([1n, 2n, 3n, 4n]); + + const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(state?.compacted_state).toMatchObject({ + op_id: 4n, + count: 4, + checksum: 70n + }); + expect(state?.estimate_since_compact).toEqual({ count: 0, bytes: 0 }); + }); + test('1. multi-batch compaction preserves checksum and creates MOVE tombstones', async () => { const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); diff --git a/modules/module-mongodb/src/replication/MongoSnapshotter.ts b/modules/module-mongodb/src/replication/MongoSnapshotter.ts index ffe4fc248..2c898442b 100644 --- a/modules/module-mongodb/src/replication/MongoSnapshotter.ts +++ b/modules/module-mongodb/src/replication/MongoSnapshotter.ts @@ -345,10 +345,10 @@ export class MongoSnapshotter { return; } - // Populate the cache _after_ initial replication, but _before_ we switch to this replication stream. + // Compact storage _after_ initial replication, but _before_ we switch to this replication stream. // Keeping snapshot_done false until this completes makes this resumable after interruption. // No checkpoint exists yet - storage defaults to its highest persisted op id. - await this.storage.populatePersistentChecksumCache({ + await this.storage.compactInitialReplication({ signal: this.abortSignal }); diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index 3fed923ce..a3cf8d442 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -296,8 +296,8 @@ export class BinLogStream { } if (lastOp != null) { - // Populate the cache _after_ initial replication, but _before_ we switch to this replication stream. - await this.storage.populatePersistentChecksumCache({ + // Compact storage _after_ initial replication, but _before_ we switch to this replication stream. + await this.storage.compactInitialReplication({ // No checkpoint yet, but we do have the opId. maxOpId: lastOp, signal: this.abortSignal diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index fc078dba6..19303de07 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -4,6 +4,8 @@ import { BucketChecksum, CHECKPOINT_INVALIDATE_ALL, CheckpointChanges, + CompactInitialReplicationOptions, + CompactInitialReplicationResults, GetCheckpointChangesOptions, InternalOpId, internalToExternalOpId, @@ -11,8 +13,6 @@ import { maxLsn, ParameterSetLimitExceededError, PartialChecksum, - PopulateChecksumCacheOptions, - PopulateChecksumCacheResults, ReplicationCheckpoint, storage, StorageVersionConfig, @@ -160,7 +160,9 @@ export class PostgresSyncRulesStorage }).compact(); } - async populatePersistentChecksumCache(_options: PopulateChecksumCacheOptions): Promise { + async compactInitialReplication( + _options: CompactInitialReplicationOptions + ): Promise { // no-op - checksum cache is not implemented for Postgres yet return { buckets: 0 }; } diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index ce13dd9b1..391df874b 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -633,8 +633,8 @@ WHERE oid = $1::regclass`, const lastOp = flushResults?.flushed_op; if (lastOp != null) { - // Populate the cache _after_ initial replication, but _before_ we switch to this replication stream. - await this.storage.populatePersistentChecksumCache({ + // Compact storage _after_ initial replication, but _before_ we switch to this replication stream. + await this.storage.compactInitialReplication({ // No checkpoint yet, but we do have the opId. maxOpId: lastOp, signal: this.abort_signal diff --git a/packages/service-core/src/storage/SyncRulesBucketStorage.ts b/packages/service-core/src/storage/SyncRulesBucketStorage.ts index 49648dbe7..5b71c3d67 100644 --- a/packages/service-core/src/storage/SyncRulesBucketStorage.ts +++ b/packages/service-core/src/storage/SyncRulesBucketStorage.ts @@ -99,9 +99,9 @@ export interface SyncRulesBucketStorage compact(options?: CompactOptions): Promise; /** - * Lightweight "compact" process to populate the checksum cache, if any. + * Compact storage after initial replication, before the first checkpoint exists. */ - populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise; + compactInitialReplication(options: CompactInitialReplicationOptions): Promise; // ## Read operations @@ -358,12 +358,12 @@ export interface CompactOptions { logger?: Logger; } -export interface PopulateChecksumCacheOptions { +export interface CompactInitialReplicationOptions { /** - * Compute checksums up to this op id. + * Compact data up to this op id. * * Defaults to the highest persisted op id for the replication stream, which covers - * the common case of populating the cache right after initial replication, before + * the common case of compacting right after initial replication, before * the first checkpoint exists. */ maxOpId?: util.InternalOpId; @@ -371,9 +371,9 @@ export interface PopulateChecksumCacheOptions { signal?: AbortSignal; } -export interface PopulateChecksumCacheResults { +export interface CompactInitialReplicationResults { /** - * Number of buckets we have calculated checksums for. + * Number of buckets processed. */ buckets: number; } From 46b9f82f80c817056cf607702ca19080cadc9af0 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 12 Aug 2026 18:02:45 +0200 Subject: [PATCH 02/38] WIP: Redo v3 compact scheduling. --- docs/storage/v3-compaction-design.md | 132 ++++ .../storage/implementation/MongoCompactor.ts | 334 +--------- .../implementation/common/PersistedBatch.ts | 36 +- .../implementation/v1/MongoCompactorV1.ts | 270 +++++++- .../implementation/v3/CompactionLease.ts | 150 +++++ .../implementation/v3/MongoCompactorV3.ts | 623 +++++++++++++----- .../v3/MongoSyncBucketStorageV3.ts | 3 - .../implementation/v3/PersistedBatchV3.ts | 43 +- .../v3/VersionedPowerSyncMongoV3.ts | 9 +- .../src/storage/implementation/v3/models.ts | 49 +- .../test/src/storage_compacting.test.ts | 224 ++++--- .../test/src/storage_s3_checksums.test.ts | 3 +- .../storage_s3_compaction_lifecycle.test.ts | 8 +- .../src/storage/SyncRulesBucketStorage.ts | 12 + 14 files changed, 1259 insertions(+), 637 deletions(-) create mode 100644 docs/storage/v3-compaction-design.md create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts diff --git a/docs/storage/v3-compaction-design.md b/docs/storage/v3-compaction-design.md new file mode 100644 index 000000000..7813cc1ba --- /dev/null +++ b/docs/storage/v3-compaction-design.md @@ -0,0 +1,132 @@ +# V3 Compaction Design + +This document explains the decisions behind the V3 MongoDB bucket compactor. It accompanies [the compacting plan](v3-compact-plan.md); the plan states the intended behaviour, while this document explains how the implementation realizes it and the tradeoffs involved. + +## Goals + +V3 compaction is designed to make background work proportional to modified buckets and modified bucket data, rather than to the entire replication stream. It must support frequent execution, resumption after interruption, concurrent workers, and a bounded interval between full compactions. + +The design intentionally does not support the earlier V3 bucket-state format. V3 has not been deployed with that format, so accepting or repairing it would add a second state model and make the normal path less clear. + +## Bucket-state ownership + +`BucketStateDocumentV3` separates three kinds of information: + +1. `bucket_stats` is the current aggregate state of persisted bucket chunks. +2. `compacted_state` is a cache for the prefix covered by the latest lite or full compact. +3. `last_full_compact` records the last full compaction for scheduling heuristics. + +Replication writers own `last_op` and `bucket_stats`. Compaction owns the compact caches and scheduling state. This division avoids making writers calculate compaction-specific estimates. + +Each writer flush atomically: + +- advances `last_op`; +- adds operation count, exact serialized chunk bytes, and chunk count to `bucket_stats`; +- sets `first_uncompacted_write` only if it was absent; and +- sets `next_compact_check` to the earlier of its existing value and the requested lite-check time. + +The byte counter uses persisted serialized chunk sizes, rather than an operation-size estimate. Full and chunk compaction also derive their byte totals from persisted chunk metadata, so both sides of the compaction delta use the same unit. + +`first_uncompacted_write` is the oldest change which has not received a full compact. It is deliberately independent of `compacted_state`: chunk compaction can reduce chunk fragmentation without declaring changes fully compacted. + +## Scheduling work + +The `next_compact_check` partial index is the work queue. Scheduled workers select only state rows whose check time is due. This replaces scanning state rows for dirty-count estimates. + +A compaction run captures a server-time job start. It claims only rows whose check time was due at or before that start. Consequently, a long “run until completion” job does not continually rediscover work that it scheduled itself during the same run. A later job processes that work. + +Claiming one bucket is an atomic `findOneAndUpdate` which requires either no lease or an expired lease. It writes a worker id and server-time expiry. A claim is therefore both a unit of work distribution and the snapshot boundary for the bucket: + +- `S` is the server time when the lease was acquired; +- `L` is the bucket `last_op` returned by that atomic claim; and +- the compactor records `C`, the greatest operation it actually covered. + +Those values are carried through the compaction call chain as a per-bucket compaction context. A dedicated `CompactionLease` owns the atomic claim result, server-time renewal, owner-fenced finalization/rescheduling, and release. Each claim is immediately bound with `await using`, so every path following a successful claim releases it when the scope ends. It is not mutable compactor-instance state, which keeps the lease, op cap, snapshot, finalization, and renewal logic explicitly tied to the claim that created them. + +If a claimed bucket is not yet eligible for either kind of compact, it is rescheduled atomically and its lease is removed. If no chunks exist beyond `compacted_state`, there is no possible chunk-compaction work, so it is scheduled directly for its calculated full-compact check rather than polled at the chunk-compaction interval. + +## Choosing full versus chunk compaction + +Full compaction is chosen when either condition holds: + +1. `first_uncompacted_write` has reached the maximum full-compaction age. +2. The elapsed age multiplied by the uncompacted operation ratio reaches the minimum full-compaction interval. + +The sliding rule prevents an isolated small write from causing a large full compact immediately, while allowing a sufficiently large update burst to compact before the maximum interval. + +Chunk compaction is chosen when new chunks exist after `compacted_state` and the chunk-compaction interval for that bucket has elapsed. It is intentionally a chunk-layout and checksum-cache operation, not a full logical rewrite. + +The relevant intervals are options with V3 defaults: + +| Setting | Default | Purpose | +| --------------------------- | ---------: | ------------------------------------------------ | +| `minCompactChunkIntervalMs` | 5 minutes | Avoid frequent tiny tail checks. | +| `minCompactFullIntervalMs` | 2 hours | Controls sliding-scale full-compaction pressure. | +| `maxCompactFullIntervalMs` | 7 days | Bounds data retention / full-compaction age. | +| `compactLeaseDurationMs` | 10 minutes | Lets another worker recover abandoned work. | + +These are operational policy choices rather than correctness constants. They should be tuned from workload and backlog metrics. + +When a bucket has no lite work, its full check is scheduled one minute after the calculated eligibility time. This small late margin prevents a worker using a slightly earlier clock from waking before the exact full-compaction condition is true and repeatedly rescheduling the same bucket. It may delay a full compact by up to that margin. + +## Chunk compaction + +Chunk compaction reads the previous compacted boundary chunk and any later chunks. Including that one previous chunk matters because it may now merge with newly appended data. It does not rescan the older compacted prefix. + +The chunk-compaction metadata scan reads chunk fields only. Chunk payloads are hydrated only when a group can actually merge. The checksum cache for the unchanged prefix is seeded from `compacted_state`; the compactor combines that cache with metadata from the processed tail rather than recalculating the whole bucket checksum. + +`compactChunksOnly` forces this operation. `compactInitialReplication()` reuses scheduled selection, extending the fixed job-start boundary by one chunk-compaction interval so it includes writes that existed when the pass began, and forces the selected work to chunk rather than full compaction. The fixed boundary means it does not chase writes that arrive while it runs. Scheduled chunk compactions running during initial replication therefore reduce the later pass to the remaining scheduled work. The lease normally prevents the two paths from duplicating work for a bucket; owner-fenced finalization and concurrent-safe operations remain necessary if a lease expires. + +## Full compaction + +Full compaction scans bucket data backwards through the claimed upper boundary, rewrites superseded operations to MOVE operations where appropriate, performs CLEAR reduction, and may merge resulting chunks. It is bounded by the claimed head and any caller-specified `maxOpId` safe buffer. + +The caller cap is never widened: the compactor uses the lower of the claimed `last_op` and the requested maximum. A full compact which cannot cover all operations present at claim time is therefore treated as partial; it must not clear the bucket’s full-compaction debt. + +## Aggregate-statistics delta + +Compaction must not hold a long transaction on `bucket_state`, because replication writes need to continue. Instead it applies the stat correction at finalization. + +At compaction start, let `A` be the aggregate stats for the data being compacted. Let `B` be the corresponding stats after compaction. Replication may concurrently add new chunks, but the final update atomically increments current aggregate state by `B - A`. + +This has the desired result whether or not replication wrote during the compact: + +- with no concurrent writes, current stats become `B`; +- with concurrent writes, their increments remain and the compacted region is corrected from `A` to `B`. + +Chunk compaction calculates `A` and `B` only for its compacted tail plus its single overlap chunk. Full compaction calculates them only for the compacted prefix. Neither needs to rescan a tail beyond `C` merely to update aggregate counters. + +## Finalization and concurrent writes + +Finalization is one lease-fenced update. It updates cache/stat state, decides scheduling state using the current `last_op` in the same update, and removes the lease. + +For a full compact where `C >= L`: + +- if current `last_op == L`, there were no concurrent writes, so `first_uncompacted_write` and `next_compact_check` are cleared; +- if current `last_op > L`, writes arrived after `S`, so `first_uncompacted_write` becomes `S` and the next check is `S + minCompactChunkInterval`. + +Using `S` can schedule slightly earlier than the actual write time, but it cannot postpone the maximum full-compaction deadline. + +For a partial full compact (`C < L`) and for every chunk compact, the original `first_uncompacted_write` is retained. Replacing it with `S` would delay work that was already waiting for a full compact. + +## Lease renewal and failure + +Long compactions renew the lease with server time. Both renewal and finalization require the same worker id. A worker that loses its lease raises a non-retryable lease-loss error: retrying the same compactor instance would still hold a stale lease identity and could race the new owner. + +Transactional replacement conflicts are different. They are retryable because a retry starts the relevant bucket work again, while the lease still belongs to that worker. + +## Legacy separation + +`MongoCompactorV1` retains V1’s dirty-estimate model, its minimum-change settings, and its legacy checksum-update logic. `MongoCompactorV3` does not inherit those paths: + +- V1 owns the legacy `compact()` flow and dirty-bucket discovery. +- V3 owns its own `compact()` flow, scheduled selection, leases, and initial-replication lite path. +- The shared base provides only common configuration, retry handling, and checkpoint-request cleanup. + +This separation prevents a V3 change from accidentally using an estimate-based fallback and makes the intended state model visible at the class boundary. + +## Deliberate non-goals + +- No read, conversion, or repair path exists for an older V3 bucket-state document. +- V3 bucket-data compaction does not use `estimate_since_compact`, `minBucketChanges`, `minChangeRatio`, or dirty-bucket scanning. +- Parameter compaction is separate from V3 bucket-data compaction and currently has no per-collection lease. The V3 compaction lease applies only to bucket-data work. diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index 519f109aa..79d222e06 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -1,55 +1,14 @@ import * as timers from 'node:timers/promises'; -import { isMongoServerError, mongo, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb'; -import { - logger as defaultLogger, - Logger, - ReplicationAssertionError, - ServiceAssertionError -} from '@powersync/lib-services-framework'; -import { InternalOpId, isPartialChecksum, storage } from '@powersync/service-core'; -import { BucketDefinitionId } from '@powersync/service-sync-rules'; +import { isMongoServerError, mongo } from '@powersync/lib-service-mongodb'; +import { logger as defaultLogger, Logger, ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; import { BucketKey } from './common/BucketDataDoc.js'; import type { VersionedPowerSyncMongo } from './db.js'; -import { BucketStateDocumentBase } from './models.js'; import type { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js'; import { isRetryableObjectStorageError } from './v3/object-storage/ObjectStorage.js'; -export interface CurrentBucketState { - /** Bucket name */ - bucket: string; - definitionId: BucketDefinitionId; - /** - * Rows seen in the bucket, with the last op_id of each. - */ - seen: Map; - /** - * Estimated memory usage of the seen Map. - */ - trackingSize: number; - /** - * Last (lowest) seen op_id that is not a PUT. - */ - lastNotPut: InternalOpId | null; - /** - * Number of REMOVE/MOVE operations seen since lastNotPut. - */ - opsSincePut: number; - /** - * Incrementally-updated checksum, up to maxOpId. - */ - checksum: number; - /** - * Op count for the checksum. - */ - opCount: number; - /** - * Byte size of ops covered by the checksum. - */ - opBytes: number; -} - export interface MongoCompactOptions extends storage.CompactOptions { /** * Only merge adjacent V3 bucket-data chunks. This is used after initial @@ -63,9 +22,6 @@ const DEFAULT_CLEAR_BATCH_LIMIT = 5000; const DEFAULT_MOVE_BATCH_LIMIT = 2000; const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000; const DEFAULT_MOVE_BATCH_BYTE_LIMIT = 16 * 1024 * 1024; -const DEFAULT_MIN_BUCKET_CHANGES = 10; -const DEFAULT_MIN_CHANGE_RATIO = 0.1; -const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000; /** This default is primarily for tests. */ const DEFAULT_MEMORY_LIMIT_MB = 64; const COMPACTION_RETRY_LIMIT = 3; @@ -78,23 +34,24 @@ export class ConcurrentCompactionError extends Error { } } -export interface DirtyBucket { - bucket: string; - definitionId: BucketDefinitionId | null; - estimatedCount: number; - dirtyRatio?: number; +/** + * A worker whose bucket lease has been replaced must stop immediately. Unlike + * a transactional replacement conflict, retrying it with the same compactor + * instance would still use the stale lease and could race the new owner. + */ +export class CompactionLeaseLostError extends ConcurrentCompactionError { + constructor(message: string) { + super(message); + this.name = 'CompactionLeaseLostError'; + } } export abstract class MongoCompactor { - protected bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; - protected readonly idLimitBytes: number; protected readonly moveBatchLimit: number; protected readonly moveBatchQueryLimit: number; protected readonly moveBatchByteLimit: number; protected readonly clearBatchLimit: number; - protected readonly minBucketChanges: number; - protected readonly minChangeRatio: number; protected readonly maxOpId: bigint; protected readonly buckets: string[] | undefined; protected readonly deleteCheckpointRequestsBefore: Date | undefined; @@ -119,8 +76,6 @@ export abstract class MongoCompactor { if (this.clearBatchLimit < 2) { throw new ReplicationAssertionError('clearBatchLimit must be >= 2'); } - this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES; - this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO; this.maxOpId = options.maxOpId ?? 0n; this.buckets = options.compactBuckets; this.deleteCheckpointRequestsBefore = options.deleteCheckpointRequestsBefore; @@ -129,28 +84,9 @@ export abstract class MongoCompactor { this.logger = options.logger ?? defaultLogger; } - /** - * Compact buckets by converting operations into MOVE and/or CLEAR operations. - * - * See /docs/storage/compacting-operations.md for details. - */ - async compact(): Promise { - await this.deleteOldCheckpointRequests(); - - if (this.buckets) { - for (const bucket of this.buckets) { - // We can make this more efficient later on by iterating through the buckets in a single query. - // That makes batching more tricky, so we leave for later. - await this.compactSingleBucketRetried(bucket); - } - } else { - await this.compactDirtyBuckets(); - } + abstract compact(): Promise; - return this.compactedBucketCount; - } - - private async deleteOldCheckpointRequests() { + protected async deleteOldCheckpointRequests() { if (this.deleteCheckpointRequestsBefore == null) { return; } @@ -169,142 +105,6 @@ export abstract class MongoCompactor { }); } - protected async *dirtyBucketBatchesForCollection( - collection: mongo.Collection, - lastId: TCollectionBucketState['_id'], - maxId: TCollectionBucketState['_id'], - options: { - minBucketChanges: number; - minChangeRatio: number; - }, - getDefinitionId: (state: TCollectionBucketState) => BucketDefinitionId | null - ): AsyncGenerator { - // Paginate through the bucket state collection using cursor-based scanning. - while (true) { - // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, use an aggregation pipeline - // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria. - const [result] = await collection - .aggregate<{ - buckets: TCollectionBucketState[]; - cursor: Pick[]; - }>( - [ - { - $match: { - _id: { $gt: lastId, $lt: maxId } - } - }, - { - $sort: { _id: 1 } - }, - { - // Scan a fixed number of docs each query so sparse matches don't block progress. - $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE - }, - { - $facet: { - buckets: [ - { - $match: { - 'estimate_since_compact.count': { $gte: options.minBucketChanges } - } - }, - { - $project: { - _id: 1, - estimate_since_compact: 1, - compacted_state: 1 - } - } - ], - // This is used for the next query. - cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }] - } - } - ], - { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } - ) - .toArray(); - - const cursor = result?.cursor?.[0]; - if (cursor == null) { - break; - } - lastId = cursor._id; - - const mapped = (result?.buckets ?? []).map((bucketState) => { - // The numbers, specifically the bytes, could be a bigint. Convert to Number to allow calculating ratios. - // BigInt precision is not needed here since this is only an estimate. - const updatedCount = bucketState.estimate_since_compact?.count ?? 0; - const totalCount = (bucketState.compacted_state?.count ?? 0) + updatedCount; - const updatedBytes = Number(bucketState.estimate_since_compact?.bytes ?? 0); - const totalBytes = Number(bucketState.compacted_state?.bytes ?? 0) + updatedBytes; - const dirtyChangeNumber = totalCount > 0 ? updatedCount / totalCount : 0; - const dirtyChangeBytes = totalBytes > 0 ? updatedBytes / totalBytes : 0; - return { - bucket: bucketState._id.b, - definitionId: getDefinitionId(bucketState), - estimatedCount: totalCount, - dirtyRatio: Math.max(dirtyChangeNumber, dirtyChangeBytes) - }; - }); - - yield mapped.filter( - (bucket) => bucket.estimatedCount >= options.minBucketChanges && bucket.dirtyRatio >= options.minChangeRatio - ); - } - } - - protected async dirtyBucketBatchForChecksumsForCollection( - collection: mongo.Collection, - filter: mongo.Filter, - getDefinitionId: (state: mongo.WithId) => BucketDefinitionId | null - ): Promise { - const dirtyBuckets = await collection - .find(filter, { - projection: { - _id: 1, - estimate_since_compact: 1, - compacted_state: 1 - }, - sort: { - 'estimate_since_compact.count': -1 - }, - limit: 200, - maxTimeMS: MONGO_OPERATION_TIMEOUT_MS - }) - .toArray(); - - return dirtyBuckets.map((bucket) => ({ - bucket: bucket._id.b, - definitionId: getDefinitionId(bucket), - estimatedCount: Number(bucket.estimate_since_compact!.count) + Number(bucket.compacted_state?.count ?? 0) - })); - } - - public abstract dirtyBucketBatches(options: { - minBucketChanges: number; - minChangeRatio: number; - }): AsyncGenerator; - - public abstract dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise; - - protected async compactDirtyBuckets() { - for await (const buckets of this.dirtyBucketBatches({ - minBucketChanges: this.minBucketChanges, - minChangeRatio: this.minChangeRatio - })) { - this.signal?.throwIfAborted(); - if (buckets.length == 0) { - continue; - } - - for (const { bucket, definitionId } of buckets) { - await this.compactSingleBucketRetried(bucket, definitionId); - } - } - } - /** * Compaction for a single bucket, with retries on failure. * @@ -313,14 +113,12 @@ export abstract class MongoCompactor { * storage paths are also prepared with lifecycle markers, so a retry can * safely overwrite or eventually clean up uploads from the failed attempt. */ - protected async compactSingleBucketRetried(bucket: string, definitionId: BucketDefinitionId | null = null) { + protected async retryCompaction(bucket: string, compact: () => Promise) { let retryCount = 0; while (true) { this.signal?.throwIfAborted(); - // Do not carry queued bucket state writes from a failed attempt into the rescan. - this.bucketStateUpdates = []; try { - await this.compactSingleBucket(bucket, definitionId); + await compact(); return; } catch (e) { if (this.signal?.aborted) { @@ -349,101 +147,6 @@ export abstract class MongoCompactor { } } } - - protected abstract compactSingleBucket(bucket: string, definitionId?: BucketDefinitionId | null): Promise; - - protected collectBucketStateUpdates( - state: CurrentBucketState, - compactedOpId: InternalOpId - ): mongo.AnyBulkWriteOperation { - if (state.opCount < 0) { - throw new ServiceAssertionError( - `Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}` - ); - } - return { - updateOne: { - filter: this.bucketStateFilter(state.bucket, state.definitionId), - update: { - $set: { - compacted_state: { - op_id: compactedOpId, - count: state.opCount, - checksum: BigInt(state.checksum), - bytes: state.opBytes - }, - estimate_since_compact: { - // There could have been a whole bunch of new operations added to the bucket while compacting, - // which we don't currently cater for. We could potentially query for that, but that adds overhead. - count: 0, - bytes: 0 - } - } - } satisfies mongo.UpdateFilter, - // We generally expect this to have been created before. - // We don't create new ones here, to avoid issues with the unique index on bucket_updates. - upsert: false - } - }; - } - - protected updateBucketChecksums(state: CurrentBucketState, compactedOpId: InternalOpId) { - this.bucketStateUpdates.push(this.collectBucketStateUpdates(state, compactedOpId)); - } - - protected async flushBucketStateUpdates() { - if (this.bucketStateUpdates.length > 0) { - this.logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`); - await this.writeBucketStateUpdates(); - this.bucketStateUpdates = []; - } - } - - protected async updateChecksumsBatch(buckets: Pick[]) { - const checksums = await this.computeChecksumsForBuckets(buckets); - const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId])); - - for (const bucketChecksum of checksums.values()) { - if (isPartialChecksum(bucketChecksum)) { - // Should never happen since we don't specify `start`. - throw new ServiceAssertionError(`Full checksum expected, got ${JSON.stringify(bucketChecksum)}`); - } - - this.bucketStateUpdates.push({ - updateOne: { - filter: this.bucketStateFilter( - bucketChecksum.bucket, - definitionIdByBucket.get(bucketChecksum.bucket) ?? null - ), - update: { - $set: { - compacted_state: { - op_id: this.maxOpId, - count: bucketChecksum.count, - checksum: BigInt(bucketChecksum.checksum), - bytes: null - }, - estimate_since_compact: { - count: 0, - bytes: 0 - } - } - } satisfies mongo.UpdateFilter, - // We don't create new ones here - it gets tricky to get the last_op right with the unique index on - // bucket_updates. - upsert: false - } - }); - } - - await this.flushBucketStateUpdates(); - } - - protected abstract writeBucketStateUpdates(): Promise; - protected abstract computeChecksumsForBuckets( - buckets: Pick[] - ): Promise; - protected abstract bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document; } export interface BucketDataCollectionContext { @@ -452,6 +155,9 @@ export interface BucketDataCollectionContext } function compactionRetryReason(error: unknown): string | null { + if (error instanceof CompactionLeaseLostError) { + return null; + } if (error instanceof ConcurrentCompactionError) { return 'concurrent compaction'; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts index 8a1df88c4..13184d048 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts @@ -234,11 +234,44 @@ export abstract class PersistedBatch { bucket, lastOp: op_id, incrementCount: 1, - incrementBytes: bytes + incrementBytes: bytes, + incrementChunks: 0 }); } } + /** + * V3 persists operations in chunks. Keep this separate from incrementBucket: + * operation counts are known while evaluating rows, while chunk counts are + * only known after the writer has chunked a flush. + */ + protected incrementBucketChunks(definitionId: BucketDefinitionId, bucket: string, chunks = 1) { + const key = `${definitionId ?? ''}:${bucket}`; + const existingState = this.bucketStates.get(key); + if (existingState != null) { + existingState.incrementChunks += chunks; + } + } + + /** + * V3's compactor calculates byte deltas from persisted chunk metadata. + * Replace the writer's per-operation estimate with those exact sizes before + * flushing the corresponding bucket state. + */ + protected resetBucketPersistedBytes(definitionId: BucketDefinitionId, bucket: string) { + const state = this.bucketStates.get(`${definitionId ?? ''}:${bucket}`); + if (state != null) { + state.incrementBytes = 0; + } + } + + protected incrementBucketPersistedBytes(definitionId: BucketDefinitionId, bucket: string, bytes: number) { + const state = this.bucketStates.get(`${definitionId ?? ''}:${bucket}`); + if (state != null) { + state.incrementBytes += bytes; + } + } + protected addBucketDataPut(options: { op_id: InternalOpId; bucketKey: BucketKey; @@ -377,4 +410,5 @@ export interface BucketStateUpdate { lastOp: InternalOpId; incrementCount: number; incrementBytes: number; + incrementChunks: number; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index 9a3be6eac..84117913c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -1,10 +1,17 @@ -import { mongo } from '@powersync/lib-service-mongodb'; -import { ReplicationAssertionError } from '@powersync/lib-services-framework'; -import { addChecksums, CompactInitialReplicationResults, storage, utils } from '@powersync/service-core'; +import { mongo, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb'; +import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; +import { + addChecksums, + CompactInitialReplicationResults, + InternalOpId, + isPartialChecksum, + storage, + utils +} from '@powersync/service-core'; import { BucketDefinitionId } from '@powersync/service-sync-rules'; import { BucketDataDoc } from '../common/BucketDataDoc.js'; import { BucketStateDocumentBase, LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js'; -import { CurrentBucketState, DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; +import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; import { cacheKey } from '../OperationBatch.js'; import { BucketDataDocumentV1, BucketStateDocumentV1 } from './models.js'; import type { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js'; @@ -13,12 +20,65 @@ import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; type CompactClearProperties = 'op' | 'checksum' | 'target_op'; +const DEFAULT_MIN_BUCKET_CHANGES = 10; +const DEFAULT_MIN_CHANGE_RATIO = 0.1; +const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000; + +interface CurrentBucketState { + bucket: string; + definitionId: BucketDefinitionId; + seen: Map; + trackingSize: number; + lastNotPut: InternalOpId | null; + opsSincePut: number; + checksum: number; + opCount: number; + opBytes: number; +} + +interface DirtyBucket { + bucket: string; + definitionId: BucketDefinitionId | null; + estimatedCount: number; + dirtyRatio?: number; +} + export class MongoCompactorV1 extends MongoCompactor { // Override types to the more specific ones declare protected readonly db: VersionedPowerSyncMongoV1; declare protected readonly storage: MongoSyncBucketStorageV1; private updates: mongo.AnyBulkWriteOperation[] = []; + private bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; + private readonly minBucketChanges: number; + private readonly minChangeRatio: number; + + constructor(bucketStorage: MongoSyncBucketStorageV1, db: VersionedPowerSyncMongoV1, options: MongoCompactOptions) { + super(bucketStorage, db, options); + this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES; + this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO; + } + + /** + * Compact buckets by converting operations into MOVE and/or CLEAR operations. + * + * See /docs/storage/compacting-operations.md for details. + */ + override async compact(): Promise { + await this.deleteOldCheckpointRequests(); + + if (this.buckets) { + for (const bucket of this.buckets) { + // We can make this more efficient later on by iterating through the buckets in a single query. + // That makes batching more tricky, so we leave for later. + await this.compactSingleBucketRetried(bucket); + } + } else { + await this.compactDirtyBuckets(); + } + + return this.compactedBucketCount; + } public async *dirtyBucketBatches(options: { minBucketChanges: number; @@ -34,8 +94,7 @@ export class MongoCompactorV1 extends MongoCompactor { this.db.bucketStateV1, { g: this.group_id, b: new mongo.MinKey() as any }, { g: this.group_id, b: new mongo.MaxKey() as any }, - options, - () => null + options ); } @@ -45,14 +104,105 @@ export class MongoCompactorV1 extends MongoCompactor { } // Unlike dirtyBucketBatches, this path is resumable after restart because populateChecksums resets // estimate_since_compact as it progresses. - return this.dirtyBucketBatchForChecksumsForCollection( - this.db.bucketStateV1, - { - '_id.g': this.group_id, - 'estimate_since_compact.count': { $gte: options.minBucketChanges } - }, - () => null - ); + return this.dirtyBucketBatchForChecksumsForCollection({ + '_id.g': this.group_id, + 'estimate_since_compact.count': { $gte: options.minBucketChanges } + }); + } + + private async *dirtyBucketBatchesForCollection( + collection: mongo.Collection, + lastId: TBucketState['_id'], + maxId: TBucketState['_id'], + options: { minBucketChanges: number; minChangeRatio: number } + ): AsyncGenerator { + // Paginate through the bucket state collection using cursor-based scanning. + while (true) { + // To avoid timeouts from too many buckets not meeting the minimum-change + // criteria, scan a fixed batch and only return matching buckets. + const [result] = await collection + .aggregate<{ buckets: TBucketState[]; cursor: Pick[] }>( + [ + { $match: { _id: { $gt: lastId, $lt: maxId } } }, + { $sort: { _id: 1 } }, + // Scan a fixed number of docs each query so sparse matches don't block progress. + { $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE }, + { + $facet: { + buckets: [ + { $match: { 'estimate_since_compact.count': { $gte: options.minBucketChanges } } }, + { $project: { _id: 1, estimate_since_compact: 1, compacted_state: 1 } } + ], + cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }] + } + } + ], + { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } + ) + .toArray(); + + const cursor = result?.cursor?.[0]; + if (cursor == null) { + break; + } + lastId = cursor._id; + + const dirtyBuckets = (result?.buckets ?? []).map((bucketState) => { + // BigInt precision is not needed here since this is only an estimate. + const updatedCount = bucketState.estimate_since_compact?.count ?? 0; + const totalCount = (bucketState.compacted_state?.count ?? 0) + updatedCount; + const updatedBytes = Number(bucketState.estimate_since_compact?.bytes ?? 0); + const totalBytes = Number(bucketState.compacted_state?.bytes ?? 0) + updatedBytes; + return { + bucket: bucketState._id.b, + definitionId: null, + estimatedCount: totalCount, + dirtyRatio: Math.max( + totalCount > 0 ? updatedCount / totalCount : 0, + totalBytes > 0 ? updatedBytes / totalBytes : 0 + ) + }; + }); + + yield dirtyBuckets.filter( + (bucket) => bucket.estimatedCount >= options.minBucketChanges && bucket.dirtyRatio >= options.minChangeRatio + ); + } + } + + private async dirtyBucketBatchForChecksumsForCollection( + filter: mongo.Filter + ): Promise { + const dirtyBuckets = await this.db.bucketStateV1 + .find(filter, { + projection: { _id: 1, estimate_since_compact: 1, compacted_state: 1 }, + sort: { 'estimate_since_compact.count': -1 }, + limit: 200, + maxTimeMS: MONGO_OPERATION_TIMEOUT_MS + }) + .toArray(); + + return dirtyBuckets.map((bucket) => ({ + bucket: bucket._id.b, + definitionId: null, + estimatedCount: Number(bucket.estimate_since_compact!.count) + Number(bucket.compacted_state?.count ?? 0) + })); + } + + private async compactSingleBucketRetried(bucket: string, _definitionId: BucketDefinitionId | null = null) { + await this.retryCompaction(bucket, () => this.compactSingleBucket(bucket)); + } + + private async compactDirtyBuckets() { + for await (const buckets of this.dirtyBucketBatches({ + minBucketChanges: this.minBucketChanges, + minChangeRatio: this.minChangeRatio + })) { + this.signal?.throwIfAborted(); + for (const { bucket, definitionId } of buckets) { + await this.compactSingleBucketRetried(bucket, definitionId); + } + } } /** @@ -90,6 +240,90 @@ export class MongoCompactorV1 extends MongoCompactor { return { buckets: count }; } + private collectBucketStateUpdate( + state: CurrentBucketState, + compactedOpId: bigint + ): mongo.AnyBulkWriteOperation { + if (state.opCount < 0) { + throw new ServiceAssertionError( + `Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}` + ); + } + return { + updateOne: { + filter: this.bucketStateFilter(state.bucket), + update: { + $set: { + compacted_state: { + op_id: compactedOpId, + count: state.opCount, + checksum: BigInt(state.checksum), + bytes: state.opBytes + }, + estimate_since_compact: { + // There could have been a whole bunch of new operations added to the bucket while compacting, + // which we don't currently cater for. We could potentially query for that, but that adds overhead. + count: 0, + bytes: 0 + } + } + } satisfies mongo.UpdateFilter, + // We generally expect this to have been created before. + // We don't create new ones here, to avoid issues with the unique index on bucket_updates. + upsert: false + } + }; + } + + private updateBucketChecksums(state: CurrentBucketState, compactedOpId: bigint) { + this.bucketStateUpdates.push(this.collectBucketStateUpdate(state, compactedOpId)); + } + + private async flushBucketStateUpdates() { + if (this.bucketStateUpdates.length > 0) { + this.logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`); + await this.writeBucketStateUpdates(); + this.bucketStateUpdates = []; + } + } + + private async updateChecksumsBatch(buckets: Pick[]) { + const checksums = await this.computeChecksumsForBuckets(buckets); + const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId])); + + for (const bucketChecksum of checksums.values()) { + if (isPartialChecksum(bucketChecksum)) { + // Should never happen since we don't specify `start`. + throw new ServiceAssertionError(`Full checksum expected, got ${JSON.stringify(bucketChecksum)}`); + } + + this.bucketStateUpdates.push({ + updateOne: { + filter: this.bucketStateFilter(bucketChecksum.bucket), + update: { + $set: { + compacted_state: { + op_id: this.maxOpId, + count: bucketChecksum.count, + checksum: BigInt(bucketChecksum.checksum), + bytes: null + }, + estimate_since_compact: { + count: 0, + bytes: 0 + } + } + } satisfies mongo.UpdateFilter, + // We don't create new ones here - it gets tricky to get the last_op right with the unique index on + // bucket_updates. + upsert: false + } + }); + } + + await this.flushBucketStateUpdates(); + } + protected async writeBucketStateUpdates(): Promise { await this.db.bucketStateV1.bulkWrite( this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], @@ -109,10 +343,7 @@ export class MongoCompactorV1 extends MongoCompactor { ); } - protected bucketStateFilter( - bucket: string, - _definitionId: BucketDefinitionId | null - ): mongo.Filter { + private bucketStateFilter(bucket: string): mongo.Filter { return { _id: { g: this.group_id, @@ -130,8 +361,9 @@ export class MongoCompactorV1 extends MongoCompactor { } protected async compactSingleBucket(bucket: string) { - // Do not carry queued writes from a failed attempt into the rescan + // Do not carry queued writes from a failed attempt into the rescan. this.updates = []; + this.bucketStateUpdates = []; const idLimitBytes = this.idLimitBytes; const bucketContext = this.getBucketDataContext(bucket); diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts new file mode 100644 index 000000000..6a1094996 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts @@ -0,0 +1,150 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { InternalOpId } from '@powersync/service-core'; +import { CompactionLeaseLostError } from '../MongoCompactor.js'; +import { BucketStateDocumentV3 } from './models.js'; + +const LEASE_RENEW_INTERVAL_MS = 60 * 1000; + +/** + * Owns one V3 bucket-compaction lease, including its server-time renewal and + * the owner-fenced operations which release it. + * + * This is intended as a way to reduce redundant work if multiple jobs attempt to + * compact the same bucket concurrently, but it's not an absolute safety guarantee. + * The individual operations must still be designed to be safe with concurrent compacting. + */ +export class CompactionLease implements AsyncDisposable { + readonly startedAt: Date; + readonly lastOp: InternalOpId; + + private timer: NodeJS.Timeout | undefined; + private renewalInFlight = false; + private renewalError: unknown; + private finalizing = false; + private finished = false; + + private constructor( + private readonly collection: mongo.Collection, + readonly state: BucketStateDocumentV3, + readonly id: mongo.ObjectId, + private readonly durationMs: number + ) { + this.startedAt = new Date(state.compact_lease!.expires_at.getTime() - durationMs); + this.lastOp = state.last_op; + } + + static async claim( + collection: mongo.Collection, + filter: mongo.Filter, + sort: mongo.Sort | undefined, + durationMs: number + ): Promise { + const id = new mongo.ObjectId(); + const state = await collection.findOneAndUpdate( + { + $and: [ + filter, + { + // $$NOW is evaluated by MongoDB, avoiding lease expiry races due + // to clocks on separate compact workers. + $expr: { + $or: [{ $eq: [{ $type: '$compact_lease' }, 'missing'] }, { $lte: ['$compact_lease.expires_at', '$$NOW'] }] + } + } + ] + }, + [ + { + $set: { + compact_lease: { + id, + expires_at: { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: durationMs } } + } + } + } + ], + { sort, returnDocument: 'after' } + ); + return state == null ? null : new CompactionLease(collection, state, id, durationMs); + } + + startRenewal() { + const interval = Math.max(1, Math.min(LEASE_RENEW_INTERVAL_MS, Math.floor(this.durationMs / 2))); + this.timer = setInterval(() => { + if (this.renewalInFlight) { + return; + } + this.renewalInFlight = true; + void this.renew() + .catch((error) => { + this.renewalError = error; + }) + .finally(() => { + this.renewalInFlight = false; + }); + }, interval); + this.timer.unref(); + } + + async throwIfLost() { + if (this.renewalError != null) { + throw this.renewalError; + } + } + + /** Allow a retry after a transient error during a fenced final update. */ + restartFinalization() { + this.finalizing = false; + } + + async reschedule(nextCompactCheck: mongo.Document) { + await this.finish([{ $set: { next_compact_check: nextCompactCheck } }, { $unset: 'compact_lease' }]); + } + + async finalize(update: mongo.Document) { + await this.finish([{ $set: update }, { $unset: 'compact_lease' }]); + } + + async [Symbol.asyncDispose]() { + if (this.timer != null) { + clearInterval(this.timer); + this.timer = undefined; + } + if (this.finished) { + return; + } + this.finalizing = true; + await this.collection.updateOne(this.filter, { $unset: { compact_lease: '' } }); + this.finished = true; + } + + private get filter(): mongo.Filter { + return { _id: this.state._id, 'compact_lease.id': this.id }; + } + + private async renew() { + const result = await this.collection.updateOne(this.filter, [ + { + $set: { + 'compact_lease.expires_at': { + $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.durationMs } + } + } + } + ]); + // A finalization in progress owns the authoritative fenced result. + if (result.matchedCount != 1 && !this.finalizing) { + throw new CompactionLeaseLostError(`Lost compaction lease for bucket ${this.state._id.b}`); + } + } + + private async finish(update: mongo.Document[]) { + await this.throwIfLost(); + this.finalizing = true; + const result = await this.collection.updateOne(this.filter, update); + if (result.matchedCount != 1) { + throw new CompactionLeaseLostError(`Lost compaction lease for bucket ${this.state._id.b}`); + } + this.finished = true; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 2f56a7c48..2edc478c0 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -1,16 +1,16 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { logger, ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; +import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; import { addChecksums, InternalOpId, storage, utils } from '@powersync/service-core'; import { BucketDefinitionId } from '@powersync/service-sync-rules'; import { BucketDataDoc } from '../common/BucketDataDoc.js'; -import { BucketDataKey, BucketStateDocumentBase } from '../models.js'; -import { ConcurrentCompactionError, CurrentBucketState, DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; +import { BucketDataKey } from '../models.js'; +import { ConcurrentCompactionError, MongoCompactor } from '../MongoCompactor.js'; import { cacheKey } from '../OperationBatch.js'; import { loadBucketDataDocument, maxOpId, serializeBucketData } from './bucket-format.js'; import { BucketDataContextV3 } from './BucketDataContextV3.js'; import { DEFAULT_MAX_DOC_SIZE_BYTES } from './chunking.js'; +import { CompactionLease } from './CompactionLease.js'; import { BucketDataDocumentV3, BucketStateDocumentV3 } from './models.js'; -import { DefinitionChecksumOperations, MongoChecksumsV3 } from './MongoChecksumsV3.js'; import type { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js'; import { BucketDataObjectStorage, hydrateBucketDataDocuments } from './object-storage/BucketDataObjectStorage.js'; import { ObjectStorageLifecycle, PreparedObjectStorageUpload } from './object-storage/ObjectStorageLifecycle.js'; @@ -27,6 +27,61 @@ interface PendingCompactionGroup { targetOp: InternalOpId | null; } +enum CompactionKind { + Full = 'full', + Chunks = 'chunks' +} + +interface ScheduledCompactionOptions { + /** Process checks scheduled this far after the captured job start. */ + dueAheadMs?: number; + /** Used by initial replication, which must not run a full compact. */ + forceKind?: CompactionKind; +} + +class CompactionContext { + constructor( + readonly lease: CompactionLease, + readonly kind: CompactionKind + ) {} + + get state() { + return this.lease.state; + } + + get startedAt() { + return this.lease.startedAt; + } + + get lastOp() { + return this.lease.lastOp; + } +} + +interface BucketStats { + count: number; + bytes: bigint; + chunks: number; +} + +/** Bucket stats read from bucket-data documents, including their checksum. */ +interface BucketStatsWithChecksum extends BucketStats { + checksum: number; +} + +interface CompactionResult { + /** Metadata cached at compacted_state.op_id. */ + compactedState: BucketStatsWithChecksum; + /** Complete bucket metadata through the op head captured at claim time. */ + bucketStats: BucketStats; +} + +const DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS = 5 * 60 * 1000; +const DEFAULT_MIN_COMPACT_FULL_INTERVAL_MS = 2 * 60 * 60 * 1000; +const DEFAULT_MAX_COMPACT_FULL_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000; +const DEFAULT_COMPACT_LEASE_DURATION_MS = 10 * 60 * 1000; +const FULL_COMPACT_RESCHEDULE_MARGIN_MS = 60 * 1000; + /** * Read one bounded prefix from a compaction cursor. * @@ -65,10 +120,56 @@ async function readCompactionBatch( } } +function bucketStats(state: BucketStateDocumentV3): BucketStats { + return { + count: state.bucket_stats.count, + bytes: state.bucket_stats.bytes, + chunks: state.bucket_stats.chunks + }; +} + +function emptyBucketStats(): BucketStatsWithChecksum { + return { count: 0, bytes: 0n, chunks: 0, checksum: 0 }; +} + +function statsForDocument( + document: Pick +): BucketStatsWithChecksum { + return { + count: document.count, + bytes: BigInt(document.size), + chunks: 1, + checksum: addChecksums(0, Number(document.checksum)) + }; +} + +/** A scheduled bucket always has writes awaiting a full compact. */ +function firstUncompactedWrite(state: BucketStateDocumentV3): Date { + if (state.first_uncompacted_write == null) { + throw new ReplicationAssertionError(`Scheduled V3 bucket ${state._id.b} has no first uncompacted write`); + } + return state.first_uncompacted_write; +} + export class MongoCompactorV3 extends MongoCompactor { declare protected readonly db: VersionedPowerSyncMongoV3; declare protected readonly storage: MongoSyncBucketStorageV3; + private readonly minCompactChunkIntervalMs: number; + private readonly minCompactFullIntervalMs: number; + private readonly maxCompactFullIntervalMs: number; + private readonly compactLeaseDurationMs: number; + private readonly maxOpIdCap: InternalOpId | undefined; + + constructor(bucketStorage: MongoSyncBucketStorageV3, db: VersionedPowerSyncMongoV3, options: storage.CompactOptions) { + super(bucketStorage, db, options); + this.minCompactChunkIntervalMs = options.minCompactChunkIntervalMs ?? DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS; + this.minCompactFullIntervalMs = options.minCompactFullIntervalMs ?? DEFAULT_MIN_COMPACT_FULL_INTERVAL_MS; + this.maxCompactFullIntervalMs = options.maxCompactFullIntervalMs ?? DEFAULT_MAX_COMPACT_FULL_INTERVAL_MS; + this.compactLeaseDurationMs = options.compactLeaseDurationMs ?? DEFAULT_COMPACT_LEASE_DURATION_MS; + this.maxOpIdCap = options.maxOpId; + } + override async compact(): Promise { if (this.storage.objectStorage) { // Clean these before compacting - should be quick in most cases. @@ -79,75 +180,178 @@ export class MongoCompactorV3 extends MongoCompactor { this.logger.error(`Failed to clean up object storage deletion markers before compaction`, e); } } - const compactedBuckets = await super.compact(); + await this.deleteOldCheckpointRequests(); + + if (this.buckets != null) { + await this.compactExplicitBuckets(this.buckets); + } else if (this.compactChunksOnly) { + // Writers defer their first chunk-compaction check by minCompactChunkIntervalMs. + // Include that interval so this synchronous initial-replication pass + // processes the work that existed when it started. + await this.compactScheduledBuckets({ + dueAheadMs: this.minCompactChunkIntervalMs, + forceKind: CompactionKind.Chunks + }); + } else { + await this.compactScheduledBuckets(); + } if (this.storage.objectStorage) { // Cleanup for any produced during compacting. // Note that markers only expire after a delay, so this may skip many produced during this compact // run. However, during long compact runs, this may also have many ones it can clean up. await this.objectStorageLifecycle.cleanup(this.logger); } - return compactedBuckets; + return this.compactedBucketCount; } - private get objectStorageLifecycle(): ObjectStorageLifecycle { - if (!this.storage.objectStorage) { - throw new Error('Object storage is not configured'); + /** An explicit compact request always runs a full compact for its buckets. */ + private async compactExplicitBuckets(buckets: string[]) { + for (const bucket of buckets) { + // This is not a super efficient query, but this is not a common use case. + // May be optimized later. + const states = await this.db + .bucketState(this.group_id) + .find({ '_id.b': bucket }, { projection: { _id: 1 } }) + .toArray(); + for (const state of states) { + await using lease = await this.claimBucket({ _id: state._id }); + if (lease != null) { + await this.compactClaimedBucket(lease, CompactionKind.Full); + } + } } - return new ObjectStorageLifecycle(this.db, this.group_id, this.storage.objectStorage); } - public async *dirtyBucketBatches(options: { - minBucketChanges: number; - minChangeRatio: number; - }): AsyncGenerator { - if (options.minBucketChanges <= 0) { - throw new ReplicationAssertionError('minBucketChanges must be >= 1'); + /** + * Claim one scheduled bucket at a time. The fixed due boundary means a + * long-running job does not repeatedly process buckets it scheduled itself. + */ + private async compactScheduledBuckets(options: ScheduledCompactionOptions = {}) { + const jobStartedAt = await this.serverNow(); + const dueBefore = new Date(jobStartedAt.getTime() + (options.dueAheadMs ?? 0)); + const forceKind = options.forceKind; + while (true) { + this.signal?.throwIfAborted(); + await using lease = await this.claimBucket( + { next_compact_check: { $lte: dueBefore } }, + { next_compact_check: 1 } + ); + if (lease == null) { + break; + } + + const kind = forceKind ?? this.chooseCompactionKind(lease, lease.startedAt); + if (kind == null) { + await this.rescheduleClaimedBucket(lease); + } else { + await this.compactClaimedBucket(lease, kind); + } } - const collection = this.db.bucketState(this.group_id) as unknown as mongo.Collection; - yield* this.dirtyBucketBatchesForCollection( - collection, - { d: new mongo.MinKey(), b: new mongo.MinKey() } as unknown as BucketStateDocumentV3['_id'], - { d: new mongo.MaxKey(), b: new mongo.MaxKey() } as unknown as BucketStateDocumentV3['_id'], - options, - (bucketState) => (bucketState as BucketStateDocumentV3)._id.d - ); } - public async dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise { - if (options.minBucketChanges <= 0) { - throw new ReplicationAssertionError('minBucketChanges must be >= 1'); + private async claimBucket( + filter: mongo.Filter, + sort?: mongo.Sort + ): Promise { + return CompactionLease.claim(this.db.bucketState(this.group_id), filter, sort, this.compactLeaseDurationMs); + } + + private async compactClaimedBucket(lease: CompactionLease, kind: CompactionKind) { + const context = new CompactionContext(lease, kind); + lease.startRenewal(); + await this.retryCompaction(context.state._id.b, () => this.compactSingleBucket(context)); + } + + private chooseCompactionKind(lease: CompactionLease, now: Date): CompactionKind | null { + const state = lease.state; + if (now >= this.fullCompactionCheckAt(state)) { + return CompactionKind.Full; } - return this.dirtyBucketBatchForChecksumsForCollection( - this.db.bucketState(this.group_id) as unknown as mongo.Collection, - { - 'estimate_since_compact.count': { $gte: options.minBucketChanges } - } as unknown as mongo.Filter, - (bucketState) => (bucketState as BucketStateDocumentV3)._id.d - ); + + // For chunk compaction, we consider the number of chunks added. + // Right now, we trigger a compact if the interval has passed and at least 1 chunk was added. + // In the future me may use a threshold for bytes/chunk or count/chunk instead, and only compact + // if one of those are low. + const compacted = state.compacted_state; + const chunksSinceCompact = Math.max(0, state.bucket_stats.chunks - (compacted?.chunks ?? 0)); + const canCheckChunks = + compacted == null || now.getTime() - compacted.at.getTime() >= this.minCompactChunkIntervalMs; + if (canCheckChunks && chunksSinceCompact >= 1) { + return CompactionKind.Chunks; + } + return null; } - protected async writeBucketStateUpdates(): Promise { - await this.db - .bucketState(this.group_id) - .bulkWrite(this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], { - ordered: false - }); + private async rescheduleClaimedBucket(lease: CompactionLease) { + const state = lease.state; + const fullCheckAt = this.fullCompactionCheckAt(state); + // Schedule a little late so a worker using a slightly earlier clock does + // not wake before the exact full-compaction condition is true. + const fullCheckWithMargin = new Date(fullCheckAt.getTime() + FULL_COMPACT_RESCHEDULE_MARGIN_MS); + const chunksSinceCompact = Math.max(0, state.bucket_stats.chunks - (state.compacted_state?.chunks ?? 0)); + if (chunksSinceCompact == 0) { + // No new chunks can make chunk compaction eligible. Do not poll this + // bucket at the chunk-compaction interval; only wake it for its full-compact check. + await lease.reschedule(fullCheckWithMargin); + return; + } + + await lease.reschedule({ + $min: [ + fullCheckWithMargin, + { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.minCompactChunkIntervalMs } } + ] + }); } /** - * The compactor operates on persisted definition ids only - never on parsed sources. - * This narrowed view makes the source-resolving checksum methods unreachable here. + * Calculate the earliest full compaction time from the first uncompacted + * write, bounded by the maximum retention interval. */ - private get definitionChecksums(): DefinitionChecksumOperations { - return this.storage.checksums as MongoChecksumsV3; + private fullCompactionCheckAt(state: BucketStateDocumentV3): Date { + const firstWrite = firstUncompactedWrite(state); + const stats = bucketStats(state); + const lastFull = state.last_full_compact; + + // The number of operations since the last full compact. + // We may make this more specific in the future, to track new updates and deletes only, ignoring + // full new inserts, but that requires more granular tracking when replicating. + const uncompactedCount = lastFull == null ? stats.count : Math.max(0, stats.count - lastFull.count); + const compactedRows = lastFull?.puts ?? 0; + + // If no full compact has ever been performed: ratio = 1, compact after minCompactFullIntervalMs. + // If every row has been updated or deleted exactly once since the last full compact: ratio = 0.5, compact after minCompactFullIntervalMs * 2. + // If 10% of rows has been updated since the last full compact, compact after minCompactFullIntervalMs * 11. + // If every row has been updated multiple times, the ratio tends closer to 1 again. + const ratio = uncompactedCount == 0 ? 0 : uncompactedCount / (compactedRows + uncompactedCount); + const fullIntervalMs = ratio == 0 ? this.maxCompactFullIntervalMs : this.minCompactFullIntervalMs / ratio; + return new Date(firstWrite.getTime() + Math.min(fullIntervalMs, this.maxCompactFullIntervalMs)); + } + + private async serverNow(): Promise { + const result = await this.db.db.command({ hello: 1 }); + return result.localTime as Date; + } + + private compactMaxOpId(context: CompactionContext): InternalOpId { + return this.maxOpIdCap == null || context.lastOp < this.maxOpIdCap ? context.lastOp : this.maxOpIdCap; + } + + private get objectStorageLifecycle(): ObjectStorageLifecycle { + if (!this.storage.objectStorage) { + throw new Error('Object storage is not configured'); + } + return new ObjectStorageLifecycle(this.db, this.group_id, this.storage.objectStorage); } - protected override async compactSingleBucket(bucket: string, definitionId: BucketDefinitionId | null = null) { - if (this.compactChunksOnly) { - return this.compactSingleBucketChunks(bucket, definitionId); + private async compactSingleBucket(context: CompactionContext) { + // A retry restarts finalization after a transient replacement failure. + context.lease.restartFinalization(); + if (context.kind == CompactionKind.Chunks) { + return this.compactSingleBucketChunks(context); } - return this.compactSingleBucketFully(bucket, definitionId); + return this.compactSingleBucketFully(context); } /** @@ -156,27 +360,34 @@ export class MongoCompactorV3 extends MongoCompactor { * update the persisted checksum state and to decide whether a group can fit * in one chunk. */ - private async compactSingleBucketChunks(bucket: string, definitionId: BucketDefinitionId | null) { - const bucketContext = await this.getBucketDataContext(bucket, definitionId); - if (bucketContext == null) { - return; - } - - const resolvedDefinitionId = bucketContext.key.definitionId; + private async compactSingleBucketChunks(context: CompactionContext) { + const bucket = context.state._id.b; + const resolvedDefinitionId = context.state._id.d; + const bucketContext = new BucketDataContextV3(this.db, { + bucket, + definitionId: resolvedDefinitionId, + replicationStreamId: this.group_id + }); const collection = this.db.bucketData(this.group_id, resolvedDefinitionId); - const context = { replicationStreamId: this.group_id, definitionId: resolvedDefinitionId }; - let lowerBound = bucketContext.minId; - const upperBound = bucketContext.docId(this.maxOpId + 1n); + const dataContext = { replicationStreamId: this.group_id, definitionId: resolvedDefinitionId }; + // Include the last previously compacted chunk as well as new chunks. It + // is the only old chunk which can become mergeable with the new tail. + let lowerBound = + context.state.compacted_state?.op_id != null && context.state.compacted_state.op_id > 0n + ? bucketContext.docId(context.state.compacted_state.op_id - 1n) + : bucketContext.minId; + const upperBound = bucketContext.docId(this.compactMaxOpId(context) + 1n); let compactedOpId: bigint | null = null; - let totalChecksum = 0; - let totalOpCount = 0; - let totalOpBytes = 0; + let overlappingCompactedChunk: BucketStatsWithChecksum | undefined; + let preCompactionTail = emptyBucketStats(); + const tailLowerBound = lowerBound; let pendingChunks: BucketDataDocumentV3[] = []; let pendingSize = 0; while (true) { this.signal?.throwIfAborted(); + await context.lease.throwIfLost(); const batch = await readCompactionBatch( collection.aggregate( @@ -218,13 +429,15 @@ export class MongoCompactorV3 extends MongoCompactor { for (const doc of batch.documents) { compactedOpId = maxOpId(compactedOpId, doc._id.o); - totalChecksum = addChecksums(totalChecksum, Number(doc.checksum)); - totalOpCount += doc.count; - totalOpBytes += doc.size; + const documentStats = statsForDocument(doc); + preCompactionTail = this.combineAdjacentStats(preCompactionTail, documentStats); + if (context.state.compacted_state?.op_id === doc._id.o) { + overlappingCompactedChunk = documentStats; + } const nextSize = pendingSize + doc.size; if (pendingChunks.length > 0 && nextSize > DEFAULT_MAX_DOC_SIZE_BYTES) { - await this.flushChunkMerge(bucket, pendingChunks, collection, context, bucketContext); + await this.flushChunkMerge(bucket, pendingChunks, collection, dataContext, bucketContext); pendingChunks = []; pendingSize = 0; } @@ -240,27 +453,24 @@ export class MongoCompactorV3 extends MongoCompactor { } if (pendingChunks.length > 1) { - await this.flushChunkMerge(bucket, pendingChunks, collection, context, bucketContext); + await this.flushChunkMerge(bucket, pendingChunks, collection, dataContext, bucketContext); } if (compactedOpId == null) { + await this.finalizeSkippedBucket(context); return; } - await this.finalizeCompactedBucket( - { - bucket, - definitionId: resolvedDefinitionId, - lastNotPut: null, - opsSincePut: 0, - checksum: totalChecksum, - opCount: totalOpCount, - opBytes: totalOpBytes - }, - compactedOpId - ); + const tailStats = await this.readBucketStats(bucket, resolvedDefinitionId, compactedOpId, tailLowerBound); + const compactedStats = this.combineChunkStats(context.state, tailStats, overlappingCompactedChunk); + const result = { + compactedState: compactedStats, + bucketStats: this.applyCompactionDelta(bucketStats(context.state), preCompactionTail, tailStats) + }; + + await this.finalizeCompactedBucket(context, compactedOpId, result); this.compactedBucketCount++; - this.logger.info(`Lightly compacted bucket ${bucket}: ${totalOpCount} ops`); + this.logger.info(`Lightly compacted bucket ${bucket}: ${result.bucketStats.count} ops`); } private async flushChunkMerge( @@ -308,101 +518,172 @@ export class MongoCompactorV3 extends MongoCompactor { } private async finalizeCompactedBucket( - state: Pick< - CurrentBucketState, - 'bucket' | 'definitionId' | 'lastNotPut' | 'opsSincePut' | 'checksum' | 'opCount' | 'opBytes' - >, - compactedOpId: InternalOpId + context: CompactionContext, + compactedOpId: InternalOpId, + compactionResult: CompactionResult, + puts = 0 ) { - this.updateBucketChecksums( - { - ...state, - seen: new Map(), - trackingSize: 0 + await context.lease.throwIfLost(); + const startedStats = bucketStats(context.state); + const delta = { + count: compactionResult.bucketStats.count - startedStats.count, + bytes: compactionResult.bucketStats.bytes - startedStats.bytes, + chunks: compactionResult.bucketStats.chunks - startedStats.chunks + }; + const coveredStart = compactedOpId >= context.lastOp; + const concurrentWriteCheck = { $gt: ['$last_op', context.lastOp] }; + const nextAfterConcurrentWrite = new Date(context.startedAt.getTime() + this.minCompactChunkIntervalMs); + const nextCheckForUncompactedWork = { + $min: [ + new Date(firstUncompactedWrite(context.state).getTime() + this.maxCompactFullIntervalMs), + { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.minCompactChunkIntervalMs } } + ] + }; + const update: mongo.Document = { + compacted_state: { + op_id: compactedOpId, + checksum: BigInt(compactionResult.compactedState.checksum), + count: compactionResult.compactedState.count, + bytes: compactionResult.compactedState.bytes, + chunks: compactionResult.compactedState.chunks, + at: '$$NOW' }, - compactedOpId - ); - await this.flushBucketStateUpdates(); + bucket_stats: { + count: { $add: ['$bucket_stats.count', delta.count] }, + bytes: { $add: ['$bucket_stats.bytes', delta.bytes] }, + chunks: { $add: ['$bucket_stats.chunks', delta.chunks] } + }, + first_uncompacted_write: + context.kind == CompactionKind.Full && coveredStart + ? { $cond: [concurrentWriteCheck, context.startedAt, '$$REMOVE'] } + : '$first_uncompacted_write', + next_compact_check: + context.kind == CompactionKind.Full && coveredStart + ? { $cond: [concurrentWriteCheck, nextAfterConcurrentWrite, '$$REMOVE'] } + : nextCheckForUncompactedWork + }; + if (context.kind == CompactionKind.Full && coveredStart) { + update.last_full_compact = { + op_id: compactedOpId, + count: compactionResult.bucketStats.count, + puts, + at: '$$NOW' + }; + } + + await context.lease.finalize(update); } - protected async computeChecksumsForBuckets( - buckets: Pick[] - ): Promise { - return this.definitionChecksums.computePartialChecksumsDirectByDefinition( - buckets.map(({ bucket, definitionId }) => { - if (definitionId == null) { - throw new ServiceAssertionError(`Missing definitionId for bucket checksum update on bucket ${bucket}`); - } - return { - bucket, - definitionId, - end: this.maxOpId - }; - }) - ); + private async finalizeSkippedBucket(context: CompactionContext) { + await this.rescheduleClaimedBucket(context.lease); } - protected bucketStateFilter( + private async readBucketStats( bucket: string, - definitionId: BucketDefinitionId | null - ): mongo.Filter { - if (definitionId == null) { - throw new ServiceAssertionError(`Missing definitionId for V3 bucket state filter on bucket ${bucket}`); - } + definitionId: BucketDefinitionId, + maxOp: InternalOpId, + lowerBound?: BucketDataKey + ): Promise { + const context = new BucketDataContextV3(this.db, { + bucket, + definitionId, + replicationStreamId: this.group_id + }); + const [stats] = await this.db + .bucketData(this.group_id, definitionId) + .aggregate<{ count: number; bytes: number | bigint; chunks: number; checksum: bigint }>([ + { + $match: { + _id: + lowerBound == null + ? { $gte: context.minId, $lte: context.docId(maxOp) } + : { $gt: lowerBound, $lte: context.docId(maxOp) } + } + }, + { + $group: { + _id: null, + count: { $sum: '$count' }, + bytes: { $sum: '$size' }, + chunks: { $sum: 1 }, + checksum: { $sum: '$checksum' } + } + } + ]) + .toArray(); return { - _id: { - d: definitionId, - b: bucket - } + count: Number(stats?.count ?? 0), + bytes: BigInt(stats?.bytes ?? 0), + chunks: Number(stats?.chunks ?? 0), + checksum: + typeof stats?.checksum == 'bigint' + ? Number(BigInt.asIntN(32, stats.checksum)) + : addChecksums(0, Number(stats?.checksum ?? 0)) }; } - private async getBucketDataContext( - bucket: string, - definitionId: BucketDefinitionId | null - ): Promise { - let resolvedDefinitionId = definitionId; - - if (resolvedDefinitionId == null) { - const allDefinitionIds = this.storage.storageIds.bucketDefinitionIds; - if (allDefinitionIds.length > 0) { - const potentialIds = allDefinitionIds.map((id) => ({ d: id, b: bucket })); - const bucketState = await this.db.bucketState(this.group_id).findOne({ - _id: { $in: potentialIds } - }); - if (bucketState != null) { - resolvedDefinitionId = bucketState._id.d; - } - } + private combineChunkStats( + state: BucketStateDocumentV3, + compactedTail: BucketStatsWithChecksum, + overlappingCompactedChunk: BucketStatsWithChecksum | undefined + ): BucketStatsWithChecksum { + const previous = state.compacted_state; + if (previous == null) { + return compactedTail; } - - if (resolvedDefinitionId == null) { - return null; + if (overlappingCompactedChunk == null) { + throw new ReplicationAssertionError(`Missing previous compacted chunk for bucket ${state._id.b}`); } + return { + count: previous.count - overlappingCompactedChunk.count + compactedTail.count, + bytes: previous.bytes - overlappingCompactedChunk.bytes + compactedTail.bytes, + chunks: previous.chunks - 1 + compactedTail.chunks, + checksum: addChecksums( + addChecksums(Number(previous.checksum), -overlappingCompactedChunk.checksum), + compactedTail.checksum + ) + }; + } + + private combineAdjacentStats( + first: BucketStatsWithChecksum, + second: BucketStatsWithChecksum + ): BucketStatsWithChecksum { + return { + count: first.count + second.count, + bytes: first.bytes + second.bytes, + chunks: first.chunks + second.chunks, + checksum: addChecksums(first.checksum, second.checksum) + }; + } - return new BucketDataContextV3(this.db, { + private applyCompactionDelta( + total: BucketStats, + before: BucketStatsWithChecksum, + after: BucketStatsWithChecksum + ): BucketStats { + return { + count: total.count - before.count + after.count, + bytes: total.bytes - before.bytes + after.bytes, + chunks: total.chunks - before.chunks + after.chunks + }; + } + + private async compactSingleBucketFully(context: CompactionContext) { + const bucket = context.state._id.b; + const resolvedDefinitionId = context.state._id.d; + const bucketContext = new BucketDataContextV3(this.db, { bucket, definitionId: resolvedDefinitionId, replicationStreamId: this.group_id }); - } - - private async compactSingleBucketFully(bucket: string, definitionId: BucketDefinitionId | null = null) { - const bucketContext = await this.getBucketDataContext(bucket, definitionId); - if (bucketContext == null) { - return; - } - - const resolvedDefinitionId = bucketContext.key.definitionId; const collection = this.db.bucketData(this.group_id, resolvedDefinitionId); - const context = { replicationStreamId: this.group_id, definitionId: resolvedDefinitionId }; - + const dataContext = { replicationStreamId: this.group_id, definitionId: resolvedDefinitionId }; const lowerBound = bucketContext.minId; - let upperBound = bucketContext.docId(this.maxOpId + 1n); + let upperBound = bucketContext.docId(this.compactMaxOpId(context) + 1n); - let totalChecksum = 0; let totalOpCount = 0; - let totalOpBytes = 0; + let preCompactionPrefix = emptyBucketStats(); let lastNotPut: bigint | null = null; let opsSincePut = 0; @@ -410,11 +691,13 @@ export class MongoCompactorV3 extends MongoCompactor { let clearBoundary: { opId: bigint; documentId: BucketDataKey } | null = null; const seen = new Map(); let trackingSize = 0; + let putCount = 0; let pendingGroup: PendingCompactionGroup | null = null; // --- Read batch from MongoDB --- while (true) { this.signal?.throwIfAborted(); + await context.lease.throwIfLost(); const pipeline: mongo.Document[] = [ { @@ -464,7 +747,8 @@ export class MongoCompactorV3 extends MongoCompactor { // merging is useful, and writes each final object at most once. for (const doc of batchDocs) { compactedOpId ??= doc._id.o; - const originalOps = Array.from(loadBucketDataDocument(context, doc)); + preCompactionPrefix = this.combineAdjacentStats(preCompactionPrefix, statsForDocument(doc)); + const originalOps = Array.from(loadBucketDataDocument(dataContext, doc)); let changed = false; const compactedOps: BucketDataDoc[] = []; @@ -497,6 +781,7 @@ export class MongoCompactorV3 extends MongoCompactor { } compactedOps.push(op); if (op.op == 'PUT') { + putCount++; lastNotPut = null; opsSincePut = 0; } else { @@ -518,10 +803,6 @@ export class MongoCompactorV3 extends MongoCompactor { } compactedOps.reverse(); - for (const op of compactedOps) { - totalChecksum = addChecksums(totalChecksum, Number(op.checksum)); - totalOpBytes += op.data?.length ?? 0; - } totalOpCount += compactedOps.length; const candidate: PendingCompactionGroup = { @@ -545,7 +826,7 @@ export class MongoCompactorV3 extends MongoCompactor { }; } else { const flushedGroup = pendingGroup; - const documentId = await this.flushCompactionGroup(bucket, flushedGroup, bucketContext, context); + const documentId = await this.flushCompactionGroup(bucket, flushedGroup, bucketContext, dataContext); if ( lastNotPut != null && flushedGroup.ops[0].o <= lastNotPut && @@ -569,7 +850,7 @@ export class MongoCompactorV3 extends MongoCompactor { } if (pendingGroup != null) { - const documentId = await this.flushCompactionGroup(bucket, pendingGroup, bucketContext, context); + const documentId = await this.flushCompactionGroup(bucket, pendingGroup, bucketContext, dataContext); if ( lastNotPut != null && pendingGroup.ops[0].o <= lastNotPut && @@ -579,6 +860,7 @@ export class MongoCompactorV3 extends MongoCompactor { } } if (compactedOpId == null) { + await this.finalizeSkippedBucket(context); return; } @@ -593,25 +875,20 @@ export class MongoCompactorV3 extends MongoCompactor { clearBoundary.documentId, bucketContext, collection, - context + dataContext ); } + const compactedStats = await this.readBucketStats(bucket, resolvedDefinitionId, compactedOpId); + const result = { + compactedState: compactedStats, + bucketStats: this.applyCompactionDelta(bucketStats(context.state), preCompactionPrefix, compactedStats) + }; + // --- Finalize: update bucket checksums and state --- - await this.finalizeCompactedBucket( - { - bucket, - definitionId: resolvedDefinitionId, - lastNotPut, - opsSincePut, - checksum: totalChecksum, - opCount: totalOpCount, - opBytes: totalOpBytes - }, - compactedOpId - ); + await this.finalizeCompactedBucket(context, compactedOpId, result, putCount); - logger.info(`Compacted bucket ${bucket}: ${totalOpCount} surviving ops`); + this.logger.info(`Compacted bucket ${bucket}: ${totalOpCount} surviving ops`); } /** diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index b71f7761c..e74d5ef45 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -203,9 +203,6 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { const compactedBuckets = await this.createMongoCompactor({ ...options, maxOpId, - // A metadata-only scan is cheap, so include buckets with any changes. - minBucketChanges: 1, - minChangeRatio: 0, compactChunksOnly: true, logger: this.logger }).compact(); diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts index 563b65033..58f8dd7df 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -232,8 +232,11 @@ export class PersistedBatchV3 extends PersistedBatch { const createInserts: (() => Promise>)[] = []; for (const [bucket, ops] of operationsByBucket) { + this.resetBucketPersistedBytes(definitionId, bucket); for (const chunk of chunkBucketData(ops)) { const serialized = serializeBucketData(bucket, chunk); + this.incrementBucketChunks(definitionId, bucket); + this.incrementBucketPersistedBytes(definitionId, bucket, serialized.size); if (lifecycle == null || serialized.size <= this.inlineThresholdBytes) { createInserts.push(async () => ({ insertOne: { @@ -379,15 +382,39 @@ export class PersistedBatchV3 extends PersistedBatch { b: state.bucket } }, - update: { - $set: { - last_op: state.lastOp - }, - $inc: { - 'estimate_since_compact.count': state.incrementCount, - 'estimate_since_compact.bytes': state.incrementBytes + // A pipeline update makes initialisation and scheduling one atomic + // writer operation. In particular, a later write cannot move an + // already-due compact check into the future. + update: [ + { + $set: { + last_op: state.lastOp, + bucket_stats: { + count: { $add: [{ $ifNull: ['$bucket_stats.count', 0] }, state.incrementCount] }, + bytes: { $add: [{ $ifNull: ['$bucket_stats.bytes', 0n] }, BigInt(state.incrementBytes)] }, + chunks: { $add: [{ $ifNull: ['$bucket_stats.chunks', 0] }, state.incrementChunks] } + }, + first_uncompacted_write: { $ifNull: ['$first_uncompacted_write', '$$NOW'] }, + next_compact_check: { + $let: { + vars: { requested: { $dateAdd: { startDate: '$$NOW', unit: 'minute', amount: 5 } } }, + in: { + $cond: [ + { + $and: [ + { $ne: ['$next_compact_check', null] }, + { $lt: ['$next_compact_check', '$$requested'] } + ] + }, + '$next_compact_check', + '$$requested' + ] + } + } + } + } } - }, + ], upsert: true } } satisfies mongo.AnyBulkWriteOperation; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts index 895a0729b..544422f1d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts @@ -117,11 +117,14 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo { }, { name: 'bucket_updates', unique: true } ); + // V3 workers claim only buckets that have reached their scheduled check + // time, keeping scheduled scans proportional to pending work. await bucketState.createIndex( + { next_compact_check: 1 }, { - 'estimate_since_compact.count': -1 - }, - { name: 'dirty_count' } + name: 'next_compact_check', + partialFilterExpression: { next_compact_check: { $exists: true } } + } ); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts index 772fe90de..5e4a64e4e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts @@ -15,7 +15,6 @@ import * as bson from 'bson'; import { BucketDataKey, BucketParameterDocumentBase, - BucketStateDocumentBase, CurrentBucket, OpType, ReplicaId, @@ -159,10 +158,54 @@ export interface SourceTableDocumentV3 { latest_pending_delete?: InternalOpId | undefined; } -export interface BucketStateDocumentV3 extends BucketStateDocumentBase { - _id: BucketStateDocumentBase['_id'] & { +export interface BucketStateDocumentV3 { + _id: { + b: string; d: BucketDefinitionId; }; + + /** Must always identify an actual operation in this logical stream. */ + last_op: bigint; + + /** The next time a compact worker should inspect this bucket. */ + next_compact_check: Date | undefined; + /** The oldest write that has not been covered by a full compact. */ + first_uncompacted_write: Date | undefined; + + /** + * A checksum cache and the statistics captured by the latest compact (full + * or lite). Keeping these separate from bucket_stats lets writers only + * update one set of counters. + */ + compacted_state?: { + op_id: InternalOpId; + checksum: bigint; + count: number; + bytes: bigint; + at: Date; + chunks: number; + }; + + /** Statistics from the most recent full compact. */ + last_full_compact?: { + op_id: InternalOpId; + count: number; + at: Date; + puts: number; + }; + + /** Current aggregate bucket statistics, maintained by writers and compactors. */ + bucket_stats: { + count: number; + bytes: bigint; + chunks: number; + }; + + /** A short-lived ownership marker used to distribute bucket compaction. */ + compact_lease?: { + expires_at: Date; + id: unknown; + }; } export interface BucketOperation { diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 504c0f61c..0c8f423d5 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -75,18 +75,15 @@ bucket_definitions: return { bucketStorage, checkpoint, factory, syncRules: syncRulesContent }; }; - test('full compact', async () => { + test('V1 full compact with blank bucket_state', async () => { const { bucketStorage, checkpoint, factory, syncRules } = await setup(); const storageDb = bucketStorage.db; - // Simulate bucket_state from old version not being available if (storageDb.storageConfig.incrementalReprocessing) { - // This should actually never happen on V3, but we test this anyway. - // Can remove this if it causes issues in the future. - await (storageDb as VersionedPowerSyncMongoV3).bucketState(bucketStorage.replicationStreamId).deleteMany({}); - } else { - await factory.db.bucket_state.deleteMany({}); + return; } + // Simulate a V1 deployment which pre-dates bucket-state population. + await factory.db.bucket_state.deleteMany({}); await bucketStorage.compact({ clearBatchLimit: 200, @@ -132,23 +129,22 @@ bucket_definitions: await populate(bucketStorage, 2); const { checkpoint } = await bucketStorage.getCheckpoint(); - // Default is to small small numbers - should be a no-op + // The initial lite pass caches and merges every bucket with no prior compact state. const result0 = await bucketStorage.compactInitialReplication({ maxOpId: checkpoint }); - expect(result0.buckets).toEqual(0); + expect(result0.buckets).toEqual(2); - // This should cache the checksums for the two buckets + // The bucket stats now match the compacted state, so another initial + // pass has no work. const result1 = await bucketStorage.compactInitialReplication({ - maxOpId: checkpoint, - minBucketChanges: 1 + maxOpId: checkpoint }); - expect(result1.buckets).toEqual(2); + expect(result1.buckets).toEqual(0); - // This should be a no-op, as the checksums are already cached + // Repeating it stays a no-op. const result2 = await bucketStorage.compactInitialReplication({ - maxOpId: checkpoint, - minBucketChanges: 1 + maxOpId: checkpoint }); expect(result2.buckets).toEqual(0); @@ -168,88 +164,33 @@ bucket_definitions: }); }); - test('dirty bucket discovery handles bigint bucket_state bytes', async () => { - await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); - const syncRules = await factory.updateSyncRules( - updateSyncRulesFromYaml(` -bucket_definitions: - global: - data: [select * from test] - `) - ); - const bucketStorage = factory.getInstance(syncRules); + test('v3 replication writes initialize scheduled compaction state', async () => { + const { bucketStorage } = await setup(); const storageDb = bucketStorage.db; - // This simulates bucket_state created using bigint bytes. - // This typically happens when buckets get very large (> 2GiB). We don't want to create that much - // data in the tests, so we directly insert the bucket_state here. - if (storageDb.storageConfig.incrementalReprocessing) { - const bucketStateCollection = (storageDb as VersionedPowerSyncMongoV3).bucketState( - bucketStorage.replicationStreamId - ); - await bucketStateCollection.insertOne({ - _id: { - d: '1', - b: 'global[]' - }, - last_op: 5n, - compacted_state: { - op_id: 3n, - count: 3, - checksum: 0n, - bytes: 7n - }, - estimate_since_compact: { - count: 2, - bytes: 5n - } - }); - } else { - await factory.db.bucket_state.insertOne({ - _id: { - g: bucketStorage.replicationStreamId, - b: 'global[]' - }, - last_op: 5n, - compacted_state: { - op_id: 3n, - count: 3, - checksum: 0n, - bytes: 7n - }, - estimate_since_compact: { - count: 2, - bytes: 5n - } - }); + if (!storageDb.storageConfig.incrementalReprocessing) { + return; + } + const bucketStateCollection = (storageDb as VersionedPowerSyncMongoV3).bucketState( + bucketStorage.replicationStreamId + ); + const states = await bucketStateCollection.find({}).toArray(); + expect(states).toHaveLength(2); + for (const state of states) { + expect(state.last_op).toBeGreaterThan(0n); + expect(state.bucket_stats.count).toBe(1); + expect(state.bucket_stats.chunks).toBe(1); + expect(state.first_uncompacted_write).toBeInstanceOf(Date); + expect(state.next_compact_check).toBeInstanceOf(Date); + expect(state.compacted_state).toBeUndefined(); + expect(state).not.toHaveProperty('estimate_since_compact'); + + const docs = await (storageDb as VersionedPowerSyncMongoV3) + .bucketData(bucketStorage.replicationStreamId, state._id.d) + .find({ '_id.b': state._id.b }) + .toArray(); + expect(state.bucket_stats.bytes).toBe(BigInt(docs.reduce((total, document) => total + document.size, 0))); } - - // This test uses a couple of "internal" APIs of the compactor. - const compactor = bucketStorage.createMongoCompactor({ maxOpId: 5n }); - - const dirtyBuckets = compactor.dirtyBucketBatches({ - minBucketChanges: 1, - minChangeRatio: 0.39 - }); - const firstBatch = await dirtyBuckets.next(); - - expect(firstBatch.done).toBe(false); - expect(firstBatch.value).toHaveLength(1); - expect(firstBatch.value[0].bucket).toBe('global[]'); - expect(firstBatch.value[0].estimatedCount).toBe(5); - expect(typeof firstBatch.value[0].estimatedCount).toBe('number'); - expect(firstBatch.value[0].dirtyRatio).toBeCloseTo(5 / 12); - - const checksumBuckets = await compactor.dirtyBucketBatchForChecksums({ - minBucketChanges: 1 - }); - expect(checksumBuckets).toEqual([ - { - bucket: 'global[]', - definitionId: storageDb.storageConfig.incrementalReprocessing ? '1' : null, - estimatedCount: 5 - } - ]); }); }); }); @@ -362,7 +303,9 @@ bucket_definitions: await bucketStateCollection.insertOne({ _id: { d: definitionId, b: BUCKET }, last_op: lastOp, - estimate_since_compact: { count: 10, bytes: 100 } + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { count: 10, bytes: 100n, chunks: 1 } }); } @@ -371,8 +314,6 @@ bucket_definitions: clearBatchLimit: 200, moveBatchLimit: 10, moveBatchQueryLimit: 10, - minBucketChanges: 1, - minChangeRatio: 0, maxOpId }); } @@ -835,7 +776,9 @@ bucket_definitions: await bucketStateCollection.insertOne({ _id: { d: definitionId, b: BUCKET }, last_op: 30n, - estimate_since_compact: { count: 3, bytes: 100 } + next_compact_check: undefined, + first_uncompacted_write: undefined, + bucket_stats: { count: 3, bytes: 100n, chunks: 1 } }); const request = checksumRequest(); @@ -874,7 +817,9 @@ bucket_definitions: await bucketStateCollection.insertOne({ _id: { d: definitionId, b: BUCKET }, last_op: 0n, - estimate_since_compact: { count: 0, bytes: 0 } + next_compact_check: undefined, + first_uncompacted_write: undefined, + bucket_stats: { count: 0, bytes: 0n, chunks: 0 } }); const request = checksumRequest(); @@ -894,7 +839,9 @@ bucket_definitions: await bucketStateCollection.insertOne({ _id: { d: definitionId, b: BUCKET }, last_op: 0n, - estimate_since_compact: { count: 0, bytes: 0 } + next_compact_check: undefined, + first_uncompacted_write: undefined, + bucket_stats: { count: 0, bytes: 0n, chunks: 0 } }); const request = checksumRequest(); @@ -913,7 +860,9 @@ bucket_definitions: await bucketStateCollection.insertOne({ _id: { d: definitionId, b: BUCKET }, last_op: 10n, - estimate_since_compact: { count: 1, bytes: 100 } + next_compact_check: undefined, + first_uncompacted_write: undefined, + bucket_stats: { count: 1, bytes: 100n, chunks: 1 } }); const request = checksumRequest(); @@ -926,7 +875,7 @@ bucket_definitions: await collection.insertOne(serializeBucketData(BUCKET, [clear, afterClear])); await bucketStateCollection.updateOne( { _id: { d: definitionId, b: BUCKET } }, - { $set: { last_op: 30n, 'estimate_since_compact.count': 2 } } + { $set: { last_op: 30n, 'bucket_stats.count': 2 } } ); const after = await bucketStorage.getChecksums(test_utils.testCheckpoint(30n), [request]); @@ -1004,7 +953,9 @@ bucket_definitions: await bucketStateCollection.insertOne({ _id: { d: definitionId, b: BUCKET }, last_op: lastOp, - estimate_since_compact: { count: 10, bytes: 100 } + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { count: 10, bytes: 100n, chunks: 1 } }); } @@ -1030,6 +981,62 @@ bucket_definitions: return collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray(); } + test('scheduled compaction claims only due buckets and clears completed full work', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); + const docs = [ + serializeBucketData(BUCKET, [makeOp(1, 'A', 'old', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(2, 'A', 'new', ctx, sourceTableId)]) + ]; + await insertDocs(collection, docs); + await bucketStateCollection.insertOne({ + _id: { d: ctx.definitionId, b: BUCKET }, + last_op: 2n, + next_compact_check: new Date(Date.now() + 60_000), + first_uncompacted_write: new Date(Date.now() - 60_000), + bucket_stats: { + count: 2, + bytes: BigInt(docs[0].size + docs[1].size), + chunks: 2 + } + }); + + await bucketStorage.compact({ maxOpId: 2n, maxCompactFullIntervalMs: 0 }); + expect( + (await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }))?.compacted_state + ).toBeUndefined(); + + await bucketStateCollection.updateOne( + { _id: { d: ctx.definitionId, b: BUCKET } }, + { $set: { next_compact_check: new Date(Date.now() - 1) } } + ); + await bucketStorage.compact({ maxOpId: 2n, maxCompactFullIntervalMs: 0 }); + + const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(state?.last_full_compact?.op_id).toBe(2n); + expect(state?.first_uncompacted_write).toBeUndefined(); + expect(state?.next_compact_check).toBeUndefined(); + expect(state?.compact_lease).toBeUndefined(); + }); + + test('concurrent scheduled compactors lease a bucket to one worker', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); + const document = serializeBucketData(BUCKET, [makeOp(1, 'A', 'value', ctx, sourceTableId)]); + await insertDocs(collection, [document]); + await bucketStateCollection.insertOne({ + _id: { d: ctx.definitionId, b: BUCKET }, + last_op: 1n, + next_compact_check: new Date(Date.now() - 1), + first_uncompacted_write: new Date(0), + bucket_stats: { count: 1, bytes: BigInt(document.size), chunks: 1 } + }); + + await Promise.all([bucketStorage.compact({ maxOpId: 1n }), bucketStorage.compact({ maxOpId: 1n })]); + + const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(state?.last_full_compact?.op_id).toBe(1n); + expect(state?.compact_lease).toBeUndefined(); + }); + test('1. superseded ops become MOVE tombstones', async () => { const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); // Doc1: [A@10, B@20, A@30] — A@10 superseded by A@30, becomes MOVE tombstone @@ -1271,7 +1278,9 @@ bucket_definitions: await bucketStateCollection.insertOne({ _id: { d: definitionId, b: BUCKET }, last_op: lastOp, - estimate_since_compact: { count: 10, bytes: 100 } + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { count: 10, bytes: 100n, chunks: 1 } }); } @@ -1483,7 +1492,9 @@ bucket_definitions: await bucketStateCollection.insertOne({ _id: { d: definitionId, b: BUCKET }, last_op: lastOp, - estimate_since_compact: { count: 10, bytes: 100 } + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { count: 10, bytes: 100n, chunks: 1 } }); } @@ -1522,7 +1533,6 @@ bucket_definitions: count: 4, checksum: 70n }); - expect(state?.estimate_since_compact).toEqual({ count: 0, bytes: 0 }); }); test('1. multi-batch compaction preserves checksum and creates MOVE tombstones', async () => { diff --git a/modules/module-mongodb-storage/test/src/storage_s3_checksums.test.ts b/modules/module-mongodb-storage/test/src/storage_s3_checksums.test.ts index d34d687f2..44e6b6a04 100644 --- a/modules/module-mongodb-storage/test/src/storage_s3_checksums.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_s3_checksums.test.ts @@ -60,8 +60,7 @@ describe('V3 checksums with S3 object storage', () => { { $set: { last_op: 3n, - compacted_state: { op_id: 3n, count: 0, checksum: 0n, bytes: null }, - estimate_since_compact: { count: 3, bytes: 100 } + compacted_state: { op_id: 3n, count: 0, checksum: 0n, bytes: 0n, chunks: 0, at: new Date() } } }, { upsert: true } diff --git a/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts b/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts index 7c4a20934..d597295ff 100644 --- a/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts @@ -337,14 +337,14 @@ describe('S3 compaction storage lifecycle', () => { const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n); const bucket = request.bucket; - // Read bucket_state before compaction to confirm it exists and has - // estimate_since_compact populated by the writer. + // Read bucket_state before compaction to confirm the writer recorded its + // aggregate statistics and scheduled a compact check. const bucketStateBefore = await bucketStateCollection.findOne({ _id: { d: definitionId, b: bucket } }); expect(bucketStateBefore).toBeDefined(); - expect(bucketStateBefore!.estimate_since_compact).toBeDefined(); - expect(bucketStateBefore!.estimate_since_compact!.count).toBeGreaterThan(0); + expect(bucketStateBefore!.bucket_stats.count).toBeGreaterThan(0); + expect(bucketStateBefore!.next_compact_check).toBeDefined(); // Record the input operations and object path. const batchBefore = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request])); diff --git a/packages/service-core/src/storage/SyncRulesBucketStorage.ts b/packages/service-core/src/storage/SyncRulesBucketStorage.ts index 5b71c3d67..c75420648 100644 --- a/packages/service-core/src/storage/SyncRulesBucketStorage.ts +++ b/packages/service-core/src/storage/SyncRulesBucketStorage.ts @@ -348,6 +348,18 @@ export interface CompactOptions { */ minChangeRatio?: number; + /** Minimum delay before a V3 bucket is checked for chunk compaction. Default: five minutes. */ + minCompactChunkIntervalMs?: number; + + /** Minimum elapsed write pressure before the v3 sliding-scale full compact. Default: two hours. */ + minCompactFullIntervalMs?: number; + + /** Maximum age of writes not covered by a v3 full compact. */ + maxCompactFullIntervalMs?: number; + + /** How long a v3 worker owns a claimed bucket before another worker may recover it. */ + compactLeaseDurationMs?: number; + /** * Internal/testing use: Cache size for compacting parameters. */ From fd1de4e5713e39f2861ee1532d6d717495f61dd2 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 12 Aug 2026 20:04:40 +0200 Subject: [PATCH 03/38] Rewrite design doc. --- docs/storage/v3-compaction-design.md | 136 ++++++++------------------- 1 file changed, 37 insertions(+), 99 deletions(-) diff --git a/docs/storage/v3-compaction-design.md b/docs/storage/v3-compaction-design.md index 7813cc1ba..19460c283 100644 --- a/docs/storage/v3-compaction-design.md +++ b/docs/storage/v3-compaction-design.md @@ -1,132 +1,70 @@ # V3 Compaction Design -This document explains the decisions behind the V3 MongoDB bucket compactor. It accompanies [the compacting plan](v3-compact-plan.md); the plan states the intended behaviour, while this document explains how the implementation realizes it and the tradeoffs involved. +This describes the design of compaction in MongoDB storage V3. ## Goals -V3 compaction is designed to make background work proportional to modified buckets and modified bucket data, rather than to the entire replication stream. It must support frequent execution, resumption after interruption, concurrent workers, and a bounded interval between full compactions. +Compaction should: -The design intentionally does not support the earlier V3 bucket-state format. V3 has not been deployed with that format, so accepting or repairing it would add a second state model and make the normal path less clear. +- make background work proportional to modified buckets and their modified data, rather than proportional to the overall number of buckets and/or operations in a stream. +- support frequent, resumable, concurrent runs; +- avoid turning regular small writes into repeated full rewrites; and +- ensure that every bucket with outstanding work eventually receives a full compact. -## Bucket-state ownership +The design makes the bucket-state collection a persistent work queue. It deliberately replaces the V1-style dirty-operation estimate with explicit scheduling and state captured at the last compact. -`BucketStateDocumentV3` separates three kinds of information: +## Scheduling -1. `bucket_stats` is the current aggregate state of persisted bucket chunks. -2. `compacted_state` is a cache for the prefix covered by the latest lite or full compact. -3. `last_full_compact` records the last full compaction for scheduling heuristics. +In earlier versions, compaction required a scheduled job, that would either: -Replication writers own `last_op` and `bucket_stats`. Compaction owns the compact caches and scheduling state. This division avoids making writers calculate compaction-specific estimates. +1. Iterate through all buckets, filter them according to stats in the bucket-state collection, then compact if needed. This was not safe for interruption. +2. Iterate through all buckets with `estimate_since_compact.count >= 10` or similar indexed condition, perform additional filtering, then compact. This had better resumability, but could repeatedly re-compact the same busy buckets, and fail to keep up with incoming changes on others. -Each writer flush atomically: +For the scheduling approach here, we instead focus on a single new field: `next_compact_check`. This field is indexed, and populated both when replicating and when compacting. It supports multiple different scenarios: -- advances `last_op`; -- adds operation count, exact serialized chunk bytes, and chunk count to `bucket_stats`; -- sets `first_uncompacted_write` only if it was absent; and -- sets `next_compact_check` to the earlier of its existing value and the requested lite-check time. +1. New data was added to the bucket while replicating, which may or may not need a compact. This schedules a _check_ on the bucket. +2. A bucket was checked for compacting, but does not meet the threshold to compact just yet. Re-schedule another check for later, when the thresholds may be met. +3. A bucket was partially compacted, and may need a full compact later. Re-schedule the full compact. -The byte counter uses persisted serialized chunk sizes, rather than an operation-size estimate. Full and chunk compaction also derive their byte totals from persisted chunk metadata, so both sides of the compaction delta use the same unit. +This lets an index on `next_compact_check` act as a time-prioritized compact queue. We can incrementally process this queue, and compact jobs have no overhead if there is no work to do. It also keeps the logic of when to compact out of the replication worker - the replication worker only has to schedule a compact check. -`first_uncompacted_write` is the oldest change which has not received a full compact. It is deliberately independent of `compacted_state`: chunk compaction can reduce chunk fragmentation without declaring changes fully compacted. +## Compact types -## Scheduling work +We use two separate compact types: -The `next_compact_check` partial index is the work queue. Scheduled workers select only state rows whose check time is due. This replaces scanning state rows for dirty-count estimates. +1. Full compact. This is similar to a v1 storage compact: Iterate through the bucket, replace duplicate operations with MOVE operations; squash a leading sequence of MOVE/REMOVE operations into a CLEAR operation. Additionally, this merges small chunks into larger ones if applicable. +2. Chunk merging. This only merges small chunks into larger ones, which can be much faster. We can compute whether chunks should be merged by just reading the metadata, avoiding reading the individual operations unless we need to merge. This can also incrementally continue from the last position, instead of re-reading the entire bucket. -A compaction run captures a server-time job start. It claims only rows whose check time was due at or before that start. Consequently, a long “run until completion” job does not continually rediscover work that it scheduled itself during the same run. A later job processes that work. +Chunk merging also replaces the separate "checksum pre-calculation" operation in MongoDB v1 storage, as a similar "fast to calculate" job. -Claiming one bucket is an atomic `findOneAndUpdate` which requires either no lease or an expired lease. It writes a worker id and server-time expiry. A claim is therefore both a unit of work distribution and the snapshot boundary for the bucket: +Both of these do still calculate and persist a checksum for the bucket. When using S3 storage, the gains from this is significantly reduced. However, S3 storage is still opt-in, and this is cheap to calculate together with compacting, so we keep the logic for now. -- `S` is the server time when the lease was acquired; -- `L` is the bucket `last_op` returned by that atomic claim; and -- the compactor records `C`, the greatest operation it actually covered. +Chunk merging is important to maintain checksum and data reading performance over large buckets. -Those values are carried through the compaction call chain as a per-bucket compaction context. A dedicated `CompactionLease` owns the atomic claim result, server-time renewal, owner-fenced finalization/rescheduling, and release. Each claim is immediately bound with `await using`, so every path following a successful claim releases it when the scope ends. It is not mutable compactor-instance state, which keeps the lease, op cap, snapshot, finalization, and renewal logic explicitly tied to the claim that created them. +Full compact is required to keep bucket sizes low if the same source rows are repeatedly modified. It is also required to "expire" historical data that should not be exposed to users indefinitely. -If a claimed bucket is not yet eligible for either kind of compact, it is rescheduled atomically and its lease is removed. If no chunks exist beyond `compacted_state`, there is no possible chunk-compaction work, so it is scheduled directly for its calculated full-compact check rather than polled at the chunk-compaction interval. +## Bucket stats -## Choosing full versus chunk compaction +To assist with deciding whether to perform a full compact, a chunk-merge compact, or no compact, we store some stats for each bucket. This includes counts and sizes at the last full compact, the last chunk-merge compact, and the overall bucket. -Full compaction is chosen when either condition holds: +To allow configuring minimum and maximum intervals between compacts, we also store timestamps of the oldest uncompacted write, and the last time a full or chunk-merge compact was performed. -1. `first_uncompacted_write` has reached the maximum full-compaction age. -2. The elapsed age multiplied by the uncompacted operation ratio reaches the minimum full-compaction interval. +## Concurrency -The sliding rule prevents an isolated small write from causing a large full compact immediately, while allowing a sufficiently large update burst to compact before the maximum interval. +To allow running concurrent compact jobs on different buckets, we store a renewable lease for each bucket. The compact job checks out a lease on a bucket before starting any compact work. The lease helps to avoid redundant work, but it is not the only correctness mechanism: the individual compact operations are also designed to be safe under concurrency. -Chunk compaction is chosen when new chunks exist after `compacted_state` and the chunk-compaction interval for that bucket has elapsed. It is intentionally a chunk-layout and checksum-cache operation, not a full logical rewrite. +## Statistics during concurrent writes -The relevant intervals are options with V3 defaults: +Compaction must not hold a long transaction over bucket state, because replication must remain writable. At the same time, bucket stats must remain correct if there are new writes to a bucket while we compact it. -| Setting | Default | Purpose | -| --------------------------- | ---------: | ------------------------------------------------ | -| `minCompactChunkIntervalMs` | 5 minutes | Avoid frequent tiny tail checks. | -| `minCompactFullIntervalMs` | 2 hours | Controls sliding-scale full-compaction pressure. | -| `maxCompactFullIntervalMs` | 7 days | Bounds data retention / full-compaction age. | -| `compactLeaseDurationMs` | 10 minutes | Lets another worker recover abandoned work. | +To cater for this, the compact process calculates the delta of statistics while compacting, then applies that to the total state after compacting. If there are no concurrent modifications while compacting, the bucket state will converge on the compacted state. -These are operational policy choices rather than correctness constants. They should be tuned from workload and backlog metrics. +Some care needs to be taken to take into account the "tail" of a bucket that exists while compacting, but cannot be included in the compact job. -When a bucket has no lite work, its full check is scheduled one minute after the calculated eligibility time. This small late margin prevents a worker using a slightly earlier clock from waking before the exact full-compaction condition is true and repeatedly rescheduling the same bucket. It may delay a full compact by up to that margin. +`next_compact_check` and `first_uncompacted_write` are also affected by this: We cannot unilaterally clear these values if there were further writes to the bucket while compacting, but we also do not capture new values for writes during compacting. We instead use a simple heuristic: If writes are detected during compacting (by checking `last_op` for the bucket), the compact process generates a new conservative value for those. Since it is only used for scheduling, the values do not have to be exact, as long as they are set. -## Chunk compaction +## Initial replication -Chunk compaction reads the previous compacted boundary chunk and any later chunks. Including that one previous chunk matters because it may now merge with newly appended data. It does not rescan the older compacted prefix. +Initial replication uses the same scheduled work model, but forces chunk-merge compaction and includes the first chunk-compaction interval of scheduled work. This makes the initial pass immediate and resumable without introducing a separate selection model. -The chunk-compaction metadata scan reads chunk fields only. Chunk payloads are hydrated only when a group can actually merge. The checksum cache for the unchanged prefix is seeded from `compacted_state`; the compactor combines that cache with metadata from the processed tail rather than recalculating the whole bucket checksum. - -`compactChunksOnly` forces this operation. `compactInitialReplication()` reuses scheduled selection, extending the fixed job-start boundary by one chunk-compaction interval so it includes writes that existed when the pass began, and forces the selected work to chunk rather than full compaction. The fixed boundary means it does not chase writes that arrive while it runs. Scheduled chunk compactions running during initial replication therefore reduce the later pass to the remaining scheduled work. The lease normally prevents the two paths from duplicating work for a bucket; owner-fenced finalization and concurrent-safe operations remain necessary if a lease expires. - -## Full compaction - -Full compaction scans bucket data backwards through the claimed upper boundary, rewrites superseded operations to MOVE operations where appropriate, performs CLEAR reduction, and may merge resulting chunks. It is bounded by the claimed head and any caller-specified `maxOpId` safe buffer. - -The caller cap is never widened: the compactor uses the lower of the claimed `last_op` and the requested maximum. A full compact which cannot cover all operations present at claim time is therefore treated as partial; it must not clear the bucket’s full-compaction debt. - -## Aggregate-statistics delta - -Compaction must not hold a long transaction on `bucket_state`, because replication writes need to continue. Instead it applies the stat correction at finalization. - -At compaction start, let `A` be the aggregate stats for the data being compacted. Let `B` be the corresponding stats after compaction. Replication may concurrently add new chunks, but the final update atomically increments current aggregate state by `B - A`. - -This has the desired result whether or not replication wrote during the compact: - -- with no concurrent writes, current stats become `B`; -- with concurrent writes, their increments remain and the compacted region is corrected from `A` to `B`. - -Chunk compaction calculates `A` and `B` only for its compacted tail plus its single overlap chunk. Full compaction calculates them only for the compacted prefix. Neither needs to rescan a tail beyond `C` merely to update aggregate counters. - -## Finalization and concurrent writes - -Finalization is one lease-fenced update. It updates cache/stat state, decides scheduling state using the current `last_op` in the same update, and removes the lease. - -For a full compact where `C >= L`: - -- if current `last_op == L`, there were no concurrent writes, so `first_uncompacted_write` and `next_compact_check` are cleared; -- if current `last_op > L`, writes arrived after `S`, so `first_uncompacted_write` becomes `S` and the next check is `S + minCompactChunkInterval`. - -Using `S` can schedule slightly earlier than the actual write time, but it cannot postpone the maximum full-compaction deadline. - -For a partial full compact (`C < L`) and for every chunk compact, the original `first_uncompacted_write` is retained. Replacing it with `S` would delay work that was already waiting for a full compact. - -## Lease renewal and failure - -Long compactions renew the lease with server time. Both renewal and finalization require the same worker id. A worker that loses its lease raises a non-retryable lease-loss error: retrying the same compactor instance would still hold a stale lease identity and could race the new owner. - -Transactional replacement conflicts are different. They are retryable because a retry starts the relevant bucket work again, while the lease still belongs to that worker. - -## Legacy separation - -`MongoCompactorV1` retains V1’s dirty-estimate model, its minimum-change settings, and its legacy checksum-update logic. `MongoCompactorV3` does not inherit those paths: - -- V1 owns the legacy `compact()` flow and dirty-bucket discovery. -- V3 owns its own `compact()` flow, scheduled selection, leases, and initial-replication lite path. -- The shared base provides only common configuration, retry handling, and checkpoint-request cleanup. - -This separation prevents a V3 change from accidentally using an estimate-based fallback and makes the intended state model visible at the class boundary. - -## Deliberate non-goals - -- No read, conversion, or repair path exists for an older V3 bucket-state document. -- V3 bucket-data compaction does not use `estimate_since_compact`, `minBucketChanges`, `minChangeRatio`, or dirty-bucket scanning. -- Parameter compaction is separate from V3 bucket-data compaction and currently has no per-collection lease. The V3 compaction lease applies only to bucket-data work. +The initial pass has a fixed boundary and does not chase writes that arrive while it runs. Chunk compaction performed concurrently during replication therefore reduces the work remaining for the post-replication pass. From 6fbffcfff9ccc5e73511237d08fe5d899fbe845c Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 12 Aug 2026 20:11:48 +0200 Subject: [PATCH 04/38] Minor refactoring. --- .../implementation/v3/MongoCompactorV3.ts | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 2edc478c0..4e4374283 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -227,7 +227,7 @@ export class MongoCompactorV3 extends MongoCompactor { * long-running job does not repeatedly process buckets it scheduled itself. */ private async compactScheduledBuckets(options: ScheduledCompactionOptions = {}) { - const jobStartedAt = await this.serverNow(); + const jobStartedAt = new Date(); const dueBefore = new Date(jobStartedAt.getTime() + (options.dueAheadMs ?? 0)); const forceKind = options.forceKind; while (true) { @@ -328,11 +328,6 @@ export class MongoCompactorV3 extends MongoCompactor { return new Date(firstWrite.getTime() + Math.min(fullIntervalMs, this.maxCompactFullIntervalMs)); } - private async serverNow(): Promise { - const result = await this.db.db.command({ hello: 1 }); - return result.localTime as Date; - } - private compactMaxOpId(context: CompactionContext): InternalOpId { return this.maxOpIdCap == null || context.lastOp < this.maxOpIdCap ? context.lastOp : this.maxOpIdCap; } @@ -468,7 +463,7 @@ export class MongoCompactorV3 extends MongoCompactor { bucketStats: this.applyCompactionDelta(bucketStats(context.state), preCompactionTail, tailStats) }; - await this.finalizeCompactedBucket(context, compactedOpId, result); + await this.finalizeCompactedBucket({ context, compactedOpId, compactionResult: result, puts: 0 }); this.compactedBucketCount++; this.logger.info(`Lightly compacted bucket ${bucket}: ${result.bucketStats.count} ops`); } @@ -517,12 +512,17 @@ export class MongoCompactorV3 extends MongoCompactor { ); } - private async finalizeCompactedBucket( - context: CompactionContext, - compactedOpId: InternalOpId, - compactionResult: CompactionResult, - puts = 0 - ) { + private async finalizeCompactedBucket({ + context, + compactedOpId, + compactionResult, + puts + }: { + context: CompactionContext; + compactedOpId: InternalOpId; + compactionResult: CompactionResult; + puts: number; + }) { await context.lease.throwIfLost(); const startedStats = bucketStats(context.state); const delta = { @@ -886,7 +886,7 @@ export class MongoCompactorV3 extends MongoCompactor { }; // --- Finalize: update bucket checksums and state --- - await this.finalizeCompactedBucket(context, compactedOpId, result, putCount); + await this.finalizeCompactedBucket({ context, compactedOpId, compactionResult: result, puts: putCount }); this.logger.info(`Compacted bucket ${bucket}: ${totalOpCount} surviving ops`); } From 68b9f22ce3433fa739a13e411c0187cfdcebc3be Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 12 Aug 2026 20:25:14 +0200 Subject: [PATCH 05/38] Increase threshold for performing chunk-merge compaction. --- .../implementation/v3/MongoCompactorV3.ts | 89 +++++++++++-------- 1 file changed, 53 insertions(+), 36 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 4e4374283..6fb17408d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -32,6 +32,11 @@ enum CompactionKind { Chunks = 'chunks' } +interface CompactionDecision { + kind: CompactionKind | null; + nextCompactCheck: mongo.Document; +} + interface ScheduledCompactionOptions { /** Process checks scheduled this far after the captured job start. */ dueAheadMs?: number; @@ -42,7 +47,8 @@ interface ScheduledCompactionOptions { class CompactionContext { constructor( readonly lease: CompactionLease, - readonly kind: CompactionKind + readonly kind: CompactionKind, + readonly decision: CompactionDecision | undefined ) {} get state() { @@ -81,6 +87,14 @@ const DEFAULT_MIN_COMPACT_FULL_INTERVAL_MS = 2 * 60 * 60 * 1000; const DEFAULT_MAX_COMPACT_FULL_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000; const DEFAULT_COMPACT_LEASE_DURATION_MS = 10 * 60 * 1000; const FULL_COMPACT_RESCHEDULE_MARGIN_MS = 60 * 1000; +/** + * Perform a chunk-merge compact if at least this many chunks have been added since hte last compaction. + * + * If the value is too low (say 1 or 2), this introduces write amplification: Every write may result in multiple object storage operations. + * + * If the value is too large, it may delay merging of chunks and negatively affect sync performance. + */ +const MERGE_CHUNKS_THRESHOLD = 8; /** * Read one bounded prefix from a compaction cursor. @@ -240,11 +254,12 @@ export class MongoCompactorV3 extends MongoCompactor { break; } - const kind = forceKind ?? this.chooseCompactionKind(lease, lease.startedAt); + const decision = this.chooseCompactionKind(lease, lease.startedAt); + const kind = forceKind ?? decision.kind; if (kind == null) { - await this.rescheduleClaimedBucket(lease); + await this.rescheduleClaimedBucket(lease, decision); } else { - await this.compactClaimedBucket(lease, kind); + await this.compactClaimedBucket(lease, kind, decision); } } } @@ -256,52 +271,51 @@ export class MongoCompactorV3 extends MongoCompactor { return CompactionLease.claim(this.db.bucketState(this.group_id), filter, sort, this.compactLeaseDurationMs); } - private async compactClaimedBucket(lease: CompactionLease, kind: CompactionKind) { - const context = new CompactionContext(lease, kind); + private async compactClaimedBucket(lease: CompactionLease, kind: CompactionKind, decision?: CompactionDecision) { + const context = new CompactionContext(lease, kind, decision); lease.startRenewal(); await this.retryCompaction(context.state._id.b, () => this.compactSingleBucket(context)); } - private chooseCompactionKind(lease: CompactionLease, now: Date): CompactionKind | null { + private chooseCompactionKind(lease: CompactionLease, now: Date): CompactionDecision { const state = lease.state; - if (now >= this.fullCompactionCheckAt(state)) { - return CompactionKind.Full; - } - // For chunk compaction, we consider the number of chunks added. - // Right now, we trigger a compact if the interval has passed and at least 1 chunk was added. + // Right now, we trigger a compact if the interval has passed and at least a threshold of chunks were added. // In the future me may use a threshold for bytes/chunk or count/chunk instead, and only compact // if one of those are low. + const fullCheckAt = this.fullCompactionCheckAt(state); + // Schedule a little late so a worker using a slightly earlier clock does + // not wake before the exact full-compaction condition is true. + const fullCheckWithMargin = new Date(fullCheckAt.getTime() + FULL_COMPACT_RESCHEDULE_MARGIN_MS); const compacted = state.compacted_state; const chunksSinceCompact = Math.max(0, state.bucket_stats.chunks - (compacted?.chunks ?? 0)); + const shouldCompactChunks = chunksSinceCompact >= MERGE_CHUNKS_THRESHOLD; const canCheckChunks = compacted == null || now.getTime() - compacted.at.getTime() >= this.minCompactChunkIntervalMs; - if (canCheckChunks && chunksSinceCompact >= 1) { - return CompactionKind.Chunks; + // Too few new chunks can make chunk compaction eligible. Do not poll this + // bucket at the chunk-compaction interval; only wake it for its full-compact check. + const nextCompactCheck: mongo.Document = !shouldCompactChunks + ? fullCheckWithMargin + : { + $min: [ + fullCheckWithMargin, + { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.minCompactChunkIntervalMs } } + ] + }; + let kind: CompactionKind | null = null; + if (now >= fullCheckAt) { + kind = CompactionKind.Full; + } else if (canCheckChunks && shouldCompactChunks) { + kind = CompactionKind.Chunks; } - return null; + return { + kind, + nextCompactCheck + }; } - private async rescheduleClaimedBucket(lease: CompactionLease) { - const state = lease.state; - const fullCheckAt = this.fullCompactionCheckAt(state); - // Schedule a little late so a worker using a slightly earlier clock does - // not wake before the exact full-compaction condition is true. - const fullCheckWithMargin = new Date(fullCheckAt.getTime() + FULL_COMPACT_RESCHEDULE_MARGIN_MS); - const chunksSinceCompact = Math.max(0, state.bucket_stats.chunks - (state.compacted_state?.chunks ?? 0)); - if (chunksSinceCompact == 0) { - // No new chunks can make chunk compaction eligible. Do not poll this - // bucket at the chunk-compaction interval; only wake it for its full-compact check. - await lease.reschedule(fullCheckWithMargin); - return; - } - - await lease.reschedule({ - $min: [ - fullCheckWithMargin, - { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.minCompactChunkIntervalMs } } - ] - }); + private async rescheduleClaimedBucket(lease: CompactionLease, decision: CompactionDecision) { + await lease.reschedule(decision.nextCompactCheck); } /** @@ -575,7 +589,10 @@ export class MongoCompactorV3 extends MongoCompactor { } private async finalizeSkippedBucket(context: CompactionContext) { - await this.rescheduleClaimedBucket(context.lease); + await this.rescheduleClaimedBucket( + context.lease, + context.decision ?? this.chooseCompactionKind(context.lease, context.startedAt) + ); } private async readBucketStats( From ad086cb836e978686d37c84e6d123a973d07c807 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 13 Aug 2026 09:06:15 +0200 Subject: [PATCH 06/38] Use batches to reduce overhead for large numbers of no-op compacts. --- docs/storage/v3-compaction-design.md | 28 ++-- .../implementation/v3/MongoCompactorV3.ts | 139 ++++++++++++++---- .../src/storage/implementation/v3/models.ts | 2 +- 3 files changed, 127 insertions(+), 42 deletions(-) diff --git a/docs/storage/v3-compaction-design.md b/docs/storage/v3-compaction-design.md index 19460c283..e78186238 100644 --- a/docs/storage/v3-compaction-design.md +++ b/docs/storage/v3-compaction-design.md @@ -15,39 +15,43 @@ The design makes the bucket-state collection a persistent work queue. It deliber ## Scheduling -In earlier versions, compaction required a scheduled job, that would either: +In earlier versions, compaction required a scheduled job that would either: 1. Iterate through all buckets, filter them according to stats in the bucket-state collection, then compact if needed. This was not safe for interruption. 2. Iterate through all buckets with `estimate_since_compact.count >= 10` or similar indexed condition, perform additional filtering, then compact. This had better resumability, but could repeatedly re-compact the same busy buckets, and fail to keep up with incoming changes on others. -For the scheduling approach here, we instead focus on a single new field: `next_compact_check`. This field is indexed, and populated both when replicating and when compacting. It supports multiple different scenarios: +For the scheduling approach here, we instead focus on a single new field: `next_compact_check`. This field is indexed and populated both during replication and during compaction. It supports multiple different scenarios: 1. New data was added to the bucket while replicating, which may or may not need a compact. This schedules a _check_ on the bucket. 2. A bucket was checked for compacting, but does not meet the threshold to compact just yet. Re-schedule another check for later, when the thresholds may be met. 3. A bucket was partially compacted, and may need a full compact later. Re-schedule the full compact. -This lets an index on `next_compact_check` act as a time-prioritized compact queue. We can incrementally process this queue, and compact jobs have no overhead if there is no work to do. It also keeps the logic of when to compact out of the replication worker - the replication worker only has to schedule a compact check. +This lets an index on `next_compact_check` act as a time-prioritized compact queue. We can incrementally process this queue without reading or rewriting bucket data when a check finds no compaction work. It also keeps the logic of when to compact out of the replication worker - the replication worker only has to schedule a compact check. + +Workers may inspect a bounded batch before taking a lease. Checks already known to be no-ops are rescheduled together from their unleased snapshots. Each reschedule is conditional on the state still matching that snapshot and no lease being held, so concurrent writes or compactors make the reschedule a no-op rather than losing work. ## Compact types We use two separate compact types: 1. Full compact. This is similar to a v1 storage compact: Iterate through the bucket, replace duplicate operations with MOVE operations; squash a leading sequence of MOVE/REMOVE operations into a CLEAR operation. Additionally, this merges small chunks into larger ones if applicable. -2. Chunk merging. This only merges small chunks into larger ones, which can be much faster. We can compute whether chunks should be merged by just reading the metadata, avoiding reading the individual operations unless we need to merge. This can also incrementally continue from the last position, instead of re-reading the entire bucket. +2. Chunk compaction. This only merges small chunks into larger ones, which can be much faster. We can compute whether chunks should be merged by just reading the metadata, avoiding reading the individual operations unless we need to merge. This can also incrementally continue from the last position, instead of re-reading the entire bucket. -Chunk merging also replaces the separate "checksum pre-calculation" operation in MongoDB v1 storage, as a similar "fast to calculate" job. +Chunk compaction also replaces the separate "checksum pre-calculation" operation in MongoDB v1 storage, as a similar "fast to calculate" job. -Both of these do still calculate and persist a checksum for the bucket. When using S3 storage, the gains from this is significantly reduced. However, S3 storage is still opt-in, and this is cheap to calculate together with compacting, so we keep the logic for now. +Both types calculate and persist checksum state for their compacted data. When using S3 storage, the benefit is significantly reduced. However, S3 storage is still opt-in, and this is cheap to calculate together with compacting, so we keep the logic for now. -Chunk merging is important to maintain checksum and data reading performance over large buckets. +Chunk compaction is important to maintain checksum and data reading performance over large buckets. Full compact is required to keep bucket sizes low if the same source rows are repeatedly modified. It is also required to "expire" historical data that should not be exposed to users indefinitely. +The delay before a full compact is inversely related to the amount of work since the previous full compact, and is capped by a maximum interval. This avoids frequent full rewrites for a small amount of new work while ensuring that outstanding work is eventually compacted. + ## Bucket stats -To assist with deciding whether to perform a full compact, a chunk-merge compact, or no compact, we store some stats for each bucket. This includes counts and sizes at the last full compact, the last chunk-merge compact, and the overall bucket. +To assist with deciding whether to perform a full compact, a chunk compact, or no compact, we store current aggregate counts and sizes, a cached snapshot at the latest compact, and the figures from the last full compact needed to schedule the next one. -To allow configuring minimum and maximum intervals between compacts, we also store timestamps of the oldest uncompacted write, and the last time a full or chunk-merge compact was performed. +To allow configuring minimum and maximum intervals between compacts, we also store timestamps of the oldest uncompacted write, the latest compact, and the most recent full compact. ## Concurrency @@ -57,14 +61,14 @@ To allow running concurrent compact jobs on different buckets, we store a renewa Compaction must not hold a long transaction over bucket state, because replication must remain writable. At the same time, bucket stats must remain correct if there are new writes to a bucket while we compact it. -To cater for this, the compact process calculates the delta of statistics while compacting, then applies that to the total state after compacting. If there are no concurrent modifications while compacting, the bucket state will converge on the compacted state. +To cater for this, the compact process calculates the delta of statistics while compacting, then applies that to the total state after compacting. If there are no concurrent modifications while compacting, the aggregate state converges on the result for the compacted range. Some care needs to be taken to take into account the "tail" of a bucket that exists while compacting, but cannot be included in the compact job. -`next_compact_check` and `first_uncompacted_write` are also affected by this: We cannot unilaterally clear these values if there were further writes to the bucket while compacting, but we also do not capture new values for writes during compacting. We instead use a simple heuristic: If writes are detected during compacting (by checking `last_op` for the bucket), the compact process generates a new conservative value for those. Since it is only used for scheduling, the values do not have to be exact, as long as they are set. +`next_compact_check` and `first_uncompacted_write` are also affected by this: We cannot unilaterally clear these values if there were further writes to the bucket while compacting, but we also do not capture new values for writes during compacting. If writes are detected while finalizing (by checking `last_op` for the bucket), the compact process retains conservative scheduling values for them. Since they are only used for scheduling, the values do not have to be exact, as long as they are set. ## Initial replication -Initial replication uses the same scheduled work model, but forces chunk-merge compaction and includes the first chunk-compaction interval of scheduled work. This makes the initial pass immediate and resumable without introducing a separate selection model. +Initial replication uses the same scheduled work model, but forces chunk compaction and includes the first chunk-compaction interval of scheduled work. This makes the initial pass immediate and resumable without introducing a separate selection model. The initial pass has a fixed boundary and does not chase writes that arrive while it runs. Chunk compaction performed concurrently during replication therefore reduces the work remaining for the post-replication pass. diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 6fb17408d..bd3b18af0 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -48,7 +48,7 @@ class CompactionContext { constructor( readonly lease: CompactionLease, readonly kind: CompactionKind, - readonly decision: CompactionDecision | undefined + readonly decision: CompactionDecision ) {} get state() { @@ -87,12 +87,14 @@ const DEFAULT_MIN_COMPACT_FULL_INTERVAL_MS = 2 * 60 * 60 * 1000; const DEFAULT_MAX_COMPACT_FULL_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000; const DEFAULT_COMPACT_LEASE_DURATION_MS = 10 * 60 * 1000; const FULL_COMPACT_RESCHEDULE_MARGIN_MS = 60 * 1000; +const SCHEDULED_COMPACTION_BATCH_SIZE = 100; /** - * Perform a chunk-merge compact if at least this many chunks have been added since hte last compaction. + * Perform chunk compaction if at least this many chunks have been added since + * the latest compaction. * - * If the value is too low (say 1 or 2), this introduces write amplification: Every write may result in multiple object storage operations. - * - * If the value is too large, it may delay merging of chunks and negatively affect sync performance. + * A lower value increases compaction frequency and can increase chunk rewrites + * and object-storage operations. A higher value leaves more small chunks for + * longer, which can hurt sync performance. */ const MERGE_CHUNKS_THRESHOLD = 8; @@ -230,15 +232,28 @@ export class MongoCompactorV3 extends MongoCompactor { for (const state of states) { await using lease = await this.claimBucket({ _id: state._id }); if (lease != null) { - await this.compactClaimedBucket(lease, CompactionKind.Full); + const decision = this.chooseCompactionKind(lease.state, lease.startedAt); + await this.compactClaimedBucket(lease, CompactionKind.Full, decision); } } } } /** - * Claim one scheduled bucket at a time. The fixed due boundary means a - * long-running job does not repeatedly process buckets it scheduled itself. + * Process scheduled work in bounded batches. + * + * Batching specifically help to cover cases of many buckets where no compaction is required: + * Instead of sequentially claiming and then rescheduling a bucket, this handles it in bulk. + * + * Buckets that do need compaction are still claimed and processed sequentially. + * + * Any concurrent workers may read the same batch. Rescheduling filters out buckets handled + * by a concurrent worker or replication write, while buckets that do need compaction are + * filtered out when claiming a compaction lease. + * + * We filter scheduled jobs by the job start date, so that the same bucket is not compacted + * multiple times in one run. One tweak to this is for the run after initial replication: + * We use dueAheadMs to also include buckets scheduled during initial replication. */ private async compactScheduledBuckets(options: ScheduledCompactionOptions = {}) { const jobStartedAt = new Date(); @@ -246,24 +261,94 @@ export class MongoCompactorV3 extends MongoCompactor { const forceKind = options.forceKind; while (true) { this.signal?.throwIfAborted(); - await using lease = await this.claimBucket( - { next_compact_check: { $lte: dueBefore } }, - { next_compact_check: 1 } - ); - if (lease == null) { + const states = await this.findScheduledBucketBatch(dueBefore); + if (states.length == 0) { break; } - const decision = this.chooseCompactionKind(lease, lease.startedAt); - const kind = forceKind ?? decision.kind; - if (kind == null) { - await this.rescheduleClaimedBucket(lease, decision); - } else { - await this.compactClaimedBucket(lease, kind, decision); + const scheduled = states.map((state) => ({ + state, + decision: this.chooseCompactionKind(state, jobStartedAt) + })); + const noOpStates = scheduled.filter( + ({ state, decision }) => state.compact_lease == null && forceKind == null && decision.kind == null + ); + await this.rescheduleUnclaimedBuckets(noOpStates); + + for (const { state, decision } of scheduled) { + const kind = forceKind ?? decision.kind; + if (state.compact_lease == null && kind == null) { + continue; + } + + await using lease = await this.claimBucket({ _id: state._id, next_compact_check: { $lte: dueBefore } }); + if (lease == null) { + continue; + } + const claimedDecision = this.chooseCompactionKind(lease.state, lease.startedAt); + const claimedKind = forceKind ?? claimedDecision.kind; + if (claimedKind == null) { + await this.rescheduleClaimedBucket(lease, claimedDecision); + } else { + await this.compactClaimedBucket(lease, claimedKind, claimedDecision); + } } } } + /** Read a bounded, priority-ordered snapshot of currently claimable scheduled work. */ + private async findScheduledBucketBatch(dueBefore: Date): Promise { + return this.db + .bucketState(this.group_id) + .find({ + next_compact_check: { $lte: dueBefore }, + $expr: { + $or: [{ $eq: [{ $type: '$compact_lease' }, 'missing'] }, { $lte: ['$compact_lease.expires_at', '$$NOW'] }] + } + }) + .sort({ next_compact_check: 1 }) + .limit(SCHEDULED_COMPACTION_BATCH_SIZE) + .toArray(); + } + + /** + * Reschedule snapshots that were already known to be no-ops without first + * taking a lease. Every decision input is compared so a concurrent writer + * or compactor simply makes the update a no-op instead of losing work. + */ + private async rescheduleUnclaimedBuckets(states: { state: BucketStateDocumentV3; decision: CompactionDecision }[]) { + if (states.length == 0) { + return; + } + await this.db.bucketState(this.group_id).bulkWrite( + states.map(({ state, decision }) => ({ + updateOne: { + filter: this.unclaimedSnapshotFilter(state), + update: [{ $set: { next_compact_check: decision.nextCompactCheck } }] + } + })), + { ordered: false } + ); + } + + /** + * This checks that the bucket state hasn't changed by a concurrent write since we checked. + * + * If it has changed, we'll re-check in the next batch. + */ + private unclaimedSnapshotFilter(state: BucketStateDocumentV3): mongo.Filter { + return { + _id: state._id, + last_op: state.last_op, + next_compact_check: state.next_compact_check, + first_uncompacted_write: state.first_uncompacted_write ?? { $exists: false }, + bucket_stats: state.bucket_stats, + compacted_state: state.compacted_state ?? { $exists: false }, + last_full_compact: state.last_full_compact ?? { $exists: false }, + compact_lease: { $exists: false } + }; + } + private async claimBucket( filter: mongo.Filter, sort?: mongo.Sort @@ -271,18 +356,17 @@ export class MongoCompactorV3 extends MongoCompactor { return CompactionLease.claim(this.db.bucketState(this.group_id), filter, sort, this.compactLeaseDurationMs); } - private async compactClaimedBucket(lease: CompactionLease, kind: CompactionKind, decision?: CompactionDecision) { + private async compactClaimedBucket(lease: CompactionLease, kind: CompactionKind, decision: CompactionDecision) { const context = new CompactionContext(lease, kind, decision); lease.startRenewal(); await this.retryCompaction(context.state._id.b, () => this.compactSingleBucket(context)); } - private chooseCompactionKind(lease: CompactionLease, now: Date): CompactionDecision { - const state = lease.state; + private chooseCompactionKind(state: BucketStateDocumentV3, now: Date): CompactionDecision { // For chunk compaction, we consider the number of chunks added. // Right now, we trigger a compact if the interval has passed and at least a threshold of chunks were added. - // In the future me may use a threshold for bytes/chunk or count/chunk instead, and only compact - // if one of those are low. + // A future policy could also use bytes per chunk or records per chunk to + // decide when to compact. const fullCheckAt = this.fullCompactionCheckAt(state); // Schedule a little late so a worker using a slightly earlier clock does // not wake before the exact full-compaction condition is true. @@ -292,7 +376,7 @@ export class MongoCompactorV3 extends MongoCompactor { const shouldCompactChunks = chunksSinceCompact >= MERGE_CHUNKS_THRESHOLD; const canCheckChunks = compacted == null || now.getTime() - compacted.at.getTime() >= this.minCompactChunkIntervalMs; - // Too few new chunks can make chunk compaction eligible. Do not poll this + // Too few new chunks cannot make chunk compaction eligible. Do not poll this // bucket at the chunk-compaction interval; only wake it for its full-compact check. const nextCompactCheck: mongo.Document = !shouldCompactChunks ? fullCheckWithMargin @@ -589,10 +673,7 @@ export class MongoCompactorV3 extends MongoCompactor { } private async finalizeSkippedBucket(context: CompactionContext) { - await this.rescheduleClaimedBucket( - context.lease, - context.decision ?? this.chooseCompactionKind(context.lease, context.startedAt) - ); + await this.rescheduleClaimedBucket(context.lease, context.decision); } private async readBucketStats( diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts index 5e4a64e4e..f9e4dc232 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts @@ -174,7 +174,7 @@ export interface BucketStateDocumentV3 { /** * A checksum cache and the statistics captured by the latest compact (full - * or lite). Keeping these separate from bucket_stats lets writers only + * or chunk). Keeping these separate from bucket_stats lets writers only * update one set of counters. */ compacted_state?: { From 557af9253b1eff33aef4ccf0486e05aac51c1484 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 13 Aug 2026 09:15:50 +0200 Subject: [PATCH 07/38] Tweaks. --- .../implementation/common/PersistedBatch.ts | 16 +++------ .../implementation/v3/MongoCompactorV3.ts | 7 ++-- .../implementation/v3/PersistedBatchV3.ts | 3 +- .../test/src/storage_compacting.test.ts | 34 +++++++++++++++++++ 4 files changed, 44 insertions(+), 16 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts index 13184d048..71ea634bb 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts @@ -242,14 +242,15 @@ export abstract class PersistedBatch { /** * V3 persists operations in chunks. Keep this separate from incrementBucket: - * operation counts are known while evaluating rows, while chunk counts are - * only known after the writer has chunked a flush. + * operation counts are known while evaluating rows, while chunk counts and + * exact persisted bytes are only known after the writer has chunked a flush. */ - protected incrementBucketChunks(definitionId: BucketDefinitionId, bucket: string, chunks = 1) { + protected incrementBucketPersistedChunk(definitionId: BucketDefinitionId, bucket: string, bytes: number) { const key = `${definitionId ?? ''}:${bucket}`; const existingState = this.bucketStates.get(key); if (existingState != null) { - existingState.incrementChunks += chunks; + existingState.incrementChunks += 1; + existingState.incrementBytes += bytes; } } @@ -265,13 +266,6 @@ export abstract class PersistedBatch { } } - protected incrementBucketPersistedBytes(definitionId: BucketDefinitionId, bucket: string, bytes: number) { - const state = this.bucketStates.get(`${definitionId ?? ''}:${bucket}`); - if (state != null) { - state.incrementBytes += bytes; - } - } - protected addBucketDataPut(options: { op_id: InternalOpId; bucketKey: BucketKey; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index bd3b18af0..4339f208f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -231,10 +231,11 @@ export class MongoCompactorV3 extends MongoCompactor { .toArray(); for (const state of states) { await using lease = await this.claimBucket({ _id: state._id }); - if (lease != null) { - const decision = this.chooseCompactionKind(lease.state, lease.startedAt); - await this.compactClaimedBucket(lease, CompactionKind.Full, decision); + if (lease == null || lease.state.first_uncompacted_write == null) { + continue; } + const decision = this.chooseCompactionKind(lease.state, lease.startedAt); + await this.compactClaimedBucket(lease, CompactionKind.Full, decision); } } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts index 58f8dd7df..5719b7084 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -235,8 +235,7 @@ export class PersistedBatchV3 extends PersistedBatch { this.resetBucketPersistedBytes(definitionId, bucket); for (const chunk of chunkBucketData(ops)) { const serialized = serializeBucketData(bucket, chunk); - this.incrementBucketChunks(definitionId, bucket); - this.incrementBucketPersistedBytes(definitionId, bucket, serialized.size); + this.incrementBucketPersistedChunk(definitionId, bucket, serialized.size); if (lifecycle == null || serialized.size <= this.inlineThresholdBytes) { createInserts.push(async () => ({ insertOne: { diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 0c8f423d5..638f162ae 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -1018,6 +1018,40 @@ bucket_definitions: expect(state?.compact_lease).toBeUndefined(); }); + test('explicit compaction skips a bucket with no outstanding full-compaction work', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); + const document = serializeBucketData(BUCKET, [makeOp(1, 'A', 'value', ctx, sourceTableId)]); + await insertDocs(collection, [document]); + await bucketStateCollection.insertOne({ + _id: { d: ctx.definitionId, b: BUCKET }, + last_op: 1n, + next_compact_check: undefined, + first_uncompacted_write: undefined, + compacted_state: { + op_id: 1n, + checksum: document.checksum, + count: document.count, + bytes: BigInt(document.size), + chunks: 1, + at: new Date() + }, + last_full_compact: { + op_id: 1n, + count: document.count, + puts: 1, + at: new Date() + }, + bucket_stats: { count: document.count, bytes: BigInt(document.size), chunks: 1 } + }); + + await expect(bucketStorage.compact({ compactBuckets: [BUCKET], maxOpId: 1n })).resolves.toBeUndefined(); + + const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(state?.compact_lease).toBeUndefined(); + expect(state?.next_compact_check).toBeUndefined(); + expect(state?.last_full_compact?.op_id).toBe(1n); + }); + test('concurrent scheduled compactors lease a bucket to one worker', async () => { const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); const document = serializeBucketData(BUCKET, [makeOp(1, 'A', 'value', ctx, sourceTableId)]); From a655a826e45c0e23e3843b0abc170bf223a7b640 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 13 Aug 2026 09:34:44 +0200 Subject: [PATCH 08/38] Add notes on job scheduling. --- docs/storage/v3-compaction-design.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/storage/v3-compaction-design.md b/docs/storage/v3-compaction-design.md index e78186238..a0be897bf 100644 --- a/docs/storage/v3-compaction-design.md +++ b/docs/storage/v3-compaction-design.md @@ -1,6 +1,6 @@ # V3 Compaction Design -This describes the design of compaction in MongoDB storage V3. +This describes the design of compaction scheduling in MongoDB storage V3. For details on what compaction means on a protocol level, see [./compating-operations.md](./compacting-operations.md). ## Goals @@ -72,3 +72,16 @@ Some care needs to be taken to take into account the "tail" of a bucket that exi Initial replication uses the same scheduled work model, but forces chunk compaction and includes the first chunk-compaction interval of scheduled work. This makes the initial pass immediate and resumable without introducing a separate selection model. The initial pass has a fixed boundary and does not chase writes that arrive while it runs. Chunk compaction performed concurrently during replication therefore reduces the work remaining for the post-replication pass. + +## Compaction jobs + +This design gives us flexibility in how compaction could be run: + +1. Scheduled job running once per day - same as before. +2. Scheduled job running once per hour or even every 5 minutes. Actual work to perform is throttled in the job itself, and does not depend on job scheduling anymore. +3. As a tweak to the above, allow concurrent compaction jobs if the previous one has not finished. This would effectively increase the number of compaction jobs as the backlog increases. +4. Run a single process continuously polling for new buckets to compact. +5. Run multiple processes continously polling for new buckets to compact. +6. Use auto-scaling to dynamically configure the number of compaction processes depending on the backlog size. + +Note that the options 4-6 are technically feasible, but not implemented yet. From ff5a2b3b8bea5546be13621eaaff7a54798efd25 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 13 Aug 2026 09:39:29 +0200 Subject: [PATCH 09/38] When re-scheduling, add a minimum delay, to avoid infinite immediate retries. --- .../implementation/v3/MongoCompactorV3.ts | 58 ++++++++++++++----- .../test/src/storage_compacting.test.ts | 2 +- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 4339f208f..866dab02b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -48,7 +48,8 @@ class CompactionContext { constructor( readonly lease: CompactionLease, readonly kind: CompactionKind, - readonly decision: CompactionDecision + readonly decision: CompactionDecision, + readonly rescheduleNotBefore: Date | undefined ) {} get state() { @@ -253,13 +254,14 @@ export class MongoCompactorV3 extends MongoCompactor { * filtered out when claiming a compaction lease. * * We filter scheduled jobs by the job start date, so that the same bucket is not compacted - * multiple times in one run. One tweak to this is for the run after initial replication: - * We use dueAheadMs to also include buckets scheduled during initial replication. + * multiple times in one run. Reschedules fall beyond the fixed boundary. For the run after + * initial replication, dueAheadMs extends that boundary to include the first deferred interval. */ private async compactScheduledBuckets(options: ScheduledCompactionOptions = {}) { const jobStartedAt = new Date(); const dueBefore = new Date(jobStartedAt.getTime() + (options.dueAheadMs ?? 0)); const forceKind = options.forceKind; + const rescheduleNotBefore = new Date(dueBefore.getTime() + 1); while (true) { this.signal?.throwIfAborted(); const states = await this.findScheduledBucketBatch(dueBefore); @@ -291,7 +293,7 @@ export class MongoCompactorV3 extends MongoCompactor { if (claimedKind == null) { await this.rescheduleClaimedBucket(lease, claimedDecision); } else { - await this.compactClaimedBucket(lease, claimedKind, claimedDecision); + await this.compactClaimedBucket(lease, claimedKind, claimedDecision, rescheduleNotBefore); } } } @@ -357,8 +359,13 @@ export class MongoCompactorV3 extends MongoCompactor { return CompactionLease.claim(this.db.bucketState(this.group_id), filter, sort, this.compactLeaseDurationMs); } - private async compactClaimedBucket(lease: CompactionLease, kind: CompactionKind, decision: CompactionDecision) { - const context = new CompactionContext(lease, kind, decision); + private async compactClaimedBucket( + lease: CompactionLease, + kind: CompactionKind, + decision: CompactionDecision, + rescheduleNotBefore?: Date + ) { + const context = new CompactionContext(lease, kind, decision, rescheduleNotBefore); lease.startRenewal(); await this.retryCompaction(context.state._id.b, () => this.compactSingleBucket(context)); } @@ -403,6 +410,10 @@ export class MongoCompactorV3 extends MongoCompactor { await lease.reschedule(decision.nextCompactCheck); } + private rescheduleAtOrAfter(nextCompactCheck: mongo.Document, notBefore: Date | undefined): mongo.Document { + return notBefore == null ? nextCompactCheck : { $max: [nextCompactCheck, notBefore] }; + } + /** * Calculate the earliest full compaction time from the first uncompacted * write, bounded by the maximum retention interval. @@ -631,13 +642,19 @@ export class MongoCompactorV3 extends MongoCompactor { }; const coveredStart = compactedOpId >= context.lastOp; const concurrentWriteCheck = { $gt: ['$last_op', context.lastOp] }; - const nextAfterConcurrentWrite = new Date(context.startedAt.getTime() + this.minCompactChunkIntervalMs); - const nextCheckForUncompactedWork = { - $min: [ - new Date(firstUncompactedWrite(context.state).getTime() + this.maxCompactFullIntervalMs), - { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.minCompactChunkIntervalMs } } - ] - }; + const nextAfterConcurrentWrite = this.rescheduleAtOrAfter( + new Date(context.startedAt.getTime() + this.minCompactChunkIntervalMs), + context.rescheduleNotBefore + ); + const nextCheckForUncompactedWork = this.rescheduleAtOrAfter( + { + $min: [ + new Date(firstUncompactedWrite(context.state).getTime() + this.maxCompactFullIntervalMs), + { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.minCompactChunkIntervalMs } } + ] + }, + context.rescheduleNotBefore + ); const update: mongo.Document = { compacted_state: { op_id: compactedOpId, @@ -674,7 +691,20 @@ export class MongoCompactorV3 extends MongoCompactor { } private async finalizeSkippedBucket(context: CompactionContext) { - await this.rescheduleClaimedBucket(context.lease, context.decision); + // A maxOpId cap can exclude the first remaining document entirely. Avoid + // immediately claiming the same no-progress bucket again in this run. + await this.rescheduleClaimedBucket(context.lease, { + ...context.decision, + nextCompactCheck: this.rescheduleAtOrAfter( + { + $max: [ + context.decision.nextCompactCheck, + { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.minCompactChunkIntervalMs } } + ] + }, + context.rescheduleNotBefore + ) + }); } private async readBucketStats( diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 638f162ae..7c007d441 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -1048,7 +1048,7 @@ bucket_definitions: const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); expect(state?.compact_lease).toBeUndefined(); - expect(state?.next_compact_check).toBeUndefined(); + expect(state?.next_compact_check).toBeNull(); expect(state?.last_full_compact?.op_id).toBe(1n); }); From 8c92dee26e20c612a19e370ff77cdd65018bffbb Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 13 Aug 2026 11:04:48 +0200 Subject: [PATCH 10/38] Fixes. --- .../implementation/v3/MongoCompactorV3.ts | 31 ++++++++++++--- .../implementation/v3/PersistedBatchV3.ts | 7 +--- .../test/src/storage_compacting.test.ts | 39 ++++++++++++------- 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 866dab02b..330748e2a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -258,7 +258,12 @@ export class MongoCompactorV3 extends MongoCompactor { * initial replication, dueAheadMs extends that boundary to include the first deferred interval. */ private async compactScheduledBuckets(options: ScheduledCompactionOptions = {}) { - const jobStartedAt = new Date(); + // Writers derive next_compact_check from MongoDB's $$NOW. Use the same + // clock for the fixed job boundary so clock skew cannot exclude work at + // the exact initial-replication interval. + const [{ now: jobStartedAt }] = await this.db.db + .aggregate<{ now: Date }>([{ $documents: [{}] }, { $project: { _id: 0, now: '$$NOW' } }]) + .toArray(); const dueBefore = new Date(jobStartedAt.getTime() + (options.dueAheadMs ?? 0)); const forceKind = options.forceKind; const rescheduleNotBefore = new Date(dueBefore.getTime() + 1); @@ -271,15 +276,17 @@ export class MongoCompactorV3 extends MongoCompactor { const scheduled = states.map((state) => ({ state, - decision: this.chooseCompactionKind(state, jobStartedAt) + decision: this.chooseCompactionKind(state, jobStartedAt), + forcedKind: this.forcedCompactionKind(state, forceKind) })); const noOpStates = scheduled.filter( - ({ state, decision }) => state.compact_lease == null && forceKind == null && decision.kind == null + ({ state, decision, forcedKind }) => + state.compact_lease == null && (forceKind == null ? decision.kind : forcedKind) == null ); await this.rescheduleUnclaimedBuckets(noOpStates); - for (const { state, decision } of scheduled) { - const kind = forceKind ?? decision.kind; + for (const { state, decision, forcedKind } of scheduled) { + const kind = forceKind == null ? decision.kind : forcedKind; if (state.compact_lease == null && kind == null) { continue; } @@ -289,7 +296,8 @@ export class MongoCompactorV3 extends MongoCompactor { continue; } const claimedDecision = this.chooseCompactionKind(lease.state, lease.startedAt); - const claimedKind = forceKind ?? claimedDecision.kind; + const claimedKind = + forceKind == null ? claimedDecision.kind : this.forcedCompactionKind(lease.state, forceKind); if (claimedKind == null) { await this.rescheduleClaimedBucket(lease, claimedDecision); } else { @@ -299,6 +307,17 @@ export class MongoCompactorV3 extends MongoCompactor { } } + private forcedCompactionKind( + state: BucketStateDocumentV3, + forceKind: CompactionKind | undefined + ): CompactionKind | null { + if (forceKind == null) { + return null; + } + const maxOpId = this.maxOpIdCap == null || state.last_op < this.maxOpIdCap ? state.last_op : this.maxOpIdCap; + return state.compacted_state == null || state.compacted_state.op_id < maxOpId ? forceKind : null; + } + /** Read a bounded, priority-ordered snapshot of currently claimable scheduled work. */ private async findScheduledBucketBatch(dueBefore: Date): Promise { return this.db diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts index 5719b7084..73f5bbe85 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -399,12 +399,7 @@ export class PersistedBatchV3 extends PersistedBatch { vars: { requested: { $dateAdd: { startDate: '$$NOW', unit: 'minute', amount: 5 } } }, in: { $cond: [ - { - $and: [ - { $ne: ['$next_compact_check', null] }, - { $lt: ['$next_compact_check', '$$requested'] } - ] - }, + { $lt: [{ $ifNull: ['$next_compact_check', '$$requested'] }, '$$requested'] }, '$next_compact_check', '$$requested' ] diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 7c007d441..b6a8addad 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -14,7 +14,7 @@ import { import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; import * as bson from 'bson'; import { describe, expect, test } from 'vitest'; -import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; +import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; describe('Mongo Sync Bucket Storage Compact', () => { register.registerCompactTests(INITIALIZED_MONGO_STORAGE_FACTORY); @@ -58,15 +58,18 @@ describe('Mongo Sync Bucket Storage Compact', () => { return bucketStorage.getCheckpoint(); }; - const setup = async () => { + const setup = async (storageVersion?: number) => { await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); const syncRules = await factory.updateSyncRules( - updateSyncRulesFromYaml(` + updateSyncRulesFromYaml( + ` bucket_definitions: by_user: parameters: select request.user_id() as user_id data: [select * from test where owner_id = bucket.user_id] - `) + `, + { storageVersion } + ) ); const bucketStorage = factory.getInstance(syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; @@ -110,18 +113,21 @@ bucket_definitions: }); }); - test('compactInitialReplication', async () => { + test.each(TEST_STORAGE_VERSIONS)('compactInitialReplication (storage v%s)', async (storageVersion) => { // Populate old replication stream - const { factory } = await setup(); + const { factory } = await setup(storageVersion); // Now populate another replication stream (bucket definition name changed) const syncRules = await factory.updateSyncRules( - updateSyncRulesFromYaml(` + updateSyncRulesFromYaml( + ` bucket_definitions: by_user2: parameters: select request.user_id() as user_id data: [select * from test where owner_id = bucket.user_id] - `) + `, + { storageVersion } + ) ); const bucketStorage = factory.getInstance(syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; @@ -129,22 +135,25 @@ bucket_definitions: await populate(bucketStorage, 2); const { checkpoint } = await bucketStorage.getCheckpoint(); - // The initial lite pass caches and merges every bucket with no prior compact state. + // V3's initial lite pass processes every bucket with no prior compact state. + // Earlier storage versions use the default minimum-change threshold. const result0 = await bucketStorage.compactInitialReplication({ maxOpId: checkpoint }); - expect(result0.buckets).toEqual(2); + expect(result0.buckets).toEqual(storageVersion >= storage.STORAGE_VERSION_3 ? 2 : 0); - // The bucket stats now match the compacted state, so another initial - // pass has no work. + // For V1/V2, lower the threshold to populate the checksum cache. V3 has + // already updated its compacted state, so another initial pass is a no-op. const result1 = await bucketStorage.compactInitialReplication({ - maxOpId: checkpoint + maxOpId: checkpoint, + minBucketChanges: 1 }); - expect(result1.buckets).toEqual(0); + expect(result1.buckets).toEqual(storageVersion >= storage.STORAGE_VERSION_3 ? 0 : 2); // Repeating it stays a no-op. const result2 = await bucketStorage.compactInitialReplication({ - maxOpId: checkpoint + maxOpId: checkpoint, + minBucketChanges: 1 }); expect(result2.buckets).toEqual(0); From 7054e54361ddce4cf8da95d68cb88389453cf1a0 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 13 Aug 2026 11:18:25 +0200 Subject: [PATCH 11/38] Fix lease renewal after retry. --- .../src/storage/implementation/v3/CompactionLease.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts index 6a1094996..f9bc938e5 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts @@ -136,6 +136,11 @@ export class CompactionLease implements AsyncDisposable { if (result.matchedCount != 1 && !this.finalizing) { throw new CompactionLeaseLostError(`Lost compaction lease for bucket ${this.state._id.b}`); } + // A successful renewal confirms that a transient failure has passed. A + // lease-loss error remains sticky: it means another worker may own it. + if (!(this.renewalError instanceof CompactionLeaseLostError)) { + this.renewalError = undefined; + } } private async finish(update: mongo.Document[]) { From 4ac02277376c96dffd5a5397b6cfbbd8bb4655f3 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 13 Aug 2026 11:19:02 +0200 Subject: [PATCH 12/38] Use shared constant for DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS. --- .../src/storage/implementation/v3/MongoCompactorV3.ts | 6 +++--- .../src/storage/implementation/v3/PersistedBatchV3.ts | 11 ++++++++++- .../storage/implementation/v3/compaction-constants.ts | 5 +++++ .../test/src/storage_compacting.test.ts | 3 ++- 4 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v3/compaction-constants.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 330748e2a..0353b7919 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -9,6 +9,7 @@ import { cacheKey } from '../OperationBatch.js'; import { loadBucketDataDocument, maxOpId, serializeBucketData } from './bucket-format.js'; import { BucketDataContextV3 } from './BucketDataContextV3.js'; import { DEFAULT_MAX_DOC_SIZE_BYTES } from './chunking.js'; +import { DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS } from './compaction-constants.js'; import { CompactionLease } from './CompactionLease.js'; import { BucketDataDocumentV3, BucketStateDocumentV3 } from './models.js'; import type { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js'; @@ -83,7 +84,6 @@ interface CompactionResult { bucketStats: BucketStats; } -const DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS = 5 * 60 * 1000; const DEFAULT_MIN_COMPACT_FULL_INTERVAL_MS = 2 * 60 * 60 * 1000; const DEFAULT_MAX_COMPACT_FULL_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000; const DEFAULT_COMPACT_LEASE_DURATION_MS = 10 * 60 * 1000; @@ -202,11 +202,11 @@ export class MongoCompactorV3 extends MongoCompactor { if (this.buckets != null) { await this.compactExplicitBuckets(this.buckets); } else if (this.compactChunksOnly) { - // Writers defer their first chunk-compaction check by minCompactChunkIntervalMs. + // Writers defer their first chunk-compaction check by this fixed default. // Include that interval so this synchronous initial-replication pass // processes the work that existed when it started. await this.compactScheduledBuckets({ - dueAheadMs: this.minCompactChunkIntervalMs, + dueAheadMs: DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS, forceKind: CompactionKind.Chunks }); } else { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts index 73f5bbe85..bc36afd40 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -15,6 +15,7 @@ import { import { SourceRecordLookupState } from '../common/SourceRecordStore.js'; import { serializeBucketData } from './bucket-format.js'; import { chunkBucketData } from './chunking.js'; +import { DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS } from './compaction-constants.js'; import { BucketDataDocumentV3, BucketStateDocumentV3, @@ -396,7 +397,15 @@ export class PersistedBatchV3 extends PersistedBatch { first_uncompacted_write: { $ifNull: ['$first_uncompacted_write', '$$NOW'] }, next_compact_check: { $let: { - vars: { requested: { $dateAdd: { startDate: '$$NOW', unit: 'minute', amount: 5 } } }, + vars: { + requested: { + $dateAdd: { + startDate: '$$NOW', + unit: 'millisecond', + amount: DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS + } + } + }, in: { $cond: [ { $lt: [{ $ifNull: ['$next_compact_check', '$$requested'] }, '$$requested'] }, diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/compaction-constants.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/compaction-constants.ts new file mode 100644 index 000000000..0ab77acf9 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/compaction-constants.ts @@ -0,0 +1,5 @@ +/** + * Writers use this delay before scheduling their first chunk compaction. + * Initial replication includes the same delay when it picks up that work. + */ +export const DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS = 5 * 60 * 1000; diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index b6a8addad..8ecc97f24 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -138,7 +138,8 @@ bucket_definitions: // V3's initial lite pass processes every bucket with no prior compact state. // Earlier storage versions use the default minimum-change threshold. const result0 = await bucketStorage.compactInitialReplication({ - maxOpId: checkpoint + maxOpId: checkpoint, + minCompactChunkIntervalMs: 1 }); expect(result0.buckets).toEqual(storageVersion >= storage.STORAGE_VERSION_3 ? 2 : 0); From 6f5d3d84c9d6fe62a8044c0aff117c1cea28c84d Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 13 Aug 2026 11:20:00 +0200 Subject: [PATCH 13/38] Add test for lease renewal. --- .../test/src/storage_compacting.test.ts | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 8ecc97f24..efa254d9e 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -2,6 +2,7 @@ import { BucketDataDoc } from '@module/storage/implementation/common/BucketDataD import { MongoSyncBucketStorage } from '@module/storage/implementation/createMongoSyncBucketStorage.js'; import { loadBucketDataDocument, serializeBucketData } from '@module/storage/implementation/v3/bucket-format.js'; import { chunkBucketData, DEFAULT_MAX_DOC_SIZE_BYTES } from '@module/storage/implementation/v3/chunking.js'; +import { CompactionLease } from '@module/storage/implementation/v3/CompactionLease.js'; import { BucketDataDocumentV3 } from '@module/storage/implementation/v3/models.js'; import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; import { @@ -13,7 +14,7 @@ import { } from '@powersync/service-core'; import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; import * as bson from 'bson'; -import { describe, expect, test } from 'vitest'; +import { describe, expect, test, vi } from 'vitest'; import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; describe('Mongo Sync Bucket Storage Compact', () => { @@ -328,6 +329,34 @@ bucket_definitions: }); } + test('a successful lease renewal clears a transient renewal error', async () => { + const { bucketStateCollection, ctx } = await setupV3Storage(); + await insertBucketState(bucketStateCollection, ctx.definitionId, 1n); + const lease = await CompactionLease.claim( + bucketStateCollection, + { _id: { d: ctx.definitionId, b: BUCKET } }, + undefined, + 10 * 60 * 1000 + ); + expect(lease).not.toBeNull(); + + try { + const transientError = new Error('temporary MongoDB error'); + const renew = (lease as any).renew.bind(lease); + const updateOne = vi.spyOn(bucketStateCollection, 'updateOne').mockRejectedValueOnce(transientError); + await renew().catch((error: unknown) => { + (lease as any).renewalError = error; + }); + await expect(lease!.throwIfLost()).rejects.toBe(transientError); + + updateOne.mockRestore(); + await renew(); + await expect(lease!.throwIfLost()).resolves.toBeUndefined(); + } finally { + await lease?.[Symbol.asyncDispose](); + } + }); + test('1. ops[] ordering - preserves caller ordering (no implicit sort)', () => { const ops = [ makeBucketDataDoc({ o: 5n, data: '{"id":"c"}' }), From 4b50bba94b7d8b94b5effa1a37b04773f6f51b02 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 13 Aug 2026 11:31:03 +0200 Subject: [PATCH 14/38] Reschedule failed compacts; fix bucket count. --- .../implementation/v1/MongoCompactorV1.ts | 1 + .../implementation/v3/MongoCompactorV3.ts | 64 +++++++++++++----- .../test/src/storage_compacting.test.ts | 65 ++++++++++++++++++- 3 files changed, 112 insertions(+), 18 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index 84117913c..dedcecd96 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -191,6 +191,7 @@ export class MongoCompactorV1 extends MongoCompactor { private async compactSingleBucketRetried(bucket: string, _definitionId: BucketDefinitionId | null = null) { await this.retryCompaction(bucket, () => this.compactSingleBucket(bucket)); + this.compactedBucketCount++; } private async compactDirtyBuckets() { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 0353b7919..a6c077ac8 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -274,11 +274,22 @@ export class MongoCompactorV3 extends MongoCompactor { break; } - const scheduled = states.map((state) => ({ - state, - decision: this.chooseCompactionKind(state, jobStartedAt), - forcedKind: this.forcedCompactionKind(state, forceKind) - })); + const scheduled: { + state: BucketStateDocumentV3; + decision: CompactionDecision; + forcedKind: CompactionKind | null; + }[] = []; + for (const state of states) { + try { + scheduled.push({ + state, + decision: this.chooseCompactionKind(state, jobStartedAt), + forcedKind: this.forcedCompactionKind(state, forceKind) + }); + } catch (error) { + await this.rescheduleFailedBucket(state, rescheduleNotBefore, error); + } + } const noOpStates = scheduled.filter( ({ state, decision, forcedKind }) => state.compact_lease == null && (forceKind == null ? decision.kind : forcedKind) == null @@ -291,17 +302,21 @@ export class MongoCompactorV3 extends MongoCompactor { continue; } - await using lease = await this.claimBucket({ _id: state._id, next_compact_check: { $lte: dueBefore } }); - if (lease == null) { - continue; - } - const claimedDecision = this.chooseCompactionKind(lease.state, lease.startedAt); - const claimedKind = - forceKind == null ? claimedDecision.kind : this.forcedCompactionKind(lease.state, forceKind); - if (claimedKind == null) { - await this.rescheduleClaimedBucket(lease, claimedDecision); - } else { - await this.compactClaimedBucket(lease, claimedKind, claimedDecision, rescheduleNotBefore); + try { + await using lease = await this.claimBucket({ _id: state._id, next_compact_check: { $lte: dueBefore } }); + if (lease == null) { + continue; + } + const claimedDecision = this.chooseCompactionKind(lease.state, lease.startedAt); + const claimedKind = + forceKind == null ? claimedDecision.kind : this.forcedCompactionKind(lease.state, forceKind); + if (claimedKind == null) { + await this.rescheduleClaimedBucket(lease, claimedDecision); + } else { + await this.compactClaimedBucket(lease, claimedKind, claimedDecision, rescheduleNotBefore); + } + } catch (error) { + await this.rescheduleFailedBucket(state, rescheduleNotBefore, error); } } } @@ -353,6 +368,22 @@ export class MongoCompactorV3 extends MongoCompactor { ); } + /** + * Isolate a malformed bucket so it cannot prevent other scheduled buckets + * from compacting. The snapshot filter preserves any concurrent write or + * compactor result instead of overwriting its next check. + */ + private async rescheduleFailedBucket(state: BucketStateDocumentV3, notBefore: Date, error: unknown) { + this.logger.error(`Failed to compact scheduled bucket ${state._id.b}; rescheduling it`, error); + try { + await this.db + .bucketState(this.group_id) + .updateOne(this.unclaimedSnapshotFilter(state), [{ $set: { next_compact_check: notBefore } }]); + } catch (rescheduleError) { + this.logger.error(`Failed to reschedule bucket ${state._id.b} after a compaction error`, rescheduleError); + } + } + /** * This checks that the bucket state hasn't changed by a concurrent write since we checked. * @@ -1036,6 +1067,7 @@ export class MongoCompactorV3 extends MongoCompactor { // --- Finalize: update bucket checksums and state --- await this.finalizeCompactedBucket({ context, compactedOpId, compactionResult: result, puts: putCount }); + this.compactedBucketCount++; this.logger.info(`Compacted bucket ${bucket}: ${totalOpCount} surviving ops`); } diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index efa254d9e..82995de2d 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -139,8 +139,7 @@ bucket_definitions: // V3's initial lite pass processes every bucket with no prior compact state. // Earlier storage versions use the default minimum-change threshold. const result0 = await bucketStorage.compactInitialReplication({ - maxOpId: checkpoint, - minCompactChunkIntervalMs: 1 + maxOpId: checkpoint }); expect(result0.buckets).toEqual(storageVersion >= storage.STORAGE_VERSION_3 ? 2 : 0); @@ -175,6 +174,19 @@ bucket_definitions: }); }); + test('v3 initial chunk compaction includes writer-scheduled work with a custom chunk interval', async () => { + const { bucketStorage, checkpoint } = await setup(storage.STORAGE_VERSION_3); + const result = await (bucketStorage as MongoSyncBucketStorage) + .createMongoCompactor({ + maxOpId: checkpoint, + compactChunksOnly: true, + minCompactChunkIntervalMs: 1 + }) + .compact(); + + expect(result).toBe(2); + }); + test('v3 replication writes initialize scheduled compaction state', async () => { const { bucketStorage } = await setup(); const storageDb = bucketStorage.db; @@ -357,6 +369,55 @@ bucket_definitions: } }); + test('a failed scheduled bucket does not block other scheduled buckets', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3Storage(); + const badBucket = 'bad[]'; + const goodBucket = 'good[]'; + const badDocument = serializeBucketData(badBucket, [ + makeOp(2, 'bad', 'bad', { ...ctx, bucket: badBucket }, sourceTableId) + ]); + const goodDocument = serializeBucketData(goodBucket, [ + makeOp(2, 'good', 'good', { ...ctx, bucket: goodBucket }, sourceTableId) + ]); + await insertDocs(collection, [badDocument, goodDocument]); + await bucketStateCollection.insertMany([ + { + _id: { d: ctx.definitionId, b: badBucket }, + last_op: 2n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { count: 1, bytes: BigInt(badDocument.size), chunks: 1 }, + compacted_state: { + op_id: 1n, + count: 1, + bytes: BigInt(badDocument.size), + chunks: 1, + checksum: 0n, + at: new Date(0) + } + }, + { + _id: { d: ctx.definitionId, b: goodBucket }, + last_op: 2n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { count: 1, bytes: BigInt(goodDocument.size), chunks: 1 } + } + ]); + + await (bucketStorage as MongoSyncBucketStorage) + .createMongoCompactor({ maxOpId: 2n, compactChunksOnly: true }) + .compact(); + + const [badState, goodState] = await Promise.all([ + bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: badBucket } }), + bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: goodBucket } }) + ]); + expect(badState?.next_compact_check).toBeInstanceOf(Date); + expect(badState!.next_compact_check!.getTime()).toBeGreaterThan(0); + expect(goodState?.compacted_state?.op_id).toBe(2n); + }); + test('1. ops[] ordering - preserves caller ordering (no implicit sort)', () => { const ops = [ makeBucketDataDoc({ o: 5n, data: '{"id":"c"}' }), From b8b5ab54ecec08973da4b76c159a32930ae6a9a5 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 13 Aug 2026 11:59:32 +0200 Subject: [PATCH 15/38] Cleanup and minor fixes. --- docs/storage/v3-compaction-design.md | 2 +- .../storage/implementation/MongoCompactor.ts | 2 -- .../implementation/v1/MongoCompactorV1.ts | 2 ++ .../implementation/v3/MongoCompactorV3.ts | 29 ++++++++++--------- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/docs/storage/v3-compaction-design.md b/docs/storage/v3-compaction-design.md index a0be897bf..e0b2a4416 100644 --- a/docs/storage/v3-compaction-design.md +++ b/docs/storage/v3-compaction-design.md @@ -1,6 +1,6 @@ # V3 Compaction Design -This describes the design of compaction scheduling in MongoDB storage V3. For details on what compaction means on a protocol level, see [./compating-operations.md](./compacting-operations.md). +This describes the design of compaction scheduling in MongoDB storage V3. For details on what compaction means on a protocol level, see [./compacting-operations.md](./compacting-operations.md). ## Goals diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index 79d222e06..789313f1e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -52,7 +52,6 @@ export abstract class MongoCompactor { protected readonly moveBatchQueryLimit: number; protected readonly moveBatchByteLimit: number; protected readonly clearBatchLimit: number; - protected readonly maxOpId: bigint; protected readonly buckets: string[] | undefined; protected readonly deleteCheckpointRequestsBefore: Date | undefined; protected readonly signal?: AbortSignal; @@ -76,7 +75,6 @@ export abstract class MongoCompactor { if (this.clearBatchLimit < 2) { throw new ReplicationAssertionError('clearBatchLimit must be >= 2'); } - this.maxOpId = options.maxOpId ?? 0n; this.buckets = options.compactBuckets; this.deleteCheckpointRequestsBefore = options.deleteCheckpointRequestsBefore; this.signal = options.signal; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index dedcecd96..e517e4d6c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -52,11 +52,13 @@ export class MongoCompactorV1 extends MongoCompactor { private bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; private readonly minBucketChanges: number; private readonly minChangeRatio: number; + private readonly maxOpId: bigint; constructor(bucketStorage: MongoSyncBucketStorageV1, db: VersionedPowerSyncMongoV1, options: MongoCompactOptions) { super(bucketStorage, db, options); this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES; this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO; + this.maxOpId = options.maxOpId ?? 0n; } /** diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index a6c077ac8..3f26395d3 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -311,7 +311,7 @@ export class MongoCompactorV3 extends MongoCompactor { const claimedKind = forceKind == null ? claimedDecision.kind : this.forcedCompactionKind(lease.state, forceKind); if (claimedKind == null) { - await this.rescheduleClaimedBucket(lease, claimedDecision); + await this.rescheduleClaimedBucket(lease, claimedDecision, rescheduleNotBefore); } else { await this.compactClaimedBucket(lease, claimedKind, claimedDecision, rescheduleNotBefore); } @@ -456,8 +456,8 @@ export class MongoCompactorV3 extends MongoCompactor { }; } - private async rescheduleClaimedBucket(lease: CompactionLease, decision: CompactionDecision) { - await lease.reschedule(decision.nextCompactCheck); + private async rescheduleClaimedBucket(lease: CompactionLease, decision: CompactionDecision, notBefore?: Date) { + await lease.reschedule(this.rescheduleAtOrAfter(decision.nextCompactCheck, notBefore)); } private rescheduleAtOrAfter(nextCompactCheck: mongo.Document, notBefore: Date | undefined): mongo.Document { @@ -565,8 +565,7 @@ export class MongoCompactorV3 extends MongoCompactor { count: 1, size: 1, target_op: 1, - storage_ref: 1, - has_clear_op: 1 + storage_ref: 1 } } ], @@ -743,18 +742,19 @@ export class MongoCompactorV3 extends MongoCompactor { private async finalizeSkippedBucket(context: CompactionContext) { // A maxOpId cap can exclude the first remaining document entirely. Avoid // immediately claiming the same no-progress bucket again in this run. - await this.rescheduleClaimedBucket(context.lease, { - ...context.decision, - nextCompactCheck: this.rescheduleAtOrAfter( - { + await this.rescheduleClaimedBucket( + context.lease, + { + ...context.decision, + nextCompactCheck: { $max: [ context.decision.nextCompactCheck, { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.minCompactChunkIntervalMs } } ] - }, - context.rescheduleNotBefore - ) - }); + } + }, + context.rescheduleNotBefore + ); } private async readBucketStats( @@ -1206,7 +1206,7 @@ export class MongoCompactorV3 extends MongoCompactor { ): Promise<{ done: boolean; opCountDiff: number }> { const bucket = bucketContext.key.bucket; this.signal?.throwIfAborted(); - const prepared = await this.prepareCompactionUploads(bucket, context, [lastNotPut]); + let prepared: PreparedObjectStorageUpload[] | undefined; let done = false; let opCountDiff = 0; @@ -1281,6 +1281,7 @@ export class MongoCompactorV3 extends MongoCompactor { return; } + prepared ??= await this.prepareCompactionUploads(bucket, context, [lastNotPut]); this.logger.info(`Flushing CLEAR for ${clearedOpCount} ops at ${lastDocId?.o}`); await collection.deleteMany( { From 34c750871bfcb172bfa8681c4fa8b3ba92d3063d Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 13 Aug 2026 12:07:40 +0200 Subject: [PATCH 16/38] Tweak logs. --- .../implementation/v3/MongoCompactorV3.ts | 10 +++++-- .../src/replication/WalStream.ts | 3 +- .../src/replication/wal-budget-utils.ts | 11 +------- .../test/src/wal_budget.test.ts | 28 +------------------ packages/service-core/src/util/utils.ts | 14 ++++++++++ .../service-core/test/src/util/utils.test.ts | 23 +++++++++++++++ 6 files changed, 48 insertions(+), 41 deletions(-) create mode 100644 packages/service-core/test/src/util/utils.test.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 3f26395d3..a0639474f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -1,6 +1,6 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; -import { addChecksums, InternalOpId, storage, utils } from '@powersync/service-core'; +import { addChecksums, formatBytes, InternalOpId, storage, utils } from '@powersync/service-core'; import { BucketDefinitionId } from '@powersync/service-sync-rules'; import { BucketDataDoc } from '../common/BucketDataDoc.js'; import { BucketDataKey } from '../models.js'; @@ -624,7 +624,9 @@ export class MongoCompactorV3 extends MongoCompactor { await this.finalizeCompactedBucket({ context, compactedOpId, compactionResult: result, puts: 0 }); this.compactedBucketCount++; - this.logger.info(`Lightly compacted bucket ${bucket}: ${result.bucketStats.count} ops`); + this.logger.info( + `Compacted bucket chunks ${bucket}: ${result.bucketStats.count} ops, ${result.bucketStats.chunks} chunks, ${formatBytes(result.bucketStats.bytes)}` + ); } private async flushChunkMerge( @@ -1068,7 +1070,9 @@ export class MongoCompactorV3 extends MongoCompactor { await this.finalizeCompactedBucket({ context, compactedOpId, compactionResult: result, puts: putCount }); this.compactedBucketCount++; - this.logger.info(`Compacted bucket ${bucket}: ${totalOpCount} surviving ops`); + this.logger.info( + `Compacted bucket ${bucket}: ${totalOpCount} ops, ${result.bucketStats.chunks} chunks, ${formatBytes(result.bucketStats.bytes)}` + ); } /** diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index 391df874b..d4e0716b3 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -10,6 +10,7 @@ import { } from '@powersync/lib-services-framework'; import { BucketStorageBatch, + formatBytes, getUuidReplicaIdentityBson, MetricsEngine, RelationCache, @@ -47,7 +48,7 @@ import { SimpleSnapshotQuery, SnapshotQuery } from './SnapshotQuery.js'; -import { computeWalBudgetReport, formatBytes, formatWalBudgetLine } from './wal-budget-utils.js'; +import { computeWalBudgetReport, formatWalBudgetLine } from './wal-budget-utils.js'; export interface WalStreamOptions { logger?: Logger; diff --git a/modules/module-postgres/src/replication/wal-budget-utils.ts b/modules/module-postgres/src/replication/wal-budget-utils.ts index 3637824f7..49ac2939d 100644 --- a/modules/module-postgres/src/replication/wal-budget-utils.ts +++ b/modules/module-postgres/src/replication/wal-budget-utils.ts @@ -1,13 +1,4 @@ -export function formatBytes(bytes: number): string { - if (bytes >= 1024 * 1024 * 1024) { - return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`; - } else if (bytes >= 1024 * 1024) { - return `${(bytes / (1024 * 1024)).toFixed(1)}MB`; - } else if (bytes >= 1024) { - return `${(bytes / 1024).toFixed(1)}KB`; - } - return `${bytes}B`; -} +import { formatBytes } from '@powersync/service-core'; export function formatDuration(hours: number): string { if (hours >= 24) { diff --git a/modules/module-postgres/test/src/wal_budget.test.ts b/modules/module-postgres/test/src/wal_budget.test.ts index b4c52235a..83ff64067 100644 --- a/modules/module-postgres/test/src/wal_budget.test.ts +++ b/modules/module-postgres/test/src/wal_budget.test.ts @@ -1,32 +1,6 @@ -import { - computeWalBudgetReport, - formatBytes, - formatDuration, - formatWalBudgetLine -} from '@module/replication/wal-budget-utils.js'; +import { computeWalBudgetReport, formatDuration, formatWalBudgetLine } from '@module/replication/wal-budget-utils.js'; import { describe, expect, test } from 'vitest'; -describe('formatBytes', () => { - test('formats bytes', () => { - expect(formatBytes(500)).toBe('500B'); - }); - - test('formats kilobytes', () => { - expect(formatBytes(1024)).toBe('1.0KB'); - expect(formatBytes(1536)).toBe('1.5KB'); - }); - - test('formats megabytes', () => { - expect(formatBytes(1024 * 1024)).toBe('1.0MB'); - expect(formatBytes(1.5 * 1024 * 1024)).toBe('1.5MB'); - }); - - test('formats gigabytes', () => { - expect(formatBytes(1024 * 1024 * 1024)).toBe('1.0GB'); - expect(formatBytes(10.5 * 1024 * 1024 * 1024)).toBe('10.5GB'); - }); -}); - describe('formatDuration', () => { test('formats minutes', () => { expect(formatDuration(0.5)).toBe('30 minutes'); diff --git a/packages/service-core/src/util/utils.ts b/packages/service-core/src/util/utils.ts index 88f9e3f2f..3d4b76439 100644 --- a/packages/service-core/src/util/utils.ts +++ b/packages/service-core/src/util/utils.ts @@ -309,3 +309,17 @@ export function estimateRowSize(record: sync_rules.ToastableSqliteRow | undefine } return size; } + +export function formatBytes(bytes: number | bigint): string { + if (typeof bytes == 'bigint') { + bytes = Number(bytes); // We round either way + } + if (bytes >= 1024 * 1024 * 1024) { + return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`; + } else if (bytes >= 1024 * 1024) { + return `${(bytes / (1024 * 1024)).toFixed(1)}MB`; + } else if (bytes >= 1024) { + return `${(bytes / 1024).toFixed(1)}KB`; + } + return `${bytes}B`; +} diff --git a/packages/service-core/test/src/util/utils.test.ts b/packages/service-core/test/src/util/utils.test.ts new file mode 100644 index 000000000..b4b47d4c4 --- /dev/null +++ b/packages/service-core/test/src/util/utils.test.ts @@ -0,0 +1,23 @@ +import { formatBytes } from '@/index.js'; +import { describe, expect, test } from 'vitest'; + +describe('formatBytes', () => { + test('formats bytes', () => { + expect(formatBytes(500)).toBe('500B'); + }); + + test('formats kilobytes', () => { + expect(formatBytes(1024)).toBe('1.0KB'); + expect(formatBytes(1536)).toBe('1.5KB'); + }); + + test('formats megabytes', () => { + expect(formatBytes(1024 * 1024)).toBe('1.0MB'); + expect(formatBytes(1.5 * 1024 * 1024)).toBe('1.5MB'); + }); + + test('formats gigabytes', () => { + expect(formatBytes(1024 * 1024 * 1024)).toBe('1.0GB'); + expect(formatBytes(10.5 * 1024 * 1024 * 1024)).toBe('10.5GB'); + }); +}); From 1698ec2215ac1fca4b1267a7c64c6c8a5c35382f Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 13 Aug 2026 12:13:40 +0200 Subject: [PATCH 17/38] Fix sync test. --- .../src/tests/register-sync-tests.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/service-core-tests/src/tests/register-sync-tests.ts b/packages/service-core-tests/src/tests/register-sync-tests.ts index 2bb4c6c75..314684d08 100644 --- a/packages/service-core-tests/src/tests/register-sync-tests.ts +++ b/packages/service-core-tests/src/tests/register-sync-tests.ts @@ -1308,6 +1308,7 @@ bucket_definitions: const bucketStorage = await f.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + const bucket = bucketRequest(syncRules.syncConfigContent[0], 'mybucket[]').bucket; await writer.markAllSnapshotDone('0/1'); await writer.save({ @@ -1386,10 +1387,7 @@ bucket_definitions: await writer.commit('0/2'); - await bucketStorage.compact({ - minBucketChanges: 1, - minChangeRatio: 0 - }); + await bucketStorage.compact({ compactBuckets: [bucket] }); const lines2 = await getCheckpointLines(iter, { consume: true }); @@ -1470,6 +1468,7 @@ bucket_definitions: const bucketStorage = await f.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + const highPriorityBucket = bucketRequest(syncRules.syncConfigContent[0], 'high_priority[]').bucket; await writer.markAllSnapshotDone('0/1'); await writer.save({ @@ -1553,8 +1552,9 @@ bucket_definitions: await writer.commit('0/2'); await bucketStorage.compact({ - minBucketChanges: 1, - minChangeRatio: 0 + // Explicitly compact the high-priority bucket: V3 schedules background + // compaction, while this test needs compaction at this exact point. + compactBuckets: [highPriorityBucket] }); const lines = await getCheckpointLines(iter, { consume: true }); From 2c76691643e37da253262c39c23f04f14e1987c3 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 13 Aug 2026 12:17:37 +0200 Subject: [PATCH 18/38] Changeset. --- .changeset/wacky-impalas-vanish.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .changeset/wacky-impalas-vanish.md diff --git a/.changeset/wacky-impalas-vanish.md b/.changeset/wacky-impalas-vanish.md new file mode 100644 index 000000000..f9e4afd45 --- /dev/null +++ b/.changeset/wacky-impalas-vanish.md @@ -0,0 +1,12 @@ +--- +'@powersync/service-module-mongodb-storage': minor +'@powersync/service-core': minor +'@powersync/service-module-postgres-storage': patch +'@powersync/service-core-tests': patch +'@powersync/service-module-postgres': patch +'@powersync/service-module-mongodb': patch +'@powersync/service-module-convex': patch +'@powersync/service-module-mysql': patch +--- + +Restructure MongoDB V3 bucket compacting. From e56441ab2191521c1c84b573a044e0256e6a17a3 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 14 Aug 2026 09:27:22 +0200 Subject: [PATCH 19/38] Add --incremental-only option. --- .../implementation/v1/MongoCompactorV1.ts | 11 +++- .../src/storage/PostgresSyncRulesStorage.ts | 5 ++ .../src/entry/commands/compact-action.ts | 62 ++++++++++++------- .../src/storage/SyncRulesBucketStorage.ts | 9 +++ 4 files changed, 62 insertions(+), 25 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index e517e4d6c..8fa00db86 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -54,7 +54,11 @@ export class MongoCompactorV1 extends MongoCompactor { private readonly minChangeRatio: number; private readonly maxOpId: bigint; - constructor(bucketStorage: MongoSyncBucketStorageV1, db: VersionedPowerSyncMongoV1, options: MongoCompactOptions) { + constructor( + bucketStorage: MongoSyncBucketStorageV1, + db: VersionedPowerSyncMongoV1, + private options: MongoCompactOptions + ) { super(bucketStorage, db, options); this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES; this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO; @@ -67,6 +71,11 @@ export class MongoCompactorV1 extends MongoCompactor { * See /docs/storage/compacting-operations.md for details. */ override async compact(): Promise { + if (this.options?.incrementalOnly) { + // Not supported for V1 + this.logger.info('Incremental compacting is not supported on MongoDB storage V1/V2'); + return 0; + } await this.deleteOldCheckpointRequests(); if (this.buckets) { diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index 19303de07..e86a41aa5 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -146,6 +146,11 @@ export class PostgresSyncRulesStorage } async compact(options?: storage.CompactOptions): Promise { + if (options?.incrementalOnly) { + // Not supported yet + this.logger.info('Incremental compacting is not supported on Postgres storage yet.'); + return; + } let maxOpId = options?.maxOpId; if (maxOpId == null) { const checkpoint = await this.getCheckpoint(); diff --git a/packages/service-core/src/entry/commands/compact-action.ts b/packages/service-core/src/entry/commands/compact-action.ts index 38b040b59..65cfd7720 100644 --- a/packages/service-core/src/entry/commands/compact-action.ts +++ b/packages/service-core/src/entry/commands/compact-action.ts @@ -5,7 +5,7 @@ import * as v8 from 'v8'; import * as system from '../../system/system-index.js'; import * as utils from '../../util/util-index.js'; -import { modules } from '../../index.js'; +import { modules, SyncRuleState } from '../../index.js'; import { extractRunnerOptions, wrapConfigCommand } from './config-command.js'; const COMMAND_NAME = 'compact'; @@ -28,7 +28,11 @@ export function registerCompactAction(program: Command) { .command(COMMAND_NAME) .option(`-b, --buckets [buckets]`, 'Full bucket names, comma-separated (e.g., "global[],mybucket[\\"user1\\"]")') .option('-p, --parameter-indexes', 'Compacting parameter indexes. Defaults to set unless --buckets is provided.') - .option('--no-parameter-indexes', 'Disabling compacting parameter indexes.'); + .option('--no-parameter-indexes', 'Disabling compacting parameter indexes.') + .option( + '--incremental-only', + '[EXPERIMENTAL] Perform incremental compacting only. Implies --no-parameter-indexes.' + ); wrapConfigCommand(compactCommand); @@ -48,7 +52,12 @@ export function registerCompactAction(program: Command) { } } + const incremental: boolean = options.incrementalOnly ?? false; + let compactParameters: boolean | null = options.parameterIndexes; + if (incremental) { + compactParameters = false; + } if (buckets == null) { logger.info('Compacting storage for all buckets...'); @@ -87,29 +96,34 @@ export function registerCompactAction(program: Command) { Date.now() - config.api_parameters.checkpoint_request_retention_minutes * MINUTE_MS ); - const active = (await bucketStorage.getActiveSyncConfig())?.storage; - if (active == null) { - logger.info('No active instance to compact'); - return; - } - if (buckets != null) { - logger.info('Performing compaction...'); - await active.compact({ - memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, - compactBuckets: buckets, - compactParameterData: compactParameters ?? false, - deleteCheckpointRequestsBefore, - signal: abortController.signal - }); - } else { - await active.compact({ - memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, - compactParameterData: compactParameters ?? true, - deleteCheckpointRequestsBefore, - signal: abortController.signal - }); + const streams = await bucketStorage.getReplicatingReplicationStreams(); + for (let stream of streams) { + if (!incremental && stream.state != SyncRuleState.ACTIVE) { + // Only compact PROCESSING streams if incremental is enabled + continue; + } + const storage = bucketStorage.getInstance(stream); + logger.info(`[${stream.replicationStreamName}] Performing compaction...`); + if (buckets != null) { + await storage.compact({ + memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, + compactBuckets: buckets, + compactParameterData: compactParameters ?? false, + incrementalOnly: incremental, + deleteCheckpointRequestsBefore, + signal: abortController.signal + }); + } else { + await storage.compact({ + memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, + compactParameterData: compactParameters ?? true, + incrementalOnly: incremental, + deleteCheckpointRequestsBefore, + signal: abortController.signal + }); + } + logger.info(`[${stream.replicationStreamName}] Successfully compacted storage.`); } - logger.info('Successfully compacted storage.'); } catch (e) { logger.error(`Failed to compact:`, e); // Indirectly triggers lifeCycleEngine.stop diff --git a/packages/service-core/src/storage/SyncRulesBucketStorage.ts b/packages/service-core/src/storage/SyncRulesBucketStorage.ts index d252b0191..e3087d173 100644 --- a/packages/service-core/src/storage/SyncRulesBucketStorage.ts +++ b/packages/service-core/src/storage/SyncRulesBucketStorage.ts @@ -321,6 +321,15 @@ export interface CompactOptions { compactParameterData?: boolean; + /** + * Only perform compaction that can be done incrementally. + * + * This includes full bucket compaction on MongoDB V3 storage. + * + * On MongoDB v1 and Postgres storage, this makes compacting a no-op. + */ + incrementalOnly?: boolean; + /** * Delete client-requested write checkpoints created before this time. * From 192a4c5e7f0a0559c5f95095ed866e12545154e8 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 14 Aug 2026 09:34:17 +0200 Subject: [PATCH 20/38] Tie compaction of PROCESSING streams to storage version only. --- .../src/storage/implementation/MongoSyncBucketStorage.ts | 3 ++- .../src/storage/implementation/v1/MongoCompactorV1.ts | 4 ++++ .../src/storage/PostgresSyncRulesStorage.ts | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 8a926a7c1..6d3c56d93 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -19,6 +19,7 @@ import { ReplicationCheckpoint, ReplicationStreamStorageIds, storage, + SyncRuleState, utils, WatchWriteCheckpointOptions } from '@powersync/service-core'; @@ -372,7 +373,7 @@ export abstract class MongoSyncBucketStorage } await this.createMongoCompactor({ ...options, maxOpId, logger: this.logger }).compact(); - if (maxOpId != null && options?.compactParameterData) { + if (maxOpId != null && options?.compactParameterData && this.replicationStream.state == SyncRuleState.PROCESSING) { await this.createMongoParameterCompactor(maxOpId, options).compact(); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index 8fa00db86..ffe18bb19 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -6,6 +6,7 @@ import { InternalOpId, isPartialChecksum, storage, + SyncRuleState, utils } from '@powersync/service-core'; import { BucketDefinitionId } from '@powersync/service-sync-rules'; @@ -75,6 +76,9 @@ export class MongoCompactorV1 extends MongoCompactor { // Not supported for V1 this.logger.info('Incremental compacting is not supported on MongoDB storage V1/V2'); return 0; + } else if (this.storage.replicationStream.state != SyncRuleState.PROCESSING) { + this.logger.info(`Skipping compacting of replication stream in ${this.storage.replicationStream.state} state.`); + return 0; } await this.deleteOldCheckpointRequests(); diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index e86a41aa5..d141197a6 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -16,6 +16,7 @@ import { ReplicationCheckpoint, storage, StorageVersionConfig, + SyncRuleState, utils, WatchWriteCheckpointOptions } from '@powersync/service-core'; @@ -150,6 +151,9 @@ export class PostgresSyncRulesStorage // Not supported yet this.logger.info('Incremental compacting is not supported on Postgres storage yet.'); return; + } else if (this.replicationStream.state != SyncRuleState.PROCESSING) { + this.logger.info(`Skipping compacting of replication stream in ${this.replicationStream.state} state.`); + return; } let maxOpId = options?.maxOpId; if (maxOpId == null) { From 30b1d824f22734ae85b5c256c1561d0f06972cc5 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 14 Aug 2026 09:37:08 +0200 Subject: [PATCH 21/38] Update comment. --- .../src/storage/implementation/MongoSyncBucketStorage.ts | 3 ++- .../src/storage/implementation/v1/MongoCompactorV1.ts | 2 +- .../src/storage/PostgresSyncRulesStorage.ts | 2 +- packages/service-core/src/entry/commands/compact-action.ts | 6 +----- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 6d3c56d93..2df749088 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -368,12 +368,13 @@ export abstract class MongoSyncBucketStorage async compact(options?: storage.CompactOptions) { let maxOpId = options?.maxOpId; if (maxOpId == null) { + // For PROCESSING streams, this will be undefined. const checkpoint = await this.getCheckpointInternal(); maxOpId = checkpoint?.checkpoint ?? undefined; } await this.createMongoCompactor({ ...options, maxOpId, logger: this.logger }).compact(); - if (maxOpId != null && options?.compactParameterData && this.replicationStream.state == SyncRuleState.PROCESSING) { + if (maxOpId != null && options?.compactParameterData && this.replicationStream.state == SyncRuleState.ACTIVE) { await this.createMongoParameterCompactor(maxOpId, options).compact(); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index ffe18bb19..f5dffb796 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -76,7 +76,7 @@ export class MongoCompactorV1 extends MongoCompactor { // Not supported for V1 this.logger.info('Incremental compacting is not supported on MongoDB storage V1/V2'); return 0; - } else if (this.storage.replicationStream.state != SyncRuleState.PROCESSING) { + } else if (this.storage.replicationStream.state != SyncRuleState.ACTIVE) { this.logger.info(`Skipping compacting of replication stream in ${this.storage.replicationStream.state} state.`); return 0; } diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index d141197a6..3dafed680 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -151,7 +151,7 @@ export class PostgresSyncRulesStorage // Not supported yet this.logger.info('Incremental compacting is not supported on Postgres storage yet.'); return; - } else if (this.replicationStream.state != SyncRuleState.PROCESSING) { + } else if (this.replicationStream.state != SyncRuleState.ACTIVE) { this.logger.info(`Skipping compacting of replication stream in ${this.replicationStream.state} state.`); return; } diff --git a/packages/service-core/src/entry/commands/compact-action.ts b/packages/service-core/src/entry/commands/compact-action.ts index 65cfd7720..78e82f781 100644 --- a/packages/service-core/src/entry/commands/compact-action.ts +++ b/packages/service-core/src/entry/commands/compact-action.ts @@ -5,7 +5,7 @@ import * as v8 from 'v8'; import * as system from '../../system/system-index.js'; import * as utils from '../../util/util-index.js'; -import { modules, SyncRuleState } from '../../index.js'; +import { modules } from '../../index.js'; import { extractRunnerOptions, wrapConfigCommand } from './config-command.js'; const COMMAND_NAME = 'compact'; @@ -98,10 +98,6 @@ export function registerCompactAction(program: Command) { const streams = await bucketStorage.getReplicatingReplicationStreams(); for (let stream of streams) { - if (!incremental && stream.state != SyncRuleState.ACTIVE) { - // Only compact PROCESSING streams if incremental is enabled - continue; - } const storage = bucketStorage.getInstance(stream); logger.info(`[${stream.replicationStreamName}] Performing compaction...`); if (buckets != null) { From 941f0b30c308d18d18fe3962d6d30aef5e50cddc Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 14 Aug 2026 10:57:45 +0200 Subject: [PATCH 22/38] Fix retries for chunk-merge compacting. --- .../implementation/v3/MongoCompactorV3.ts | 45 +++-- .../test/src/storage_compacting.test.ts | 178 ++++++++++++++++-- 2 files changed, 196 insertions(+), 27 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index a0639474f..ff12f606c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -615,12 +615,34 @@ export class MongoCompactorV3 extends MongoCompactor { return; } - const tailStats = await this.readBucketStats(bucket, resolvedDefinitionId, compactedOpId, tailLowerBound); - const compactedStats = this.combineChunkStats(context.state, tailStats, overlappingCompactedChunk); - const result = { - compactedState: compactedStats, - bucketStats: this.applyCompactionDelta(bucketStats(context.state), preCompactionTail, tailStats) - }; + const previousCompactedState = context.state.compacted_state; + let result: CompactionResult; + if (previousCompactedState != null && overlappingCompactedChunk == null) { + // A previous attempt may have committed document replacements without + // finalizing bucket state. When the exact cached chunk no longer exists, + // the cached prefix cannot be combined with the current tail. Rebuild the + // metadata from the authoritative documents while keeping the old op id + // as a conservative resume hint. + const [compactedState, currentBucketStats] = await Promise.all([ + this.readBucketStats(bucket, resolvedDefinitionId, compactedOpId), + compactedOpId == context.lastOp + ? Promise.resolve(undefined) + : this.readBucketStats(bucket, resolvedDefinitionId, context.lastOp) + ]); + result = { + compactedState, + bucketStats: currentBucketStats ?? compactedState + }; + } else { + const tailStats = await this.readBucketStats(bucket, resolvedDefinitionId, compactedOpId, tailLowerBound); + result = { + compactedState: + previousCompactedState == null || overlappingCompactedChunk == null + ? tailStats + : this.combineChunkStats(previousCompactedState, tailStats, overlappingCompactedChunk), + bucketStats: this.applyCompactionDelta(bucketStats(context.state), preCompactionTail, tailStats) + }; + } await this.finalizeCompactedBucket({ context, compactedOpId, compactionResult: result, puts: 0 }); this.compactedBucketCount++; @@ -804,17 +826,10 @@ export class MongoCompactorV3 extends MongoCompactor { } private combineChunkStats( - state: BucketStateDocumentV3, + previous: NonNullable, compactedTail: BucketStatsWithChecksum, - overlappingCompactedChunk: BucketStatsWithChecksum | undefined + overlappingCompactedChunk: BucketStatsWithChecksum ): BucketStatsWithChecksum { - const previous = state.compacted_state; - if (previous == null) { - return compactedTail; - } - if (overlappingCompactedChunk == null) { - throw new ReplicationAssertionError(`Missing previous compacted chunk for bucket ${state._id.b}`); - } return { count: previous.count - overlappingCompactedChunk.count + compactedTail.count, bytes: previous.bytes - overlappingCompactedChunk.bytes + compactedTail.bytes, diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 82995de2d..720fe279c 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -4,6 +4,7 @@ import { loadBucketDataDocument, serializeBucketData } from '@module/storage/imp import { chunkBucketData, DEFAULT_MAX_DOC_SIZE_BYTES } from '@module/storage/implementation/v3/chunking.js'; import { CompactionLease } from '@module/storage/implementation/v3/CompactionLease.js'; import { BucketDataDocumentV3 } from '@module/storage/implementation/v3/models.js'; +import { ObjectStorageError } from '@module/storage/implementation/v3/object-storage/ObjectStorage.js'; import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; import { addChecksums, @@ -386,15 +387,7 @@ bucket_definitions: last_op: 2n, next_compact_check: new Date(0), first_uncompacted_write: new Date(0), - bucket_stats: { count: 1, bytes: BigInt(badDocument.size), chunks: 1 }, - compacted_state: { - op_id: 1n, - count: 1, - bytes: BigInt(badDocument.size), - chunks: 1, - checksum: 0n, - at: new Date(0) - } + bucket_stats: { count: 1, bytes: BigInt(badDocument.size), chunks: 1 } }, { _id: { d: ctx.definitionId, b: goodBucket }, @@ -405,9 +398,18 @@ bucket_definitions: } ]); - await (bucketStorage as MongoSyncBucketStorage) - .createMongoCompactor({ maxOpId: 2n, compactChunksOnly: true }) - .compact(); + const compactor = (bucketStorage as MongoSyncBucketStorage).createMongoCompactor({ + maxOpId: 2n, + compactChunksOnly: true + }); + const compactSingleBucket = (compactor as any).compactSingleBucket.bind(compactor); + vi.spyOn(compactor as any, 'compactSingleBucket').mockImplementation(async (context: any) => { + if (context.state._id.b == badBucket) { + throw new Error('malformed bucket'); + } + return compactSingleBucket(context); + }); + await compactor.compact(); const [badState, goodState] = await Promise.all([ bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: badBucket } }), @@ -1646,6 +1648,60 @@ bucket_definitions: ); } + async function setupCompactedTail() { + const setup = await setupV3(); + const { collection, bucketStateCollection, ctx, sourceTableId } = setup; + const documents = [ + serializeBucketData(BUCKET, [makeOp(1, 'A', 'a', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(2, 'B', 'b', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(3, 'C', 'c', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(4, 'D', 'd', ctx, sourceTableId)]) + ]; + await insertDocs(collection, documents); + await bucketStateCollection.insertOne({ + _id: { d: ctx.definitionId, b: BUCKET }, + last_op: 4n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + compacted_state: { + op_id: 2n, + checksum: documents[0].checksum + documents[1].checksum, + count: documents[0].count + documents[1].count, + bytes: BigInt(documents[0].size + documents[1].size), + chunks: 2, + at: new Date(0) + }, + bucket_stats: { + count: documents.reduce((total, document) => total + document.count, 0), + bytes: BigInt(documents.reduce((total, document) => total + document.size, 0)), + chunks: documents.length + } + }); + return setup; + } + + async function expectRecoveredCompactedTail(collection: any, bucketStateCollection: any, definitionId: string) { + const documents = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray(); + expect(documents).toHaveLength(2); + expect(documents.flatMap((document: BucketDataDocumentV3) => document.ops!.map((op) => op.o))).toEqual([ + 1n, + 2n, + 3n, + 4n + ]); + + const bytes = BigInt(documents.reduce((total: number, document: BucketDataDocumentV3) => total + document.size, 0)); + const state = await bucketStateCollection.findOne({ _id: { d: definitionId, b: BUCKET } }); + expect(state?.compacted_state).toMatchObject({ + op_id: 4n, + checksum: 70n, + count: 4, + bytes, + chunks: 2 + }); + expect(state?.bucket_stats).toEqual({ count: 4, bytes, chunks: 2 }); + } + test('initial compaction merges small chunks and refreshes bucket metadata', async () => { const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); await insertDocs(collection, [ @@ -1669,6 +1725,104 @@ bucket_definitions: }); }); + test('chunk compaction retry rebuilds state after a committed partial merge', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx } = await setupCompactedTail(); + const compactor = bucketStorage.createMongoCompactor({ + maxOpId: 4n, + compactChunksOnly: true + }); + const originalFlush = (compactor as any).flushCompactionGroup.bind(compactor); + let injectedFailure = false; + vi.spyOn(compactor as any, 'flushCompactionGroup').mockImplementation(async (...args: any[]) => { + const result = await originalFlush(...args); + if (!injectedFailure) { + injectedFailure = true; + throw new ObjectStorageError('failure after committed chunk merge', { + cause: new Error('socket reset'), + retryable: true + }); + } + return result; + }); + + await expect(compactor.compact()).resolves.toBe(1); + + expect(injectedFailure).toBe(true); + await expectRecoveredCompactedTail(collection, bucketStateCollection, ctx.definitionId); + }); + + test('chunk compaction treats a missing cached op as a resume hint', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); + const documents = [ + serializeBucketData(BUCKET, [makeOp(1, 'A', 'a', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(3, 'C', 'c', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(4, 'D', 'd', ctx, sourceTableId)]) + ]; + await insertDocs(collection, documents); + await bucketStateCollection.insertOne({ + _id: { d: ctx.definitionId, b: BUCKET }, + last_op: 4n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + compacted_state: { + op_id: 2n, + checksum: 21n, + count: 2, + bytes: 100n, + chunks: 2, + at: new Date(0) + }, + bucket_stats: { count: 4, bytes: 200n, chunks: 4 } + }); + + await expect(bucketStorage.createMongoCompactor({ maxOpId: 4n, compactChunksOnly: true }).compact()).resolves.toBe( + 1 + ); + + const currentDocuments = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray(); + const bytes = BigInt(currentDocuments.reduce((total, document) => total + document.size, 0)); + const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(state?.compacted_state).toMatchObject({ + op_id: 4n, + checksum: 56n, + count: 3, + bytes, + chunks: 2 + }); + expect(state?.bucket_stats).toEqual({ count: 3, bytes, chunks: 2 }); + }); + + test('later chunk compaction rebuilds state after a committed partial merge', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx } = await setupCompactedTail(); + const failedCompactor = bucketStorage.createMongoCompactor({ + maxOpId: 4n, + compactChunksOnly: true + }); + const originalFlush = (failedCompactor as any).flushCompactionGroup.bind(failedCompactor); + const flushSpy = vi + .spyOn(failedCompactor as any, 'flushCompactionGroup') + .mockImplementation(async (...args: any[]) => { + const result = await originalFlush(...args); + throw new Error('failure after committed chunk merge'); + }); + + await expect(failedCompactor.compact()).resolves.toBe(0); + flushSpy.mockRestore(); + + const interruptedState = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(interruptedState?.compacted_state?.op_id).toBe(2n); + expect(await collection.findOne({ _id: { b: BUCKET, o: 2n } })).toBeNull(); + await bucketStateCollection.updateOne( + { _id: { d: ctx.definitionId, b: BUCKET } }, + { $set: { next_compact_check: new Date(0) } } + ); + + const result = await bucketStorage.createMongoCompactor({ maxOpId: 4n, compactChunksOnly: true }).compact(); + + expect(result).toBe(1); + await expectRecoveredCompactedTail(collection, bucketStateCollection, ctx.definitionId); + }); + test('1. multi-batch compaction preserves checksum and creates MOVE tombstones', async () => { const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); From 9a58b627523802fb4ac99f743728296a286e8fa1 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 14 Aug 2026 12:39:50 +0200 Subject: [PATCH 23/38] Fix tests. --- .../src/tests/register-compacting-tests.ts | 16 ++++++++-------- .../tests/register-parameter-compacting-tests.ts | 5 +++-- .../src/tests/register-sync-tests.ts | 5 +++-- packages/service-core-tests/src/tests/util.ts | 13 +++++++++++++ 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/packages/service-core-tests/src/tests/register-compacting-tests.ts b/packages/service-core-tests/src/tests/register-compacting-tests.ts index a30a6cca9..916952855 100644 --- a/packages/service-core-tests/src/tests/register-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-compacting-tests.ts @@ -2,7 +2,7 @@ import { addChecksums, storage, updateSyncRulesFromYaml } from '@powersync/servi import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; import { bucketRequest } from '../test-utils/test-utils-index.js'; -import { bucketRequestMap, bucketRequests } from './util.js'; +import { bucketRequestMap, bucketRequests, compactActive } from './util.js'; export function registerCompactTests(config: storage.TestStorageConfig) { const generateStorageFactory = config.factory; @@ -81,7 +81,7 @@ bucket_definitions: ]); expect(batchBefore.targetOp).toEqual(null); - await bucketStorage.compact({ + await compactActive(factory, { clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1, @@ -202,7 +202,7 @@ bucket_definitions: } ]); - await bucketStorage.compact({ + await compactActive(factory, { clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1, @@ -301,7 +301,7 @@ bucket_definitions: await writer2.flush(); const checkpoint2 = writer2.last_flushed_op!; - await bucketStorage.compact({ + await compactActive(factory, { clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1, @@ -416,7 +416,7 @@ bucket_definitions: const checkpoint = writer.last_flushed_op!; - await bucketStorage.compact({ + await compactActive(factory, { clearBatchLimit: 100, moveBatchLimit: 100, moveBatchQueryLimit: 100, // Larger limit for a larger window of operations @@ -507,7 +507,7 @@ bucket_definitions: await writer.commit('1/1'); await writer.flush(); - await bucketStorage.compact({ + await compactActive(factory, { clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1, @@ -595,7 +595,7 @@ bucket_definitions: await writer2.commit('2/1'); await writer2.flush(); - await bucketStorage.compact({ + await compactActive(factory, { clearBatchLimit: 20, moveBatchLimit: 10, moveBatchQueryLimit: 10, @@ -658,7 +658,7 @@ bucket_definitions: expect(checkpointBeforeCompact.checkpoint).toEqual(checkpoint1); // With default options, Postgres compaction should use the active checkpoint. - await bucketStorage.compact({ + await compactActive(factory, { moveBatchLimit: 1, moveBatchQueryLimit: 1, minBucketChanges: 1, diff --git a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts index 2783260c9..ff67ada62 100644 --- a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts @@ -2,6 +2,7 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { ScopedParameterLookup } from '@powersync/service-sync-rules'; import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; +import { compactActive } from './util.js'; export function registerParameterCompactTests(config: storage.TestStorageConfig) { const generateStorageFactory = config.factory; @@ -74,7 +75,7 @@ bucket_definitions: expect(parameters2).toEqual([]); const statsBefore = await bucketStorage.factory.getStorageMetrics(); - await bucketStorage.compact({ compactParameterData: true }); + await compactActive(factory, { compactParameterData: true }); // Check consistency const parameters1b = await checkpoint1.getParameterSets([lookup], 1000); @@ -154,7 +155,7 @@ bucket_definitions: expect(parameters1).toEqual([]); const statsBefore = await bucketStorage.factory.getStorageMetrics(); - await bucketStorage.compact({ compactParameterData: true, compactParameterCacheLimit: cacheLimit }); + await compactActive(factory, { compactParameterData: true, compactParameterCacheLimit: cacheLimit }); // Check consistency const parameters1b = await checkpoint1.getParameterSets([lookup], 1000); diff --git a/packages/service-core-tests/src/tests/register-sync-tests.ts b/packages/service-core-tests/src/tests/register-sync-tests.ts index 314684d08..f3387f1b3 100644 --- a/packages/service-core-tests/src/tests/register-sync-tests.ts +++ b/packages/service-core-tests/src/tests/register-sync-tests.ts @@ -15,6 +15,7 @@ import { fileURLToPath } from 'url'; import { expect, test, vi } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; import { bucketRequest, METRICS_HELPER } from '../test-utils/test-utils-index.js'; +import { compactActive } from './util.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -1387,7 +1388,7 @@ bucket_definitions: await writer.commit('0/2'); - await bucketStorage.compact({ compactBuckets: [bucket] }); + await compactActive(f, { compactBuckets: [bucket] }); const lines2 = await getCheckpointLines(iter, { consume: true }); @@ -1551,7 +1552,7 @@ bucket_definitions: }); await writer.commit('0/2'); - await bucketStorage.compact({ + await compactActive(f, { // Explicitly compact the high-priority bucket: V3 schedules background // compaction, while this test needs compaction at this exact point. compactBuckets: [highPriorityBucket] diff --git a/packages/service-core-tests/src/tests/util.ts b/packages/service-core-tests/src/tests/util.ts index 9e5c99e83..f70d19df3 100644 --- a/packages/service-core-tests/src/tests/util.ts +++ b/packages/service-core-tests/src/tests/util.ts @@ -8,6 +8,19 @@ import { } from '@powersync/service-sync-rules'; import { bucketRequest } from '../test-utils/general-utils.js'; +/** + * Resolve storage again after test writes activate the sync config. The storage + * instance used by the writer retains its original PROCESSING stream snapshot, + * while compact() expects an instance constructed with the current state. + */ +export async function compactActive(factory: storage.BucketStorageFactory, options: storage.CompactOptions) { + const active = await factory.getActiveSyncConfig(); + if (active == null) { + throw new Error('Expected an active sync config before compacting'); + } + await active.storage.compact(options); +} + export function bucketRequestMap( syncRules: storage.PersistedSyncConfigContent, buckets: Iterable From c60abe283fee6c2992f99e3e13d744b62f70a238 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 14 Aug 2026 12:59:07 +0200 Subject: [PATCH 24/38] Add notes on backwards-compatibility for agents. --- AGENTS.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 6363cbaf8..005732503 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,3 +66,12 @@ After loading the relevant spec context, inspect the closest existing implementa - Import real test storage factories from storage modules, such as `@powersync/service-module-mongodb-storage` and `@powersync/service-module-postgres-storage`, and use `describeWithStorage`-style coverage where practical. - Add a stream test context for new modules, following existing examples such as `WalStreamTestContext`, `ChangeStreamTestContext`, `BinlogStreamTestContext`, `CDCStreamTestContext`, or `ConvexStreamTestContext`. - For when to use spies versus mocks, follow the General Workflow testing guidance above. + +### Backwards-compatibility + +The NPM packages here do not follow semver: We do not guarantee any backwards-compatibility on package APIs. Backwards-compatibility is only relevant for the service itself. + +For storage: + +1. We preserve backwards-compatibility for Postres storage and MongoDB storage with storage_version: 1 and 2, and future even versions. +2. We do not preserve backwards-compatibility for MongoDB storage with storage_version: 3, or any future odd version numbers. From e4c1a6b7a427a65cb62d44d2596c3cc7bea92dd0 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 14 Aug 2026 15:57:47 +0200 Subject: [PATCH 25/38] Further test fixes. --- modules/module-mongodb-storage/test/src/storage.test.ts | 4 ++-- modules/module-postgres-storage/test/src/storage.test.ts | 9 +++++++-- .../test/src/storage_compacting.test.ts | 4 ++-- .../src/tests/register-data-storage-checkpoint-tests.ts | 4 +++- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/modules/module-mongodb-storage/test/src/storage.test.ts b/modules/module-mongodb-storage/test/src/storage.test.ts index 5606af296..26adfaea2 100644 --- a/modules/module-mongodb-storage/test/src/storage.test.ts +++ b/modules/module-mongodb-storage/test/src/storage.test.ts @@ -1,7 +1,7 @@ import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js'; import { mongo } from '@powersync/lib-service-mongodb'; import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { register, test_utils } from '@powersync/service-core-tests'; +import { compactActive, register, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { env } from './env.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; @@ -115,7 +115,7 @@ bucket_definitions: await managedWriter.markAllSnapshotDone('1/1'); await managedWriter.keepalive('8/0'); - await bucketStorage.compact({ + await compactActive(factory, { compactBuckets: [], deleteCheckpointRequestsBefore: new Date('2024-02-01T00:00:00.000Z') }); diff --git a/modules/module-postgres-storage/test/src/storage.test.ts b/modules/module-postgres-storage/test/src/storage.test.ts index 0620a2b4a..725967838 100644 --- a/modules/module-postgres-storage/test/src/storage.test.ts +++ b/modules/module-postgres-storage/test/src/storage.test.ts @@ -1,5 +1,5 @@ import { framework, storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, compactActive, register, test_utils } from '@powersync/service-core-tests'; import * as t from 'ts-codec'; import { describe, expect, test } from 'vitest'; import { CLEAR_BATCH_LIMIT } from '../../src/storage/PostgresSyncRulesStorage.js'; @@ -105,7 +105,12 @@ bucket_definitions: WHERE user_id = 'user2' `.execute(); - await bucketStorage.compact({ + { + await using activationWriter = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await activationWriter.markAllSnapshotDone('1/1'); + await activationWriter.keepalive('1/1'); + } + await compactActive(factory, { compactBuckets: [], deleteCheckpointRequestsBefore: new Date('2024-02-01T00:00:00.000Z') }); diff --git a/modules/module-postgres-storage/test/src/storage_compacting.test.ts b/modules/module-postgres-storage/test/src/storage_compacting.test.ts index 43d4132ae..b6ba586c0 100644 --- a/modules/module-postgres-storage/test/src/storage_compacting.test.ts +++ b/modules/module-postgres-storage/test/src/storage_compacting.test.ts @@ -1,5 +1,5 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, compactActive, register, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { PostgresCompactor } from '../../src/storage/PostgresCompactor.js'; import { POSTGRES_STORAGE_FACTORY } from './util.js'; @@ -44,7 +44,7 @@ bucket_definitions: // Compact with an explicit bucket name — exercises the this.buckets // iteration path, NOT the compactAllBuckets discovery path. - await bucketStorage.compact({ + await compactActive(factory, { compactBuckets: [bucketRequest(syncRulesContent, 'global[]').bucket], minBucketChanges: 1 }); diff --git a/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts index 93fc8a0d8..f7ba779b7 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts @@ -1,6 +1,7 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; +import { compactActive } from './util.js'; /** * @example @@ -399,11 +400,12 @@ bucket_definitions: checkpoint_requested_at: new Date('2024-01-01T00:00:00.000Z') }); await writer.flush(); + await writer.keepalive('1/1'); await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'persistent' })).resolves.toEqual(5n); await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'temporary' })).resolves.toEqual(6n); - await bucketStorage.compact({ + await compactActive(factory, { compactBuckets: [], deleteCheckpointRequestsBefore: new Date(Date.now() + 1_000) }); From 4ed22d41b02cf05d1ac683d6641bba8ed29a459b Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 14 Aug 2026 16:14:22 +0200 Subject: [PATCH 26/38] Further test tweaks. --- .../test/src/storage_compacting.test.ts | 4 ++-- .../test/src/storage_s3_checksums.test.ts | 4 ++-- .../src/storage_s3_compaction_lifecycle.test.ts | 12 ++++++------ modules/module-postgres/test/src/slow_tests.test.ts | 13 +++++++++---- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 720fe279c..31fdea5d7 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -13,7 +13,7 @@ import { SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, compactActive, register, test_utils } from '@powersync/service-core-tests'; import * as bson from 'bson'; import { describe, expect, test, vi } from 'vitest'; import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; @@ -90,7 +90,7 @@ bucket_definitions: // Simulate a V1 deployment which pre-dates bucket-state population. await factory.db.bucket_state.deleteMany({}); - await bucketStorage.compact({ + await compactActive(factory, { clearBatchLimit: 200, moveBatchLimit: 10, moveBatchQueryLimit: 10, diff --git a/modules/module-mongodb-storage/test/src/storage_s3_checksums.test.ts b/modules/module-mongodb-storage/test/src/storage_s3_checksums.test.ts index 44e6b6a04..aaaa8ee8d 100644 --- a/modules/module-mongodb-storage/test/src/storage_s3_checksums.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_s3_checksums.test.ts @@ -4,7 +4,7 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, compactActive, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js'; import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; @@ -153,7 +153,7 @@ describe('V3 checksums with S3 object storage', () => { const checkpoint = await bucketStorage.getCheckpoint(); // Compact — produces CLEAR doc from collapsed MOVEs - await bucketStorage.compact({ + await compactActive(factory, { maxOpId: checkpoint.checkpoint, compactBuckets: [request.bucket], clearBatchLimit: 200, diff --git a/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts b/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts index d597295ff..34c3400a3 100644 --- a/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts @@ -1,5 +1,5 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, compactActive, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js'; import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; @@ -63,7 +63,7 @@ describe('S3 compaction storage lifecycle', () => { const checkpoint = await bucketStorage.getCheckpoint(); const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n); - await bucketStorage.compact({ + await compactActive(factory, { maxOpId: checkpoint.checkpoint, compactBuckets: [request.bucket], minBucketChanges: 1, @@ -107,7 +107,7 @@ describe('S3 compaction storage lifecycle', () => { const checkpoint = await bucketStorage.getCheckpoint(); const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n); - await bucketStorage.compact({ + await compactActive(factory, { maxOpId: checkpoint.checkpoint, compactBuckets: [request.bucket], minBucketChanges: 1, @@ -167,7 +167,7 @@ describe('S3 compaction storage lifecycle', () => { const checkpoint = await bucketStorage.getCheckpoint(); const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n); - await bucketStorage.compact({ + await compactActive(factory, { maxOpId: checkpoint.checkpoint, compactBuckets: [request.bucket], clearBatchLimit: 200, @@ -230,7 +230,7 @@ describe('S3 compaction storage lifecycle', () => { const lowerPaths = new Set(docsBefore.slice(0, 6).map((doc) => doc.storage_ref!.path)); const upperPaths = new Set(docsBefore.slice(6).map((doc) => doc.storage_ref!.path)); - await bucketStorage.compact({ + await compactActive(factory, { maxOpId, compactBuckets: [request.bucket], clearBatchLimit: 200, @@ -357,7 +357,7 @@ describe('S3 compaction storage lifecycle', () => { expect(expectedCompactedOpId).not.toBeNull(); // Compact the bucket. - await bucketStorage.compact({ + await compactActive(factory, { maxOpId: checkpoint.checkpoint, compactBuckets: [bucket], clearBatchLimit: 200, diff --git a/modules/module-postgres/test/src/slow_tests.test.ts b/modules/module-postgres/test/src/slow_tests.test.ts index d8e37300a..832508549 100644 --- a/modules/module-postgres/test/src/slow_tests.test.ts +++ b/modules/module-postgres/test/src/slow_tests.test.ts @@ -183,10 +183,15 @@ bucket_definitions: break; } - const checkpoint = await storage.getCheckpoint(); - const opsBefore = await helpers.getBucketData('global[]', checkpoint); - await storage.compact({ maxOpId: checkpoint.checkpoint }); - const opsAfter = await helpers.getBucketData('global[]', checkpoint); + const active = await f.getActiveSyncConfig(); + if (active == null) { + continue; + } + const activeHelpers = new StorageDataHelpers(active.storage, syncRulesContent); + const checkpoint = await active.storage.getCheckpoint(); + const opsBefore = await activeHelpers.getBucketData('global[]', checkpoint); + await active.storage.compact({ maxOpId: checkpoint.checkpoint }); + const opsAfter = await activeHelpers.getBucketData('global[]', checkpoint); test_utils.validateCompactedBucket(opsBefore, opsAfter); } From b194b9ab6cd687c5eabf9a46aa98277c5a3d1d97 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 17 Aug 2026 09:19:36 +0200 Subject: [PATCH 27/38] Ignore expired leases where filtering for leases. --- .../implementation/v3/CompactionLease.ts | 22 ++++++++++--------- .../implementation/v3/MongoCompactorV3.ts | 8 +++---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts index f9bc938e5..6dc4dc1c4 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts @@ -5,6 +5,17 @@ import { BucketStateDocumentV3 } from './models.js'; const LEASE_RENEW_INTERVAL_MS = 60 * 1000; +/** + * Filters for buckets that have no lease or an expired lease. + */ +export const AVAILABLE_LEASE_EXPR = { + // $$NOW is evaluated by MongoDB, avoiding lease expiry races due + // to clocks on separate compact workers. + $expr: { + $or: [{ $eq: [{ $type: '$compact_lease' }, 'missing'] }, { $lte: ['$compact_lease.expires_at', '$$NOW'] }] + } +}; + /** * Owns one V3 bucket-compaction lease, including its server-time renewal and * the owner-fenced operations which release it. @@ -42,16 +53,7 @@ export class CompactionLease implements AsyncDisposable { const id = new mongo.ObjectId(); const state = await collection.findOneAndUpdate( { - $and: [ - filter, - { - // $$NOW is evaluated by MongoDB, avoiding lease expiry races due - // to clocks on separate compact workers. - $expr: { - $or: [{ $eq: [{ $type: '$compact_lease' }, 'missing'] }, { $lte: ['$compact_lease.expires_at', '$$NOW'] }] - } - } - ] + $and: [filter, AVAILABLE_LEASE_EXPR] }, [ { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index ff12f606c..a5ecb288c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -10,7 +10,7 @@ import { loadBucketDataDocument, maxOpId, serializeBucketData } from './bucket-f import { BucketDataContextV3 } from './BucketDataContextV3.js'; import { DEFAULT_MAX_DOC_SIZE_BYTES } from './chunking.js'; import { DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS } from './compaction-constants.js'; -import { CompactionLease } from './CompactionLease.js'; +import { AVAILABLE_LEASE_EXPR, CompactionLease } from './CompactionLease.js'; import { BucketDataDocumentV3, BucketStateDocumentV3 } from './models.js'; import type { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js'; import { BucketDataObjectStorage, hydrateBucketDataDocuments } from './object-storage/BucketDataObjectStorage.js'; @@ -339,9 +339,7 @@ export class MongoCompactorV3 extends MongoCompactor { .bucketState(this.group_id) .find({ next_compact_check: { $lte: dueBefore }, - $expr: { - $or: [{ $eq: [{ $type: '$compact_lease' }, 'missing'] }, { $lte: ['$compact_lease.expires_at', '$$NOW'] }] - } + ...AVAILABLE_LEASE_EXPR }) .sort({ next_compact_check: 1 }) .limit(SCHEDULED_COMPACTION_BATCH_SIZE) @@ -398,7 +396,7 @@ export class MongoCompactorV3 extends MongoCompactor { bucket_stats: state.bucket_stats, compacted_state: state.compacted_state ?? { $exists: false }, last_full_compact: state.last_full_compact ?? { $exists: false }, - compact_lease: { $exists: false } + ...AVAILABLE_LEASE_EXPR }; } From d59cca3d9a294ad08e98269d4f64642ec7226539 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 17 Aug 2026 09:22:53 +0200 Subject: [PATCH 28/38] Use MongoDB timestamps for locks everywhere. --- .../implementation/MongoSyncRulesLock.ts | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts index 928c04224..527eeda9d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts @@ -5,6 +5,8 @@ import { ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framewo import { storage } from '@powersync/service-core'; import { VersionedPowerSyncMongo } from './db.js'; +const LOCK_DURATION_MS = 60 * 1000; + /** * Manages a lock on a replication stream document, so that only one process * processes that replication stream at a time. @@ -22,15 +24,20 @@ export class MongoSyncRulesLock implements storage.ReplicationLock { ): Promise { const lockId = crypto.randomBytes(8).toString('hex'); const doc = await db.sync_rules.findOneAndUpdate( - { _id: sync_rules.replicationStreamId, $or: [{ lock: null }, { 'lock.expires_at': { $lt: new Date() } }] }, { - $set: { - lock: { - id: lockId, - expires_at: new Date(Date.now() + 60 * 1000) + _id: sync_rules.replicationStreamId, + $or: [{ lock: null }, { $expr: { $lt: ['$lock.expires_at', '$$NOW'] } }] + }, + [ + { + $set: { + lock: { + id: lockId, + expires_at: { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: LOCK_DURATION_MS } } + } } } - }, + ], { projection: { lock: 1 }, returnDocument: 'before', @@ -96,9 +103,15 @@ export class MongoSyncRulesLock implements storage.ReplicationLock { _id: this.sync_rules_id, 'lock.id': this.lock_id }, - { - $set: { 'lock.expires_at': new Date(Date.now() + 60 * 1000) } - }, + [ + { + $set: { + 'lock.expires_at': { + $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: LOCK_DURATION_MS } + } + } + } + ], { returnDocument: 'after' } ); if (result == null) { From 032e615c01f11f55385e9dfa35384738bd6e47b7 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 17 Aug 2026 09:35:22 +0200 Subject: [PATCH 29/38] Move stateless functions to a separate utilities module. --- .../implementation/v3/MongoCompactorV3.ts | 333 +++--------------- .../implementation/v3/compact-utils.ts | 297 ++++++++++++++++ .../test/src/compact-utils.test.ts | 208 +++++++++++ .../test/src/storage_compacting.test.ts | 29 +- 4 files changed, 557 insertions(+), 310 deletions(-) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v3/compact-utils.ts create mode 100644 modules/module-mongodb-storage/test/src/compact-utils.test.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index a5ecb288c..e633d8a6d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -9,6 +9,28 @@ import { cacheKey } from '../OperationBatch.js'; import { loadBucketDataDocument, maxOpId, serializeBucketData } from './bucket-format.js'; import { BucketDataContextV3 } from './BucketDataContextV3.js'; import { DEFAULT_MAX_DOC_SIZE_BYTES } from './chunking.js'; +import { + applyCompactionDelta, + bucketStats, + BucketStatsWithChecksum, + chooseCompactionKind, + combineAdjacentStats, + combineChunkStats, + CompactIntervalConfig, + CompactionContext, + CompactionDecision, + CompactionKind, + CompactionResult, + CompactTargetConfig, + emptyBucketStats, + firstUncompactedWrite, + forcedCompactionKind, + PendingCompactionGroup, + readCompactionBatch, + ScheduledCompactionOptions, + statsForDocument, + unclaimedSnapshotFilter +} from './compact-utils.js'; import { DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS } from './compaction-constants.js'; import { AVAILABLE_LEASE_EXPR, CompactionLease } from './CompactionLease.js'; import { BucketDataDocumentV3, BucketStateDocumentV3 } from './models.js'; @@ -17,166 +39,20 @@ import { BucketDataObjectStorage, hydrateBucketDataDocuments } from './object-st import { ObjectStorageLifecycle, PreparedObjectStorageUpload } from './object-storage/ObjectStorageLifecycle.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; -interface PendingCompactionGroup { - /** - * Input documents are ordered from oldest to newest, matching `ops`. - * Keeping the inputs intact lets unchanged singletons retain their object. - */ - inputs: BucketDataDocumentV3[]; - ops: BucketDataDoc[]; - changed: boolean; - targetOp: InternalOpId | null; -} - -enum CompactionKind { - Full = 'full', - Chunks = 'chunks' -} - -interface CompactionDecision { - kind: CompactionKind | null; - nextCompactCheck: mongo.Document; -} - -interface ScheduledCompactionOptions { - /** Process checks scheduled this far after the captured job start. */ - dueAheadMs?: number; - /** Used by initial replication, which must not run a full compact. */ - forceKind?: CompactionKind; -} - -class CompactionContext { - constructor( - readonly lease: CompactionLease, - readonly kind: CompactionKind, - readonly decision: CompactionDecision, - readonly rescheduleNotBefore: Date | undefined - ) {} - - get state() { - return this.lease.state; - } - - get startedAt() { - return this.lease.startedAt; - } - - get lastOp() { - return this.lease.lastOp; - } -} - -interface BucketStats { - count: number; - bytes: bigint; - chunks: number; -} - -/** Bucket stats read from bucket-data documents, including their checksum. */ -interface BucketStatsWithChecksum extends BucketStats { - checksum: number; -} - -interface CompactionResult { - /** Metadata cached at compacted_state.op_id. */ - compactedState: BucketStatsWithChecksum; - /** Complete bucket metadata through the op head captured at claim time. */ - bucketStats: BucketStats; -} - const DEFAULT_MIN_COMPACT_FULL_INTERVAL_MS = 2 * 60 * 60 * 1000; const DEFAULT_MAX_COMPACT_FULL_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000; const DEFAULT_COMPACT_LEASE_DURATION_MS = 10 * 60 * 1000; -const FULL_COMPACT_RESCHEDULE_MARGIN_MS = 60 * 1000; const SCHEDULED_COMPACTION_BATCH_SIZE = 100; -/** - * Perform chunk compaction if at least this many chunks have been added since - * the latest compaction. - * - * A lower value increases compaction frequency and can increase chunk rewrites - * and object-storage operations. A higher value leaves more small chunks for - * longer, which can hurt sync performance. - */ -const MERGE_CHUNKS_THRESHOLD = 8; - -/** - * Read one bounded prefix from a compaction cursor. - * - * The document that would cross the byte limit is deliberately not returned: - * pagination resumes past the last returned `_id`, so that document remains - * eligible for the next query. The first document is always accepted to - * ensure progress when a single document exceeds the configured byte limit. - * - * `hasMore` is conservative when the document limit is reached. An extra empty - * query is preferable to exhausting the cursor just to determine whether the - * limited MongoDB query contained another document. - */ -async function readCompactionBatch( - cursor: mongo.AggregationCursor, - options: { byteLimit: number; documentLimit: number } -): Promise<{ documents: BucketDataDocumentV3[]; hasMore: boolean }> { - const documents: BucketDataDocumentV3[] = []; - let cumulativeBytes = 0; - - try { - for await (const document of cursor) { - if (documents.length > 0 && cumulativeBytes + document.size > options.byteLimit) { - return { documents, hasMore: true }; - } - - documents.push(document); - cumulativeBytes += document.size; - - if (documents.length >= options.documentLimit) { - return { documents, hasMore: true }; - } - } - return { documents, hasMore: false }; - } finally { - await cursor.close(); - } -} - -function bucketStats(state: BucketStateDocumentV3): BucketStats { - return { - count: state.bucket_stats.count, - bytes: state.bucket_stats.bytes, - chunks: state.bucket_stats.chunks - }; -} - -function emptyBucketStats(): BucketStatsWithChecksum { - return { count: 0, bytes: 0n, chunks: 0, checksum: 0 }; -} - -function statsForDocument( - document: Pick -): BucketStatsWithChecksum { - return { - count: document.count, - bytes: BigInt(document.size), - chunks: 1, - checksum: addChecksums(0, Number(document.checksum)) - }; -} -/** A scheduled bucket always has writes awaiting a full compact. */ -function firstUncompactedWrite(state: BucketStateDocumentV3): Date { - if (state.first_uncompacted_write == null) { - throw new ReplicationAssertionError(`Scheduled V3 bucket ${state._id.b} has no first uncompacted write`); - } - return state.first_uncompacted_write; -} - -export class MongoCompactorV3 extends MongoCompactor { +export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalConfig, CompactTargetConfig { declare protected readonly db: VersionedPowerSyncMongoV3; declare protected readonly storage: MongoSyncBucketStorageV3; - private readonly minCompactChunkIntervalMs: number; - private readonly minCompactFullIntervalMs: number; - private readonly maxCompactFullIntervalMs: number; - private readonly compactLeaseDurationMs: number; - private readonly maxOpIdCap: InternalOpId | undefined; + readonly minCompactChunkIntervalMs: number; + readonly minCompactFullIntervalMs: number; + readonly maxCompactFullIntervalMs: number; + readonly compactLeaseDurationMs: number; + readonly maxOpIdCap: InternalOpId | undefined; constructor(bucketStorage: MongoSyncBucketStorageV3, db: VersionedPowerSyncMongoV3, options: storage.CompactOptions) { super(bucketStorage, db, options); @@ -235,7 +111,7 @@ export class MongoCompactorV3 extends MongoCompactor { if (lease == null || lease.state.first_uncompacted_write == null) { continue; } - const decision = this.chooseCompactionKind(lease.state, lease.startedAt); + const decision = chooseCompactionKind(lease.state, lease.startedAt, this); await this.compactClaimedBucket(lease, CompactionKind.Full, decision); } } @@ -283,8 +159,8 @@ export class MongoCompactorV3 extends MongoCompactor { try { scheduled.push({ state, - decision: this.chooseCompactionKind(state, jobStartedAt), - forcedKind: this.forcedCompactionKind(state, forceKind) + decision: chooseCompactionKind(state, jobStartedAt, this), + forcedKind: forcedCompactionKind(state, forceKind, this) }); } catch (error) { await this.rescheduleFailedBucket(state, rescheduleNotBefore, error); @@ -307,9 +183,9 @@ export class MongoCompactorV3 extends MongoCompactor { if (lease == null) { continue; } - const claimedDecision = this.chooseCompactionKind(lease.state, lease.startedAt); + const claimedDecision = chooseCompactionKind(lease.state, lease.startedAt, this); const claimedKind = - forceKind == null ? claimedDecision.kind : this.forcedCompactionKind(lease.state, forceKind); + forceKind == null ? claimedDecision.kind : forcedCompactionKind(lease.state, forceKind, this); if (claimedKind == null) { await this.rescheduleClaimedBucket(lease, claimedDecision, rescheduleNotBefore); } else { @@ -322,17 +198,6 @@ export class MongoCompactorV3 extends MongoCompactor { } } - private forcedCompactionKind( - state: BucketStateDocumentV3, - forceKind: CompactionKind | undefined - ): CompactionKind | null { - if (forceKind == null) { - return null; - } - const maxOpId = this.maxOpIdCap == null || state.last_op < this.maxOpIdCap ? state.last_op : this.maxOpIdCap; - return state.compacted_state == null || state.compacted_state.op_id < maxOpId ? forceKind : null; - } - /** Read a bounded, priority-ordered snapshot of currently claimable scheduled work. */ private async findScheduledBucketBatch(dueBefore: Date): Promise { return this.db @@ -358,7 +223,7 @@ export class MongoCompactorV3 extends MongoCompactor { await this.db.bucketState(this.group_id).bulkWrite( states.map(({ state, decision }) => ({ updateOne: { - filter: this.unclaimedSnapshotFilter(state), + filter: unclaimedSnapshotFilter(state), update: [{ $set: { next_compact_check: decision.nextCompactCheck } }] } })), @@ -376,30 +241,12 @@ export class MongoCompactorV3 extends MongoCompactor { try { await this.db .bucketState(this.group_id) - .updateOne(this.unclaimedSnapshotFilter(state), [{ $set: { next_compact_check: notBefore } }]); + .updateOne(unclaimedSnapshotFilter(state), [{ $set: { next_compact_check: notBefore } }]); } catch (rescheduleError) { this.logger.error(`Failed to reschedule bucket ${state._id.b} after a compaction error`, rescheduleError); } } - /** - * This checks that the bucket state hasn't changed by a concurrent write since we checked. - * - * If it has changed, we'll re-check in the next batch. - */ - private unclaimedSnapshotFilter(state: BucketStateDocumentV3): mongo.Filter { - return { - _id: state._id, - last_op: state.last_op, - next_compact_check: state.next_compact_check, - first_uncompacted_write: state.first_uncompacted_write ?? { $exists: false }, - bucket_stats: state.bucket_stats, - compacted_state: state.compacted_state ?? { $exists: false }, - last_full_compact: state.last_full_compact ?? { $exists: false }, - ...AVAILABLE_LEASE_EXPR - }; - } - private async claimBucket( filter: mongo.Filter, sort?: mongo.Sort @@ -418,42 +265,6 @@ export class MongoCompactorV3 extends MongoCompactor { await this.retryCompaction(context.state._id.b, () => this.compactSingleBucket(context)); } - private chooseCompactionKind(state: BucketStateDocumentV3, now: Date): CompactionDecision { - // For chunk compaction, we consider the number of chunks added. - // Right now, we trigger a compact if the interval has passed and at least a threshold of chunks were added. - // A future policy could also use bytes per chunk or records per chunk to - // decide when to compact. - const fullCheckAt = this.fullCompactionCheckAt(state); - // Schedule a little late so a worker using a slightly earlier clock does - // not wake before the exact full-compaction condition is true. - const fullCheckWithMargin = new Date(fullCheckAt.getTime() + FULL_COMPACT_RESCHEDULE_MARGIN_MS); - const compacted = state.compacted_state; - const chunksSinceCompact = Math.max(0, state.bucket_stats.chunks - (compacted?.chunks ?? 0)); - const shouldCompactChunks = chunksSinceCompact >= MERGE_CHUNKS_THRESHOLD; - const canCheckChunks = - compacted == null || now.getTime() - compacted.at.getTime() >= this.minCompactChunkIntervalMs; - // Too few new chunks cannot make chunk compaction eligible. Do not poll this - // bucket at the chunk-compaction interval; only wake it for its full-compact check. - const nextCompactCheck: mongo.Document = !shouldCompactChunks - ? fullCheckWithMargin - : { - $min: [ - fullCheckWithMargin, - { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.minCompactChunkIntervalMs } } - ] - }; - let kind: CompactionKind | null = null; - if (now >= fullCheckAt) { - kind = CompactionKind.Full; - } else if (canCheckChunks && shouldCompactChunks) { - kind = CompactionKind.Chunks; - } - return { - kind, - nextCompactCheck - }; - } - private async rescheduleClaimedBucket(lease: CompactionLease, decision: CompactionDecision, notBefore?: Date) { await lease.reschedule(this.rescheduleAtOrAfter(decision.nextCompactCheck, notBefore)); } @@ -462,30 +273,6 @@ export class MongoCompactorV3 extends MongoCompactor { return notBefore == null ? nextCompactCheck : { $max: [nextCompactCheck, notBefore] }; } - /** - * Calculate the earliest full compaction time from the first uncompacted - * write, bounded by the maximum retention interval. - */ - private fullCompactionCheckAt(state: BucketStateDocumentV3): Date { - const firstWrite = firstUncompactedWrite(state); - const stats = bucketStats(state); - const lastFull = state.last_full_compact; - - // The number of operations since the last full compact. - // We may make this more specific in the future, to track new updates and deletes only, ignoring - // full new inserts, but that requires more granular tracking when replicating. - const uncompactedCount = lastFull == null ? stats.count : Math.max(0, stats.count - lastFull.count); - const compactedRows = lastFull?.puts ?? 0; - - // If no full compact has ever been performed: ratio = 1, compact after minCompactFullIntervalMs. - // If every row has been updated or deleted exactly once since the last full compact: ratio = 0.5, compact after minCompactFullIntervalMs * 2. - // If 10% of rows has been updated since the last full compact, compact after minCompactFullIntervalMs * 11. - // If every row has been updated multiple times, the ratio tends closer to 1 again. - const ratio = uncompactedCount == 0 ? 0 : uncompactedCount / (compactedRows + uncompactedCount); - const fullIntervalMs = ratio == 0 ? this.maxCompactFullIntervalMs : this.minCompactFullIntervalMs / ratio; - return new Date(firstWrite.getTime() + Math.min(fullIntervalMs, this.maxCompactFullIntervalMs)); - } - private compactMaxOpId(context: CompactionContext): InternalOpId { return this.maxOpIdCap == null || context.lastOp < this.maxOpIdCap ? context.lastOp : this.maxOpIdCap; } @@ -582,7 +369,7 @@ export class MongoCompactorV3 extends MongoCompactor { for (const doc of batch.documents) { compactedOpId = maxOpId(compactedOpId, doc._id.o); const documentStats = statsForDocument(doc); - preCompactionTail = this.combineAdjacentStats(preCompactionTail, documentStats); + preCompactionTail = combineAdjacentStats(preCompactionTail, documentStats); if (context.state.compacted_state?.op_id === doc._id.o) { overlappingCompactedChunk = documentStats; } @@ -637,8 +424,8 @@ export class MongoCompactorV3 extends MongoCompactor { compactedState: previousCompactedState == null || overlappingCompactedChunk == null ? tailStats - : this.combineChunkStats(previousCompactedState, tailStats, overlappingCompactedChunk), - bucketStats: this.applyCompactionDelta(bucketStats(context.state), preCompactionTail, tailStats) + : combineChunkStats(previousCompactedState, tailStats, overlappingCompactedChunk), + bucketStats: applyCompactionDelta(bucketStats(context.state), preCompactionTail, tailStats) }; } @@ -823,46 +610,6 @@ export class MongoCompactorV3 extends MongoCompactor { }; } - private combineChunkStats( - previous: NonNullable, - compactedTail: BucketStatsWithChecksum, - overlappingCompactedChunk: BucketStatsWithChecksum - ): BucketStatsWithChecksum { - return { - count: previous.count - overlappingCompactedChunk.count + compactedTail.count, - bytes: previous.bytes - overlappingCompactedChunk.bytes + compactedTail.bytes, - chunks: previous.chunks - 1 + compactedTail.chunks, - checksum: addChecksums( - addChecksums(Number(previous.checksum), -overlappingCompactedChunk.checksum), - compactedTail.checksum - ) - }; - } - - private combineAdjacentStats( - first: BucketStatsWithChecksum, - second: BucketStatsWithChecksum - ): BucketStatsWithChecksum { - return { - count: first.count + second.count, - bytes: first.bytes + second.bytes, - chunks: first.chunks + second.chunks, - checksum: addChecksums(first.checksum, second.checksum) - }; - } - - private applyCompactionDelta( - total: BucketStats, - before: BucketStatsWithChecksum, - after: BucketStatsWithChecksum - ): BucketStats { - return { - count: total.count - before.count + after.count, - bytes: total.bytes - before.bytes + after.bytes, - chunks: total.chunks - before.chunks + after.chunks - }; - } - private async compactSingleBucketFully(context: CompactionContext) { const bucket = context.state._id.b; const resolvedDefinitionId = context.state._id.d; @@ -941,7 +688,7 @@ export class MongoCompactorV3 extends MongoCompactor { // merging is useful, and writes each final object at most once. for (const doc of batchDocs) { compactedOpId ??= doc._id.o; - preCompactionPrefix = this.combineAdjacentStats(preCompactionPrefix, statsForDocument(doc)); + preCompactionPrefix = combineAdjacentStats(preCompactionPrefix, statsForDocument(doc)); const originalOps = Array.from(loadBucketDataDocument(dataContext, doc)); let changed = false; @@ -1076,7 +823,7 @@ export class MongoCompactorV3 extends MongoCompactor { const compactedStats = await this.readBucketStats(bucket, resolvedDefinitionId, compactedOpId); const result = { compactedState: compactedStats, - bucketStats: this.applyCompactionDelta(bucketStats(context.state), preCompactionPrefix, compactedStats) + bucketStats: applyCompactionDelta(bucketStats(context.state), preCompactionPrefix, compactedStats) }; // --- Finalize: update bucket checksums and state --- diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/compact-utils.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/compact-utils.ts new file mode 100644 index 000000000..4a7dc8631 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/compact-utils.ts @@ -0,0 +1,297 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { addChecksums, InternalOpId } from '@powersync/service-core'; +import { BucketDataDoc } from '../common/BucketDataDoc.js'; +import { AVAILABLE_LEASE_EXPR, CompactionLease } from './CompactionLease.js'; +import { BucketDataDocumentV3, BucketStateDocumentV3 } from './models.js'; + +/** + * Perform chunk compaction if at least this many chunks have been added since + * the latest compaction. + * + * A lower value increases compaction frequency and can increase chunk rewrites + * and object-storage operations. A higher value leaves more small chunks for + * longer, which can hurt sync performance. + */ +const MERGE_CHUNKS_THRESHOLD = 8; + +export const FULL_COMPACT_RESCHEDULE_MARGIN_MS = 60 * 1000; + +export interface PendingCompactionGroup { + /** + * Input documents are ordered from oldest to newest, matching `ops`. + * Keeping the inputs intact lets unchanged singletons retain their object. + */ + inputs: BucketDataDocumentV3[]; + ops: BucketDataDoc[]; + changed: boolean; + targetOp: InternalOpId | null; +} + +export interface CompactIntervalConfig { + readonly minCompactChunkIntervalMs: number; + readonly minCompactFullIntervalMs: number; + readonly maxCompactFullIntervalMs: number; +} +export interface CompactTargetConfig { + readonly maxOpIdCap: InternalOpId | undefined; +} + +export enum CompactionKind { + Full = 'full', + Chunks = 'chunks' +} + +export interface CompactionDecision { + kind: CompactionKind | null; + nextCompactCheck: mongo.Document; +} + +export interface ScheduledCompactionOptions { + /** Process checks scheduled this far after the captured job start. */ + dueAheadMs?: number; + /** Used by initial replication, which must not run a full compact. */ + forceKind?: CompactionKind; +} + +export class CompactionContext { + constructor( + readonly lease: CompactionLease, + readonly kind: CompactionKind, + readonly decision: CompactionDecision, + readonly rescheduleNotBefore: Date | undefined + ) {} + + get state() { + return this.lease.state; + } + + get startedAt() { + return this.lease.startedAt; + } + + get lastOp() { + return this.lease.lastOp; + } +} + +export interface BucketStats { + count: number; + bytes: bigint; + chunks: number; +} + +/** Bucket stats read from bucket-data documents, including their checksum. */ +export interface BucketStatsWithChecksum extends BucketStats { + checksum: number; +} + +export interface CompactionResult { + /** Metadata cached at compacted_state.op_id. */ + compactedState: BucketStatsWithChecksum; + /** Complete bucket metadata through the op head captured at claim time. */ + bucketStats: BucketStats; +} + +export function bucketStats(state: BucketStateDocumentV3): BucketStats { + return { + count: state.bucket_stats.count, + bytes: state.bucket_stats.bytes, + chunks: state.bucket_stats.chunks + }; +} + +export function emptyBucketStats(): BucketStatsWithChecksum { + return { count: 0, bytes: 0n, chunks: 0, checksum: 0 }; +} + +export function statsForDocument( + document: Pick +): BucketStatsWithChecksum { + return { + count: document.count, + bytes: BigInt(document.size), + chunks: 1, + checksum: addChecksums(0, Number(document.checksum)) + }; +} + +/** A scheduled bucket always has writes awaiting a full compact. */ +export function firstUncompactedWrite(state: BucketStateDocumentV3): Date { + if (state.first_uncompacted_write == null) { + throw new ReplicationAssertionError(`Scheduled V3 bucket ${state._id.b} has no first uncompacted write`); + } + return state.first_uncompacted_write; +} + +export function combineChunkStats( + previous: NonNullable, + compactedTail: BucketStatsWithChecksum, + overlappingCompactedChunk: BucketStatsWithChecksum +): BucketStatsWithChecksum { + return { + count: previous.count - overlappingCompactedChunk.count + compactedTail.count, + bytes: previous.bytes - overlappingCompactedChunk.bytes + compactedTail.bytes, + chunks: previous.chunks - 1 + compactedTail.chunks, + checksum: addChecksums( + addChecksums(Number(previous.checksum), -overlappingCompactedChunk.checksum), + compactedTail.checksum + ) + }; +} + +export function combineAdjacentStats( + first: BucketStatsWithChecksum, + second: BucketStatsWithChecksum +): BucketStatsWithChecksum { + return { + count: first.count + second.count, + bytes: first.bytes + second.bytes, + chunks: first.chunks + second.chunks, + checksum: addChecksums(first.checksum, second.checksum) + }; +} + +export function applyCompactionDelta( + total: BucketStats, + before: BucketStatsWithChecksum, + after: BucketStatsWithChecksum +): BucketStats { + return { + count: total.count - before.count + after.count, + bytes: total.bytes - before.bytes + after.bytes, + chunks: total.chunks - before.chunks + after.chunks + }; +} + +/** + * This checks that the bucket state hasn't changed by a concurrent write since we checked. + * + * If it has changed, we'll re-check in the next batch. + */ +export function unclaimedSnapshotFilter(state: BucketStateDocumentV3): mongo.Filter { + return { + _id: state._id, + last_op: state.last_op, + next_compact_check: state.next_compact_check, + first_uncompacted_write: state.first_uncompacted_write ?? { $exists: false }, + bucket_stats: state.bucket_stats, + compacted_state: state.compacted_state ?? { $exists: false }, + last_full_compact: state.last_full_compact ?? { $exists: false }, + ...AVAILABLE_LEASE_EXPR + }; +} + +/** + * Calculate the earliest full compaction time from the first uncompacted + * write, bounded by the maximum retention interval. + */ +export function fullCompactionCheckAt(state: BucketStateDocumentV3, config: CompactIntervalConfig): Date { + const firstWrite = firstUncompactedWrite(state); + const stats = bucketStats(state); + const lastFull = state.last_full_compact; + + // The number of operations since the last full compact. + // We may make this more specific in the future, to track new updates and deletes only, ignoring + // full new inserts, but that requires more granular tracking when replicating. + const uncompactedCount = lastFull == null ? stats.count : Math.max(0, stats.count - lastFull.count); + const compactedRows = lastFull?.puts ?? 0; + + // If no full compact has ever been performed: ratio = 1, compact after minCompactFullIntervalMs. + // If every row has been updated or deleted exactly once since the last full compact: ratio = 0.5, compact after minCompactFullIntervalMs * 2. + // If 10% of rows has been updated since the last full compact, compact after minCompactFullIntervalMs * 11. + // If every row has been updated multiple times, the ratio tends closer to 1 again. + const ratio = uncompactedCount == 0 ? 0 : uncompactedCount / (compactedRows + uncompactedCount); + const fullIntervalMs = ratio == 0 ? config.maxCompactFullIntervalMs : config.minCompactFullIntervalMs / ratio; + return new Date(firstWrite.getTime() + Math.min(fullIntervalMs, config.maxCompactFullIntervalMs)); +} + +export function chooseCompactionKind( + state: BucketStateDocumentV3, + now: Date, + config: CompactIntervalConfig +): CompactionDecision { + // For chunk compaction, we consider the number of chunks added. + // Right now, we trigger a compact if the interval has passed and at least a threshold of chunks were added. + // A future policy could also use bytes per chunk or records per chunk to + // decide when to compact. + const fullCheckAt = fullCompactionCheckAt(state, config); + // Schedule a little late so a worker using a slightly earlier clock does + // not wake before the exact full-compaction condition is true. + const fullCheckWithMargin = new Date(fullCheckAt.getTime() + FULL_COMPACT_RESCHEDULE_MARGIN_MS); + const compacted = state.compacted_state; + const chunksSinceCompact = Math.max(0, state.bucket_stats.chunks - (compacted?.chunks ?? 0)); + const shouldCompactChunks = chunksSinceCompact >= MERGE_CHUNKS_THRESHOLD; + const canCheckChunks = + compacted == null || now.getTime() - compacted.at.getTime() >= config.minCompactChunkIntervalMs; + // Too few new chunks cannot make chunk compaction eligible. Do not poll this + // bucket at the chunk-compaction interval; only wake it for its full-compact check. + const nextCompactCheck: mongo.Document = !shouldCompactChunks + ? fullCheckWithMargin + : { + $min: [ + fullCheckWithMargin, + { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: config.minCompactChunkIntervalMs } } + ] + }; + let kind: CompactionKind | null = null; + if (now >= fullCheckAt) { + kind = CompactionKind.Full; + } else if (canCheckChunks && shouldCompactChunks) { + kind = CompactionKind.Chunks; + } + return { + kind, + nextCompactCheck + }; +} + +export function forcedCompactionKind( + state: BucketStateDocumentV3, + forceKind: CompactionKind | undefined, + config: CompactTargetConfig +): CompactionKind | null { + if (forceKind == null) { + return null; + } + const maxOpId = config.maxOpIdCap == null || state.last_op < config.maxOpIdCap ? state.last_op : config.maxOpIdCap; + return state.compacted_state == null || state.compacted_state.op_id < maxOpId ? forceKind : null; +} + +/** + * Read one bounded prefix from a compaction cursor. + * + * The document that would cross the byte limit is deliberately not returned: + * pagination resumes past the last returned `_id`, so that document remains + * eligible for the next query. The first document is always accepted to + * ensure progress when a single document exceeds the configured byte limit. + * + * `hasMore` is conservative when the document limit is reached. An extra empty + * query is preferable to exhausting the cursor just to determine whether the + * limited MongoDB query contained another document. + */ +export async function readCompactionBatch( + cursor: mongo.AggregationCursor, + options: { byteLimit: number; documentLimit: number } +): Promise<{ documents: BucketDataDocumentV3[]; hasMore: boolean }> { + const documents: BucketDataDocumentV3[] = []; + let cumulativeBytes = 0; + + try { + for await (const document of cursor) { + if (documents.length > 0 && cumulativeBytes + document.size > options.byteLimit) { + return { documents, hasMore: true }; + } + + documents.push(document); + cumulativeBytes += document.size; + + if (documents.length >= options.documentLimit) { + return { documents, hasMore: true }; + } + } + return { documents, hasMore: false }; + } finally { + await cursor.close(); + } +} diff --git a/modules/module-mongodb-storage/test/src/compact-utils.test.ts b/modules/module-mongodb-storage/test/src/compact-utils.test.ts new file mode 100644 index 000000000..afab59719 --- /dev/null +++ b/modules/module-mongodb-storage/test/src/compact-utils.test.ts @@ -0,0 +1,208 @@ +import { + applyCompactionDelta, + bucketStats, + chooseCompactionKind, + combineAdjacentStats, + combineChunkStats, + CompactIntervalConfig, + CompactionKind, + emptyBucketStats, + forcedCompactionKind, + fullCompactionCheckAt, + statsForDocument +} from '@module/storage/implementation/v3/compact-utils.js'; +import { BucketStateDocumentV3 } from '@module/storage/implementation/v3/models.js'; +import { describe, expect, test } from 'vitest'; + +const INTERVAL_CONFIG: CompactIntervalConfig = { + minCompactChunkIntervalMs: 1_000, + minCompactFullIntervalMs: 10_000, + maxCompactFullIntervalMs: 100_000 +}; + +function bucketState(overrides: Partial = {}): BucketStateDocumentV3 { + return { + _id: { d: 'definition', b: 'bucket[]' }, + last_op: 10n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { count: 10, bytes: 100n, chunks: 1 }, + ...overrides + }; +} + +function compactedState(overrides: Partial> = {}) { + return { + op_id: 5n, + checksum: 50n, + count: 5, + bytes: 50n, + chunks: 1, + at: new Date(0), + ...overrides + }; +} + +describe('V3 compact utilities', () => { + test.each([ + { + name: 'uses the minimum interval before the first full compact', + state: bucketState(), + expected: new Date(10_000) + }, + { + name: 'scales the interval by the uncompacted row ratio', + state: bucketState({ + bucket_stats: { count: 100, bytes: 100n, chunks: 1 }, + last_full_compact: { op_id: 90n, count: 90, puts: 90, at: new Date(0) } + }), + expected: new Date(100_000) + }, + { + name: 'uses the maximum interval when there are no uncompacted rows', + state: bucketState({ + bucket_stats: { count: 100, bytes: 100n, chunks: 1 }, + last_full_compact: { op_id: 10n, count: 100, puts: 100, at: new Date(0) } + }), + expected: new Date(100_000) + } + ])('$name', ({ state, expected }) => { + expect(fullCompactionCheckAt(state, INTERVAL_CONFIG)).toEqual(expected); + }); + + test('rejects a scheduled bucket without an uncompacted write', () => { + expect(() => fullCompactionCheckAt(bucketState({ first_uncompacted_write: undefined }), INTERVAL_CONFIG)).toThrow( + 'Scheduled V3 bucket bucket[] has no first uncompacted write' + ); + }); + + test('chooses a full compact when its check is due', () => { + const decision = chooseCompactionKind( + bucketState({ bucket_stats: { count: 10, bytes: 100n, chunks: 10 } }), + new Date(10_000), + INTERVAL_CONFIG + ); + + expect(decision.kind).toBe(CompactionKind.Full); + expect(decision.nextCompactCheck).toEqual({ + $min: [new Date(70_000), { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: 1_000 } }] + }); + }); + + test('chooses a chunk compact after enough chunks and the minimum interval', () => { + const decision = chooseCompactionKind( + bucketState({ + bucket_stats: { count: 10, bytes: 100n, chunks: 9 }, + compacted_state: compactedState({ chunks: 1 }) + }), + new Date(5_000), + INTERVAL_CONFIG + ); + + expect(decision.kind).toBe(CompactionKind.Chunks); + }); + + test('waits when the chunk interval has not elapsed', () => { + const decision = chooseCompactionKind( + bucketState({ + bucket_stats: { count: 10, bytes: 100n, chunks: 9 }, + compacted_state: compactedState({ chunks: 1, at: new Date(4_500) }) + }), + new Date(5_000), + INTERVAL_CONFIG + ); + + expect(decision.kind).toBeNull(); + }); + + test('schedules only the full check when too few chunks were added', () => { + const decision = chooseCompactionKind( + bucketState({ bucket_stats: { count: 7, bytes: 70n, chunks: 7 } }), + new Date(5_000), + INTERVAL_CONFIG + ); + + expect(decision).toEqual({ kind: null, nextCompactCheck: new Date(70_000) }); + }); + + test.each([ + { + name: 'does not force an unspecified kind', + forceKind: undefined, + compactedOpId: undefined, + maxOpIdCap: undefined, + expected: null + }, + { + name: 'forces work without previous compacted state', + forceKind: CompactionKind.Chunks, + compactedOpId: undefined, + maxOpIdCap: undefined, + expected: CompactionKind.Chunks + }, + { + name: 'skips work already compacted through the cap', + forceKind: CompactionKind.Chunks, + compactedOpId: 5n, + maxOpIdCap: 5n, + expected: null + }, + { + name: 'skips work already compacted through the bucket head', + forceKind: CompactionKind.Chunks, + compactedOpId: 10n, + maxOpIdCap: undefined, + expected: null + }, + { + name: 'forces work beyond the compacted state', + forceKind: CompactionKind.Chunks, + compactedOpId: 5n, + maxOpIdCap: 6n, + expected: CompactionKind.Chunks + } + ])('$name', ({ forceKind, compactedOpId, maxOpIdCap, expected }) => { + const state = bucketState({ + compacted_state: compactedOpId == null ? undefined : compactedState({ op_id: compactedOpId }) + }); + expect(forcedCompactionKind(state, forceKind, { maxOpIdCap })).toBe(expected); + }); + + test('derives and combines bucket statistics', () => { + const state = bucketState({ bucket_stats: { count: 12, bytes: 120n, chunks: 3 } }); + expect(bucketStats(state)).toEqual({ count: 12, bytes: 120n, chunks: 3 }); + expect(emptyBucketStats()).toEqual({ count: 0, bytes: 0n, chunks: 0, checksum: 0 }); + expect(statsForDocument({ count: 2, size: 20, checksum: 10n })).toEqual({ + count: 2, + bytes: 20n, + chunks: 1, + checksum: 10 + }); + expect( + combineAdjacentStats( + { count: 2, bytes: 20n, chunks: 1, checksum: 10 }, + { count: 3, bytes: 30n, chunks: 2, checksum: 20 } + ) + ).toEqual({ count: 5, bytes: 50n, chunks: 3, checksum: 30 }); + }); + + test('replaces an overlapping chunk in cached statistics', () => { + expect( + combineChunkStats( + compactedState({ count: 10, bytes: 100n, chunks: 4, checksum: 100n }), + { count: 6, bytes: 60n, chunks: 2, checksum: 50 }, + { count: 3, bytes: 30n, chunks: 1, checksum: 30 } + ) + ).toEqual({ count: 13, bytes: 130n, chunks: 5, checksum: 120 }); + }); + + test('applies compacted statistics as a delta to total bucket statistics', () => { + expect( + applyCompactionDelta( + { count: 20, bytes: 200n, chunks: 8 }, + { count: 5, bytes: 50n, chunks: 3, checksum: 10 }, + { count: 3, bytes: 30n, chunks: 1, checksum: 10 } + ) + ).toEqual({ count: 18, bytes: 180n, chunks: 6 }); + }); +}); diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 31fdea5d7..5281b06de 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -374,20 +374,20 @@ bucket_definitions: const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3Storage(); const badBucket = 'bad[]'; const goodBucket = 'good[]'; - const badDocument = serializeBucketData(badBucket, [ - makeOp(2, 'bad', 'bad', { ...ctx, bucket: badBucket }, sourceTableId) - ]); const goodDocument = serializeBucketData(goodBucket, [ makeOp(2, 'good', 'good', { ...ctx, bucket: goodBucket }, sourceTableId) ]); - await insertDocs(collection, [badDocument, goodDocument]); + await insertDocs(collection, [goodDocument]); await bucketStateCollection.insertMany([ { _id: { d: ctx.definitionId, b: badBucket }, last_op: 2n, next_compact_check: new Date(0), + // A malformed scheduled bucket with an expired lease must be + // rescheduled without preventing valid buckets from compacting. first_uncompacted_write: new Date(0), - bucket_stats: { count: 1, bytes: BigInt(badDocument.size), chunks: 1 } + bucket_stats: { count: 1, bytes: 1n, chunks: 1 }, + compact_lease: { id: new bson.ObjectId(), expires_at: new Date(0) } }, { _id: { d: ctx.definitionId, b: goodBucket }, @@ -397,19 +397,14 @@ bucket_definitions: bucket_stats: { count: 1, bytes: BigInt(goodDocument.size), chunks: 1 } } ]); + await bucketStateCollection.updateOne( + { _id: { d: ctx.definitionId, b: badBucket } }, + { $unset: { first_uncompacted_write: '' } } + ); - const compactor = (bucketStorage as MongoSyncBucketStorage).createMongoCompactor({ - maxOpId: 2n, - compactChunksOnly: true - }); - const compactSingleBucket = (compactor as any).compactSingleBucket.bind(compactor); - vi.spyOn(compactor as any, 'compactSingleBucket').mockImplementation(async (context: any) => { - if (context.state._id.b == badBucket) { - throw new Error('malformed bucket'); - } - return compactSingleBucket(context); - }); - await compactor.compact(); + await (bucketStorage as MongoSyncBucketStorage) + .createMongoCompactor({ maxOpId: 2n, compactChunksOnly: true }) + .compact(); const [badState, goodState] = await Promise.all([ bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: badBucket } }), From ef9502001950a32e0a42d00bcf97730da63d0638 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 17 Aug 2026 09:57:57 +0200 Subject: [PATCH 30/38] Fix bucket stats calculations after retry for full compact. --- .../implementation/v3/MongoCompactorV3.ts | 56 ++++++++++++------ .../test/src/storage_compacting.test.ts | 58 +++++++++++++++++++ 2 files changed, 97 insertions(+), 17 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index e633d8a6d..e30fa01fc 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -247,6 +247,11 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC } } + /** + * Given a bucket filter, claim a lease on the bucket. The filter should include a filter on _id. + * + * Resolves to null if the bucket is already claimed, not found, or filtered out. + */ private async claimBucket( filter: mongo.Filter, sort?: mongo.Sort @@ -408,16 +413,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC // the cached prefix cannot be combined with the current tail. Rebuild the // metadata from the authoritative documents while keeping the old op id // as a conservative resume hint. - const [compactedState, currentBucketStats] = await Promise.all([ - this.readBucketStats(bucket, resolvedDefinitionId, compactedOpId), - compactedOpId == context.lastOp - ? Promise.resolve(undefined) - : this.readBucketStats(bucket, resolvedDefinitionId, context.lastOp) - ]); - result = { - compactedState, - bucketStats: currentBucketStats ?? compactedState - }; + result = await this.readAuthoritativeCompactionResult(context, bucketContext, compactedOpId); } else { const tailStats = await this.readBucketStats(bucket, resolvedDefinitionId, compactedOpId, tailLowerBound); result = { @@ -566,6 +562,38 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC ); } + /** + * Calculate bucket statistics from the data that is currently stored. + * + * A previous attempt may have saved some compacted data before it failed, + * without updating the bucket summary. Reading the stored data again avoids + * carrying that outdated summary into the successful attempt. + * + * Most compactions cover the whole bucket and need only one calculation. If + * this compaction stops at an operation limit, also include the unchanged + * data after that limit in the bucket total. + */ + private async readAuthoritativeCompactionResult( + context: CompactionContext, + bucketContext: BucketDataContextV3, + compactedOpId: InternalOpId + ): Promise { + const { bucket, definitionId } = bucketContext.key; + const [compactedState, tailStats] = await Promise.all([ + this.readBucketStats(bucket, definitionId, compactedOpId), + compactedOpId == context.lastOp + ? Promise.resolve(undefined) + : this.readBucketStats(bucket, definitionId, context.lastOp, bucketContext.docId(compactedOpId)) + ]); + return { + compactedState, + bucketStats: tailStats == null ? compactedState : combineAdjacentStats(compactedState, tailStats) + }; + } + + /** + * Read bucket stats directly from bucket_data documents. + */ private async readBucketStats( bucket: string, definitionId: BucketDefinitionId, @@ -624,7 +652,6 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC let upperBound = bucketContext.docId(this.compactMaxOpId(context) + 1n); let totalOpCount = 0; - let preCompactionPrefix = emptyBucketStats(); let lastNotPut: bigint | null = null; let opsSincePut = 0; @@ -688,7 +715,6 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC // merging is useful, and writes each final object at most once. for (const doc of batchDocs) { compactedOpId ??= doc._id.o; - preCompactionPrefix = combineAdjacentStats(preCompactionPrefix, statsForDocument(doc)); const originalOps = Array.from(loadBucketDataDocument(dataContext, doc)); let changed = false; @@ -820,11 +846,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC ); } - const compactedStats = await this.readBucketStats(bucket, resolvedDefinitionId, compactedOpId); - const result = { - compactedState: compactedStats, - bucketStats: applyCompactionDelta(bucketStats(context.state), preCompactionPrefix, compactedStats) - }; + const result = await this.readAuthoritativeCompactionResult(context, bucketContext, compactedOpId); // --- Finalize: update bucket checksums and state --- await this.finalizeCompactedBucket({ context, compactedOpId, compactionResult: result, puts: putCount }); diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 5281b06de..c30b17171 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -1720,6 +1720,57 @@ bucket_definitions: }); }); + test('full compaction retry rebuilds stats after a committed replacement', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); + const documents = [ + serializeBucketData(BUCKET, [makeOp(1, 'A', 'a', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(2, 'B', 'b', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(3, 'C', 'c', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(4, 'D', 'd', ctx, sourceTableId)]) + ]; + await insertDocs(collection, documents); + await bucketStateCollection.insertOne({ + _id: { d: ctx.definitionId, b: BUCKET }, + last_op: 4n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { + count: documents.reduce((total, document) => total + document.count, 0), + bytes: BigInt(documents.reduce((total, document) => total + document.size, 0)), + chunks: documents.length + } + }); + + const compactor = bucketStorage.createMongoCompactor({ maxOpId: 4n, compactBuckets: [BUCKET] }); + const originalFlush = (compactor as any).flushCompactionGroup.bind(compactor); + let injectedFailure = false; + vi.spyOn(compactor as any, 'flushCompactionGroup').mockImplementation(async (...args: any[]) => { + const result = await originalFlush(...args); + if (!injectedFailure) { + injectedFailure = true; + throw new ObjectStorageError('failure after committed full-compaction replacement', { + cause: new Error('socket reset'), + retryable: true + }); + } + return result; + }); + + await expect(compactor.compact()).resolves.toBe(1); + + expect(injectedFailure).toBe(true); + const currentDocuments = await collection.find({ '_id.b': BUCKET }).toArray(); + expect(currentDocuments).toHaveLength(1); + const expectedStats = { + count: currentDocuments.reduce((total, document) => total + document.count, 0), + bytes: BigInt(currentDocuments.reduce((total, document) => total + document.size, 0)), + chunks: currentDocuments.length + }; + const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(state?.bucket_stats).toEqual(expectedStats); + expect(state?.compacted_state).toMatchObject({ op_id: 4n, ...expectedStats }); + }); + test('chunk compaction retry rebuilds state after a committed partial merge', async () => { const { bucketStorage, collection, bucketStateCollection, ctx } = await setupCompactedTail(); const compactor = bucketStorage.createMongoCompactor({ @@ -1933,6 +1984,13 @@ bucket_definitions: expect(op600).toBeDefined(); expect(op600!.op).toBe('PUT'); expect(op600!.row_id).toBe('F'); + + const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(state?.bucket_stats).toEqual({ + count: docsAfter.reduce((total, document) => total + document.count, 0), + bytes: BigInt(docsAfter.reduce((total, document) => total + document.size, 0)), + chunks: docsAfter.length + }); }); test('3. seen map overflow - some old ops pass through without tombstoning', async () => { From bcfa5bd0c72177adba0049313f1fbc674b51748b Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 17 Aug 2026 10:21:21 +0200 Subject: [PATCH 31/38] Use in-memory stats where feasible. --- .../implementation/v3/MongoCompactorV3.ts | 143 +++++++++++++----- .../implementation/v3/compact-utils.ts | 22 +++ .../test/src/compact-utils.test.ts | 20 ++- 3 files changed, 147 insertions(+), 38 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index e30fa01fc..246a9d140 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -11,6 +11,7 @@ import { BucketDataContextV3 } from './BucketDataContextV3.js'; import { DEFAULT_MAX_DOC_SIZE_BYTES } from './chunking.js'; import { applyCompactionDelta, + applyStatsReplacement, bucketStats, BucketStatsWithChecksum, chooseCompactionKind, @@ -29,6 +30,7 @@ import { readCompactionBatch, ScheduledCompactionOptions, statsForDocument, + statsForDocuments, unclaimedSnapshotFilter } from './compact-utils.js'; import { DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS } from './compaction-constants.js'; @@ -44,6 +46,20 @@ const DEFAULT_MAX_COMPACT_FULL_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000; const DEFAULT_COMPACT_LEASE_DURATION_MS = 10 * 60 * 1000; const SCHEDULED_COMPACTION_BATCH_SIZE = 100; +interface CompactionGroupResult { + documentId: BucketDataKey; + stats: BucketStatsWithChecksum; +} + +interface CompactionStatsReplacement { + before: BucketStatsWithChecksum; + after: BucketStatsWithChecksum; +} + +interface ClearCompactionResult extends CompactionStatsReplacement { + opCountDiff: number; +} + export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalConfig, CompactTargetConfig { declare protected readonly db: VersionedPowerSyncMongoV3; declare protected readonly storage: MongoSyncBucketStorageV3; @@ -326,7 +342,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC let compactedOpId: bigint | null = null; let overlappingCompactedChunk: BucketStatsWithChecksum | undefined; let preCompactionTail = emptyBucketStats(); - const tailLowerBound = lowerBound; + let compactedTail = emptyBucketStats(); let pendingChunks: BucketDataDocumentV3[] = []; let pendingSize = 0; @@ -381,7 +397,8 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC const nextSize = pendingSize + doc.size; if (pendingChunks.length > 0 && nextSize > DEFAULT_MAX_DOC_SIZE_BYTES) { - await this.flushChunkMerge(bucket, pendingChunks, collection, dataContext, bucketContext); + const groupStats = await this.flushChunkMerge(bucket, pendingChunks, collection, dataContext, bucketContext); + compactedTail = combineAdjacentStats(compactedTail, groupStats); pendingChunks = []; pendingSize = 0; } @@ -396,8 +413,9 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC } } - if (pendingChunks.length > 1) { - await this.flushChunkMerge(bucket, pendingChunks, collection, dataContext, bucketContext); + if (pendingChunks.length > 0) { + const groupStats = await this.flushChunkMerge(bucket, pendingChunks, collection, dataContext, bucketContext); + compactedTail = combineAdjacentStats(compactedTail, groupStats); } if (compactedOpId == null) { @@ -415,13 +433,12 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC // as a conservative resume hint. result = await this.readAuthoritativeCompactionResult(context, bucketContext, compactedOpId); } else { - const tailStats = await this.readBucketStats(bucket, resolvedDefinitionId, compactedOpId, tailLowerBound); result = { compactedState: previousCompactedState == null || overlappingCompactedChunk == null - ? tailStats - : combineChunkStats(previousCompactedState, tailStats, overlappingCompactedChunk), - bucketStats: applyCompactionDelta(bucketStats(context.state), preCompactionTail, tailStats) + ? compactedTail + : combineChunkStats(previousCompactedState, compactedTail, overlappingCompactedChunk), + bucketStats: applyCompactionDelta(bucketStats(context.state), preCompactionTail, compactedTail) }; } @@ -438,9 +455,9 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC collection: mongo.Collection, context: { replicationStreamId: number; definitionId: string }, bucketContext: BucketDataContextV3 - ) { - if (inputs.length < 2) { - return; + ): Promise { + if (inputs.length == 1) { + return statsForDocument(inputs[0]); } // The metadata scan deliberately excluded ops. Read inline payloads only @@ -463,7 +480,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC (maxTarget, input) => maxOpId(maxTarget, input.target_op), null ); - await this.flushCompactionGroup( + const result = await this.flushCompactionGroup( bucket, { inputs, @@ -474,6 +491,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC bucketContext, context ); + return result.stats; } private async finalizeCompactedBucket({ @@ -563,15 +581,13 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC } /** - * Calculate bucket statistics from the data that is currently stored. + * Recover chunk statistics from the data that is currently stored. * - * A previous attempt may have saved some compacted data before it failed, - * without updating the bucket summary. Reading the stored data again avoids - * carrying that outdated summary into the successful attempt. - * - * Most compactions cover the whole bucket and need only one calculation. If - * this compaction stops at an operation limit, also include the unchanged - * data after that limit in the bucket total. + * Chunk compaction normally calculates statistics while processing its + * working range. If a previous attempt replaced the cached boundary before + * failing, that range no longer contains enough information to update the + * older cached prefix. In that case, rebuild the prefix and include any + * untouched data after an operation limit in the bucket total. */ private async readAuthoritativeCompactionResult( context: CompactionContext, @@ -657,6 +673,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC let opsSincePut = 0; let compactedOpId: bigint | null = null; let clearBoundary: { opId: bigint; documentId: BucketDataKey } | null = null; + let compactedStats = emptyBucketStats(); const seen = new Map(); let trackingSize = 0; let putCount = 0; @@ -793,13 +810,14 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC }; } else { const flushedGroup = pendingGroup; - const documentId = await this.flushCompactionGroup(bucket, flushedGroup, bucketContext, dataContext); + const result = await this.flushCompactionGroup(bucket, flushedGroup, bucketContext, dataContext); + compactedStats = combineAdjacentStats(compactedStats, result.stats); if ( lastNotPut != null && flushedGroup.ops[0].o <= lastNotPut && flushedGroup.ops[flushedGroup.ops.length - 1].o >= lastNotPut ) { - clearBoundary = { opId: lastNotPut, documentId }; + clearBoundary = { opId: lastNotPut, documentId: result.documentId }; } pendingGroup = candidate; } @@ -817,13 +835,14 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC } if (pendingGroup != null) { - const documentId = await this.flushCompactionGroup(bucket, pendingGroup, bucketContext, dataContext); + const result = await this.flushCompactionGroup(bucket, pendingGroup, bucketContext, dataContext); + compactedStats = combineAdjacentStats(compactedStats, result.stats); if ( lastNotPut != null && pendingGroup.ops[0].o <= lastNotPut && pendingGroup.ops[pendingGroup.ops.length - 1].o >= lastNotPut ) { - clearBoundary = { opId: lastNotPut, documentId }; + clearBoundary = { opId: lastNotPut, documentId: result.documentId }; } } if (compactedOpId == null) { @@ -837,16 +856,25 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC throw new ReplicationAssertionError(`Missing CLEAR boundary document for bucket ${bucket}`); } - totalOpCount += await this.clearBucketLeading( + const clearResult = await this.clearBucketLeading( lastNotPut, clearBoundary.documentId, bucketContext, collection, dataContext ); + totalOpCount += clearResult.opCountDiff; + compactedStats = applyStatsReplacement(compactedStats, clearResult.before, clearResult.after); } - const result = await this.readAuthoritativeCompactionResult(context, bucketContext, compactedOpId); + const tailStats = + compactedOpId == context.lastOp + ? undefined + : await this.readBucketStats(bucket, resolvedDefinitionId, context.lastOp, bucketContext.docId(compactedOpId)); + const result: CompactionResult = { + compactedState: compactedStats, + bucketStats: tailStats == null ? compactedStats : combineAdjacentStats(compactedStats, tailStats) + }; // --- Finalize: update bucket checksums and state --- await this.finalizeCompactedBucket({ context, compactedOpId, compactionResult: result, puts: putCount }); @@ -868,9 +896,12 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC group: PendingCompactionGroup, bucketContext: BucketDataContextV3, context: { replicationStreamId: number; definitionId: string } - ): Promise { + ): Promise { if (group.inputs.length == 1 && !group.changed) { - return group.inputs[0]._id; + return { + documentId: group.inputs[0]._id, + stats: statsForDocument(group.inputs[0]) + }; } const inputs = group.inputs; @@ -928,7 +959,10 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC } finally { await session.endSession(); } - return documents[0]._id; + return { + documentId: documents[0]._id, + stats: statsForDocuments(documents) + }; } /** @@ -937,7 +971,8 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC * before the known boundary document, then splits that boundary document * if it contains ops on both sides of lastNotPut. * - * Returns the op count diff after replacing cleared ops with CLEAR ops. + * Returns the op count and stored-stat changes after replacing cleared ops + * with CLEAR ops. */ private async clearBucketLeading( lastNotPut: bigint, @@ -945,8 +980,10 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC bucketContext: BucketDataContextV3, collection: mongo.Collection, context: { replicationStreamId: number; definitionId: string } - ): Promise { + ): Promise { let opCountDiff = 0; + let before = emptyBucketStats(); + let after = emptyBucketStats(); const session = this.db.client.startSession(); try { let done = false; @@ -963,11 +1000,13 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC ); done = batch.done; opCountDiff += batch.opCountDiff; + before = combineAdjacentStats(before, batch.before); + after = combineAdjacentStats(after, batch.after); } // The final step is to process the "boundary" document: It may contain some CLEAR/MOVE/REMOVE operations, // potentially followed by PUT operations. This is only a single document, so no need for batching. - opCountDiff += await this.clearBoundaryDocument( + const boundaryResult = await this.clearBoundaryDocument( session, lastNotPut, boundaryDocId, @@ -975,11 +1014,14 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC collection, context ); + opCountDiff += boundaryResult.opCountDiff; + before = combineAdjacentStats(before, boundaryResult.before); + after = combineAdjacentStats(after, boundaryResult.after); } finally { await session.endSession(); } - return opCountDiff; + return { opCountDiff, before, after }; } private async clearLeadingFullDocuments( @@ -989,17 +1031,21 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC bucketContext: BucketDataContextV3, collection: mongo.Collection, context: { replicationStreamId: number; definitionId: string } - ): Promise<{ done: boolean; opCountDiff: number }> { + ): Promise<{ done: boolean; opCountDiff: number } & CompactionStatsReplacement> { const bucket = bucketContext.key.bucket; this.signal?.throwIfAborted(); let prepared: PreparedObjectStorageUpload[] | undefined; let done = false; let opCountDiff = 0; + let before = emptyBucketStats(); + let after = emptyBucketStats(); await session.withTransaction( async () => { done = false; opCountDiff = 0; + before = emptyBucketStats(); + after = emptyBucketStats(); const oldStoragePaths: string[] = []; const query = collection.find( { @@ -1016,6 +1062,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC min_op: 1, checksum: 1, count: 1, + size: 1, target_op: 1, has_clear_op: 1, storage_ref: 1 @@ -1030,6 +1077,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC let lastDocId: BucketDataKey | null = null; let clearOpCount = 0; let gotNonClearOp = false; + const inputStats = emptyBucketStats(); for await (const doc of query.stream()) { if (doc.min_op > lastNotPut) { @@ -1039,6 +1087,11 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC } lastDocId = doc._id; + const documentStats = statsForDocument(doc); + inputStats.count += documentStats.count; + inputStats.bytes += documentStats.bytes; + inputStats.chunks += documentStats.chunks; + inputStats.checksum = addChecksums(inputStats.checksum, documentStats.checksum); if (doc.storage_ref) { oldStoragePaths.push(doc.storage_ref.path); } @@ -1093,6 +1146,8 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC await this.finishObjectStorageReplacement(oldStoragePaths, persisted.storagePaths, persisted.uploads, session); opCountDiff = -clearedOpCount + 1; + before = inputStats; + after = statsForDocuments(persisted.documents); }, { writeConcern: { w: 'majority' }, @@ -1100,7 +1155,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC } ); - return { done, opCountDiff }; + return { done, opCountDiff, before, after }; } private async clearBoundaryDocument( @@ -1110,15 +1165,19 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC bucketContext: BucketDataContextV3, collection: mongo.Collection, context: { replicationStreamId: number; definitionId: string } - ): Promise { + ): Promise { const bucket = bucketContext.key.bucket; this.signal?.throwIfAborted(); const prepared = await this.prepareCompactionUploads(bucket, context, [lastNotPut, boundaryDocId.o]); let opCountDiff = 0; + let before = emptyBucketStats(); + let after = emptyBucketStats(); await session.withTransaction( async () => { opCountDiff = 0; + before = emptyBucketStats(); + after = emptyBucketStats(); const oldStoragePaths: string[] = []; const query = collection.find( { @@ -1138,6 +1197,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC min_op: 1, checksum: 1, count: 1, + size: 1, target_op: 1, ops: 1, storage_ref: 1 @@ -1151,6 +1211,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC let clearedOpCount = 0; let maxTargetOp: bigint | null = null; const boundarySurvivors: BucketDataDoc[] = []; + const inputStats = emptyBucketStats(); for await (const doc of query.stream()) { docsRead++; @@ -1158,6 +1219,12 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC throw new ReplicationAssertionError(`Unexpected extra document before CLEAR boundary in bucket ${bucket}`); } + const documentStats = statsForDocument(doc); + inputStats.count += documentStats.count; + inputStats.bytes += documentStats.bytes; + inputStats.chunks += documentStats.chunks; + inputStats.checksum = addChecksums(inputStats.checksum, documentStats.checksum); + const isBoundaryDoc = doc._id.o == boundaryDocId.o; if (doc.storage_ref) { oldStoragePaths.push(doc.storage_ref.path); @@ -1224,6 +1291,8 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC await this.finishObjectStorageReplacement(oldStoragePaths, persisted.storagePaths, persisted.uploads, session); opCountDiff = -clearedOpCount + 1; + before = inputStats; + after = statsForDocuments(persisted.documents); }, { writeConcern: { w: 'majority' }, @@ -1231,7 +1300,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC } ); - return opCountDiff; + return { opCountDiff, before, after }; } /** diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/compact-utils.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/compact-utils.ts index 4a7dc8631..1366486cc 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/compact-utils.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/compact-utils.ts @@ -116,6 +116,16 @@ export function statsForDocument( }; } +export function statsForDocuments( + documents: Iterable> +): BucketStatsWithChecksum { + let result = emptyBucketStats(); + for (const document of documents) { + result = combineAdjacentStats(result, statsForDocument(document)); + } + return result; +} + /** A scheduled bucket always has writes awaiting a full compact. */ export function firstUncompactedWrite(state: BucketStateDocumentV3): Date { if (state.first_uncompacted_write == null) { @@ -164,6 +174,18 @@ export function applyCompactionDelta( }; } +/** Replace one stored range in an accumulated statistics snapshot. */ +export function applyStatsReplacement( + total: BucketStatsWithChecksum, + before: BucketStatsWithChecksum, + after: BucketStatsWithChecksum +): BucketStatsWithChecksum { + return { + ...applyCompactionDelta(total, before, after), + checksum: addChecksums(addChecksums(total.checksum, -before.checksum), after.checksum) + }; +} + /** * This checks that the bucket state hasn't changed by a concurrent write since we checked. * diff --git a/modules/module-mongodb-storage/test/src/compact-utils.test.ts b/modules/module-mongodb-storage/test/src/compact-utils.test.ts index afab59719..61bb2f6c8 100644 --- a/modules/module-mongodb-storage/test/src/compact-utils.test.ts +++ b/modules/module-mongodb-storage/test/src/compact-utils.test.ts @@ -1,5 +1,6 @@ import { applyCompactionDelta, + applyStatsReplacement, bucketStats, chooseCompactionKind, combineAdjacentStats, @@ -9,7 +10,8 @@ import { emptyBucketStats, forcedCompactionKind, fullCompactionCheckAt, - statsForDocument + statsForDocument, + statsForDocuments } from '@module/storage/implementation/v3/compact-utils.js'; import { BucketStateDocumentV3 } from '@module/storage/implementation/v3/models.js'; import { describe, expect, test } from 'vitest'; @@ -178,6 +180,12 @@ describe('V3 compact utilities', () => { chunks: 1, checksum: 10 }); + expect( + statsForDocuments([ + { count: 2, size: 20, checksum: 10n }, + { count: 3, size: 30, checksum: 20n } + ]) + ).toEqual({ count: 5, bytes: 50n, chunks: 2, checksum: 30 }); expect( combineAdjacentStats( { count: 2, bytes: 20n, chunks: 1, checksum: 10 }, @@ -205,4 +213,14 @@ describe('V3 compact utilities', () => { ) ).toEqual({ count: 18, bytes: 180n, chunks: 6 }); }); + + test('applies a stored range replacement to statistics including its checksum', () => { + expect( + applyStatsReplacement( + { count: 20, bytes: 200n, chunks: 8, checksum: 100 }, + { count: 5, bytes: 50n, chunks: 3, checksum: 40 }, + { count: 3, bytes: 30n, chunks: 1, checksum: 20 } + ) + ).toEqual({ count: 18, bytes: 180n, chunks: 6, checksum: 80 }); + }); }); From f71abfbf0fb8492e9f9818a7b6418bdc1a23f5a6 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 17 Aug 2026 11:08:09 +0200 Subject: [PATCH 32/38] Fix rescheduling. --- .../implementation/v3/MongoCompactorV3.ts | 12 ++++-- .../test/src/storage_compacting.test.ts | 37 +++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 246a9d140..9e6c1e18c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -186,7 +186,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC ({ state, decision, forcedKind }) => state.compact_lease == null && (forceKind == null ? decision.kind : forcedKind) == null ); - await this.rescheduleUnclaimedBuckets(noOpStates); + await this.rescheduleUnclaimedBuckets(noOpStates, rescheduleNotBefore); for (const { state, decision, forcedKind } of scheduled) { const kind = forceKind == null ? decision.kind : forcedKind; @@ -230,9 +230,13 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC /** * Reschedule snapshots that were already known to be no-ops without first * taking a lease. Every decision input is compared so a concurrent writer - * or compactor simply makes the update a no-op instead of losing work. + * or compactor simply makes the update a no-op instead of losing work. A + * successful reschedule moves beyond this run's fixed selection boundary. */ - private async rescheduleUnclaimedBuckets(states: { state: BucketStateDocumentV3; decision: CompactionDecision }[]) { + private async rescheduleUnclaimedBuckets( + states: { state: BucketStateDocumentV3; decision: CompactionDecision }[], + notBefore: Date + ) { if (states.length == 0) { return; } @@ -240,7 +244,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC states.map(({ state, decision }) => ({ updateOne: { filter: unclaimedSnapshotFilter(state), - update: [{ $set: { next_compact_check: decision.nextCompactCheck } }] + update: [{ $set: { next_compact_check: this.rescheduleAtOrAfter(decision.nextCompactCheck, notBefore) } }] } })), { ordered: false } diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index c30b17171..44c92d2a0 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -2,6 +2,7 @@ import { BucketDataDoc } from '@module/storage/implementation/common/BucketDataD import { MongoSyncBucketStorage } from '@module/storage/implementation/createMongoSyncBucketStorage.js'; import { loadBucketDataDocument, serializeBucketData } from '@module/storage/implementation/v3/bucket-format.js'; import { chunkBucketData, DEFAULT_MAX_DOC_SIZE_BYTES } from '@module/storage/implementation/v3/chunking.js'; +import { DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS } from '@module/storage/implementation/v3/compaction-constants.js'; import { CompactionLease } from '@module/storage/implementation/v3/CompactionLease.js'; import { BucketDataDocumentV3 } from '@module/storage/implementation/v3/models.js'; import { ObjectStorageError } from '@module/storage/implementation/v3/object-storage/ObjectStorage.js'; @@ -188,6 +189,42 @@ bucket_definitions: expect(result).toBe(2); }); + test('v3 repeated initial chunk compaction reschedules overdue full work beyond the current run', async () => { + const { bucketStorage, checkpoint } = await setup(storage.STORAGE_VERSION_3); + const firstResult = await bucketStorage.compactInitialReplication({ maxOpId: checkpoint }); + expect(firstResult.buckets).toBe(2); + + const bucketStateCollection = (bucketStorage.db as VersionedPowerSyncMongoV3).bucketState( + bucketStorage.replicationStreamId + ); + await bucketStateCollection.updateMany( + {}, + { + $set: { + first_uncompacted_write: new Date(0), + next_compact_check: new Date(0) + } + } + ); + + const [{ now }] = await (bucketStorage.db as VersionedPowerSyncMongoV3).db + .aggregate<{ now: Date }>([{ $documents: [{}] }, { $project: { _id: 0, now: '$$NOW' } }]) + .toArray(); + const result = await bucketStorage.compactInitialReplication({ + maxOpId: checkpoint, + signal: AbortSignal.timeout(2_000) + }); + + expect(result.buckets).toBe(0); + const states = await bucketStateCollection.find({}).toArray(); + expect(states).toHaveLength(2); + for (const state of states) { + expect(state.next_compact_check!.getTime()).toBeGreaterThan( + now.getTime() + DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS + ); + } + }); + test('v3 replication writes initialize scheduled compaction state', async () => { const { bucketStorage } = await setup(); const storageDb = bucketStorage.db; From 95e1163de8495dc398a63eecd2229ac08680eba4 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 17 Aug 2026 11:12:27 +0200 Subject: [PATCH 33/38] Fix stats calculation for chunked compact. --- .../implementation/v3/MongoCompactorV3.ts | 23 ++++++---- .../test/src/storage_compacting.test.ts | 46 +++++++++++++++++++ 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 9e6c1e18c..c36e289c7 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -10,7 +10,6 @@ import { loadBucketDataDocument, maxOpId, serializeBucketData } from './bucket-f import { BucketDataContextV3 } from './BucketDataContextV3.js'; import { DEFAULT_MAX_DOC_SIZE_BYTES } from './chunking.js'; import { - applyCompactionDelta, applyStatsReplacement, bucketStats, BucketStatsWithChecksum, @@ -345,7 +344,6 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC let compactedOpId: bigint | null = null; let overlappingCompactedChunk: BucketStatsWithChecksum | undefined; - let preCompactionTail = emptyBucketStats(); let compactedTail = emptyBucketStats(); let pendingChunks: BucketDataDocumentV3[] = []; let pendingSize = 0; @@ -394,7 +392,6 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC for (const doc of batch.documents) { compactedOpId = maxOpId(compactedOpId, doc._id.o); const documentStats = statsForDocument(doc); - preCompactionTail = combineAdjacentStats(preCompactionTail, documentStats); if (context.state.compacted_state?.op_id === doc._id.o) { overlappingCompactedChunk = documentStats; } @@ -437,12 +434,22 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC // as a conservative resume hint. result = await this.readAuthoritativeCompactionResult(context, bucketContext, compactedOpId); } else { + const compactedState = + previousCompactedState == null + ? compactedTail + : combineChunkStats(previousCompactedState, compactedTail, overlappingCompactedChunk!); + const tailStats = + compactedOpId == context.lastOp + ? undefined + : await this.readBucketStats( + bucket, + resolvedDefinitionId, + context.lastOp, + bucketContext.docId(compactedOpId) + ); result = { - compactedState: - previousCompactedState == null || overlappingCompactedChunk == null - ? compactedTail - : combineChunkStats(previousCompactedState, compactedTail, overlappingCompactedChunk), - bucketStats: applyCompactionDelta(bucketStats(context.state), preCompactionTail, compactedTail) + compactedState, + bucketStats: tailStats == null ? compactedState : combineAdjacentStats(compactedState, tailStats) }; } diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 44c92d2a0..d304ffe62 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -1757,6 +1757,52 @@ bucket_definitions: }); }); + test('capped chunk compaction repairs stale bucket stats after a committed first merge', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); + const operations = [ + makeOp(1, 'A', 'a', ctx, sourceTableId), + makeOp(2, 'B', 'b', ctx, sourceTableId), + makeOp(3, 'C', 'c', ctx, sourceTableId), + makeOp(4, 'D', 'd', ctx, sourceTableId) + ]; + const originalDocuments = [ + serializeBucketData(BUCKET, operations.slice(0, 2)), + serializeBucketData(BUCKET, operations.slice(2)) + ]; + const committedMerge = serializeBucketData(BUCKET, operations); + const untouchedTail = serializeBucketData(BUCKET, [makeOp(5, 'E', 'e', ctx, sourceTableId)]); + await insertDocs(collection, [committedMerge, untouchedTail]); + await bucketStateCollection.insertOne({ + _id: { d: ctx.definitionId, b: BUCKET }, + last_op: 5n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { + count: originalDocuments.reduce((total, document) => total + document.count, untouchedTail.count), + bytes: BigInt(originalDocuments.reduce((total, document) => total + document.size, untouchedTail.size)), + chunks: originalDocuments.length + 1 + } + }); + + const result = await bucketStorage.compactInitialReplication({ maxOpId: 4n }); + + expect(result).toEqual({ buckets: 1 }); + const expectedStats = { + count: committedMerge.count + untouchedTail.count, + bytes: BigInt(committedMerge.size + untouchedTail.size), + chunks: 2 + }; + const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(state?.bucket_stats).toEqual(expectedStats); + expect(state?.compacted_state).toMatchObject({ + op_id: 4n, + checksum: committedMerge.checksum, + count: committedMerge.count, + bytes: BigInt(committedMerge.size), + chunks: 1 + }); + }); + test('full compaction retry rebuilds stats after a committed replacement', async () => { const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); const documents = [ From 92af0601c5528e79e3459feacf9697fd16ffbd1f Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 17 Aug 2026 11:26:22 +0200 Subject: [PATCH 34/38] Avoid repeatedly re-scheduling a full compact under concurrent writes. --- docs/storage/v3-compaction-design.md | 4 +- .../implementation/v3/MongoCompactorV3.ts | 28 ++++++--- .../src/storage/implementation/v3/models.ts | 7 ++- .../test/src/storage_compacting.test.ts | 60 +++++++++++++++++++ 4 files changed, 86 insertions(+), 13 deletions(-) diff --git a/docs/storage/v3-compaction-design.md b/docs/storage/v3-compaction-design.md index e0b2a4416..372fc93fc 100644 --- a/docs/storage/v3-compaction-design.md +++ b/docs/storage/v3-compaction-design.md @@ -51,7 +51,7 @@ The delay before a full compact is inversely related to the amount of work since To assist with deciding whether to perform a full compact, a chunk compact, or no compact, we store current aggregate counts and sizes, a cached snapshot at the latest compact, and the figures from the last full compact needed to schedule the next one. -To allow configuring minimum and maximum intervals between compacts, we also store timestamps of the oldest uncompacted write, the latest compact, and the most recent full compact. +To allow configuring minimum and maximum intervals between compacts, we also store a scheduling timestamp for outstanding full-compaction work, the latest compact, and the most recent full compact. Writers initially set that scheduling timestamp from the first uncompacted write. A full compact that leaves a capped or concurrently-written tail advances it to the completion time, starting a fresh scheduling window for the remaining work. ## Concurrency @@ -65,7 +65,7 @@ To cater for this, the compact process calculates the delta of statistics while Some care needs to be taken to take into account the "tail" of a bucket that exists while compacting, but cannot be included in the compact job. -`next_compact_check` and `first_uncompacted_write` are also affected by this: We cannot unilaterally clear these values if there were further writes to the bucket while compacting, but we also do not capture new values for writes during compacting. If writes are detected while finalizing (by checking `last_op` for the bucket), the compact process retains conservative scheduling values for them. Since they are only used for scheduling, the values do not have to be exact, as long as they are set. +`next_compact_check` and `first_uncompacted_write` are also affected by this. A full compact clears them only when it reaches the claimed bucket head and no later writes are visible during finalization. If a configured operation limit or concurrent writes leave a tail, the compact records statistics for the prefix it covered and advances `first_uncompacted_write` to the database completion time. This prevents common concurrent writes from immediately triggering another full scan while keeping the tail scheduled. Since the timestamp is only used for scheduling, it does not have to equal the oldest tail write exactly. ## Initial replication diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index c36e289c7..5643db10b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -203,6 +203,10 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC forceKind == null ? claimedDecision.kind : forcedCompactionKind(lease.state, forceKind, this); if (claimedKind == null) { await this.rescheduleClaimedBucket(lease, claimedDecision, rescheduleNotBefore); + } else if (claimedKind == CompactionKind.Full && this.isFullCompactionTargetCovered(lease.state)) { + // Outstanding writes may all be above this run's fixed op limit. + // Keep them scheduled without rescanning an already-compacted prefix. + await this.rescheduleClaimedBucket(lease, claimedDecision, rescheduleNotBefore); } else { await this.compactClaimedBucket(lease, claimedKind, claimedDecision, rescheduleNotBefore); } @@ -301,6 +305,11 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC return this.maxOpIdCap == null || context.lastOp < this.maxOpIdCap ? context.lastOp : this.maxOpIdCap; } + private isFullCompactionTargetCovered(state: BucketStateDocumentV3): boolean { + const maxOpId = this.maxOpIdCap == null || state.last_op < this.maxOpIdCap ? state.last_op : this.maxOpIdCap; + return state.last_full_compact != null && state.last_full_compact.op_id >= maxOpId; + } + private get objectStorageLifecycle(): ObjectStorageLifecycle { if (!this.storage.objectStorage) { throw new Error('Object storage is not configured'); @@ -523,10 +532,11 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC bytes: compactionResult.bucketStats.bytes - startedStats.bytes, chunks: compactionResult.bucketStats.chunks - startedStats.chunks }; - const coveredStart = compactedOpId >= context.lastOp; + const coveredClaimedHead = compactedOpId >= context.lastOp; const concurrentWriteCheck = { $gt: ['$last_op', context.lastOp] }; - const nextAfterConcurrentWrite = this.rescheduleAtOrAfter( - new Date(context.startedAt.getTime() + this.minCompactChunkIntervalMs), + const remainingFullWorkCheck = coveredClaimedHead ? concurrentWriteCheck : true; + const nextAfterPartialFullCompact = this.rescheduleAtOrAfter( + { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.minCompactChunkIntervalMs } }, context.rescheduleNotBefore ); const nextCheckForUncompactedWork = this.rescheduleAtOrAfter( @@ -553,18 +563,18 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC chunks: { $add: ['$bucket_stats.chunks', delta.chunks] } }, first_uncompacted_write: - context.kind == CompactionKind.Full && coveredStart - ? { $cond: [concurrentWriteCheck, context.startedAt, '$$REMOVE'] } + context.kind == CompactionKind.Full + ? { $cond: [remainingFullWorkCheck, '$$NOW', '$$REMOVE'] } : '$first_uncompacted_write', next_compact_check: - context.kind == CompactionKind.Full && coveredStart - ? { $cond: [concurrentWriteCheck, nextAfterConcurrentWrite, '$$REMOVE'] } + context.kind == CompactionKind.Full + ? { $cond: [remainingFullWorkCheck, nextAfterPartialFullCompact, '$$REMOVE'] } : nextCheckForUncompactedWork }; - if (context.kind == CompactionKind.Full && coveredStart) { + if (context.kind == CompactionKind.Full) { update.last_full_compact = { op_id: compactedOpId, - count: compactionResult.bucketStats.count, + count: compactionResult.compactedState.count, puts, at: '$$NOW' }; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts index 7f52bcfaa..3e2273c3e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts @@ -174,7 +174,10 @@ export interface BucketStateDocumentV3 { /** The next time a compact worker should inspect this bucket. */ next_compact_check: Date | undefined; - /** The oldest write that has not been covered by a full compact. */ + /** + * Scheduling epoch for work not covered by a full compact. Writers set the + * first actual write time; a partial full compact advances it to completion. + */ first_uncompacted_write: Date | undefined; /** @@ -191,7 +194,7 @@ export interface BucketStateDocumentV3 { chunks: number; }; - /** Statistics from the most recent full compact. */ + /** Statistics for the prefix covered by the most recent full compact. */ last_full_compact?: { op_id: InternalOpId; count: number; diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index d304ffe62..0a2631068 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -1152,6 +1152,66 @@ bucket_definitions: expect(state?.compact_lease).toBeUndefined(); }); + test('capped full compaction records its prefix and starts a fresh scheduling window', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); + const prefix = serializeBucketData(BUCKET, [makeOp(1, 'A', 'a', ctx, sourceTableId)]); + const untouchedTail = serializeBucketData(BUCKET, [makeOp(2, 'B', 'b', ctx, sourceTableId)]); + await insertDocs(collection, [prefix, untouchedTail]); + await bucketStateCollection.insertOne({ + _id: { d: ctx.definitionId, b: BUCKET }, + last_op: 2n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { + count: prefix.count + untouchedTail.count, + bytes: BigInt(prefix.size + untouchedTail.size), + chunks: 2 + } + }); + + await bucketStorage.compact({ maxOpId: 1n }); + + const partialState = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(partialState?.last_full_compact).toMatchObject({ + op_id: 1n, + count: prefix.count, + puts: 1 + }); + expect(partialState?.first_uncompacted_write).toBeInstanceOf(Date); + expect(partialState!.first_uncompacted_write!.getTime()).toBeGreaterThan(0); + expect(partialState!.next_compact_check!.getTime()).toBeGreaterThanOrEqual( + partialState!.first_uncompacted_write!.getTime() + DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS + ); + + const lastFullCompactAt = partialState!.last_full_compact!.at; + const freshFirstUncompactedWrite = partialState!.first_uncompacted_write!; + await bucketStateCollection.updateOne( + { _id: { d: ctx.definitionId, b: BUCKET } }, + { $set: { next_compact_check: new Date(0) } } + ); + + await bucketStorage.compact({ maxOpId: 2n }); + + const deferredState = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(deferredState?.last_full_compact?.at).toEqual(lastFullCompactAt); + expect(deferredState?.first_uncompacted_write).toEqual(freshFirstUncompactedWrite); + expect(deferredState!.next_compact_check!.getTime()).toBeGreaterThan(Date.now()); + + // Even after the scheduling window has elapsed, a run with the old cap + // must not scan and record the same full-compaction prefix again. + await bucketStateCollection.updateOne( + { _id: { d: ctx.definitionId, b: BUCKET } }, + { $set: { first_uncompacted_write: new Date(0), next_compact_check: new Date(0) } } + ); + + await bucketStorage.compact({ maxOpId: 1n }); + + const rescheduledState = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(rescheduledState?.last_full_compact?.at).toEqual(lastFullCompactAt); + expect(rescheduledState?.first_uncompacted_write).toEqual(new Date(0)); + expect(rescheduledState!.next_compact_check!.getTime()).toBeGreaterThan(0); + }); + test('explicit compaction skips a bucket with no outstanding full-compaction work', async () => { const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); const document = serializeBucketData(BUCKET, [makeOp(1, 'A', 'value', ctx, sourceTableId)]); From 3337141e6e900af3e1e6e483559e37f7703afe72 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 17 Aug 2026 12:16:51 +0200 Subject: [PATCH 35/38] Simplify retries. --- .../implementation/v3/CompactionLease.ts | 5 -- .../implementation/v3/MongoCompactorV3.ts | 4 +- .../test/src/storage_compacting.test.ts | 59 +++++++++---------- .../storage_s3_compaction_lifecycle.test.ts | 8 ++- 4 files changed, 35 insertions(+), 41 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts index 6dc4dc1c4..546bccd39 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts @@ -94,11 +94,6 @@ export class CompactionLease implements AsyncDisposable { } } - /** Allow a retry after a transient error during a fenced final update. */ - restartFinalization() { - this.finalizing = false; - } - async reschedule(nextCompactCheck: mongo.Document) { await this.finish([{ $set: { next_compact_check: nextCompactCheck } }, { $unset: 'compact_lease' }]); } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 5643db10b..c5b0e3cd9 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -290,7 +290,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC ) { const context = new CompactionContext(lease, kind, decision, rescheduleNotBefore); lease.startRenewal(); - await this.retryCompaction(context.state._id.b, () => this.compactSingleBucket(context)); + await this.compactSingleBucket(context); } private async rescheduleClaimedBucket(lease: CompactionLease, decision: CompactionDecision, notBefore?: Date) { @@ -318,8 +318,6 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC } private async compactSingleBucket(context: CompactionContext) { - // A retry restarts finalization after a transient replacement failure. - context.lease.restartFinalization(); if (context.kind == CompactionKind.Chunks) { return this.compactSingleBucketChunks(context); } diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 0a2631068..c68b99b2e 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -1863,7 +1863,7 @@ bucket_definitions: }); }); - test('full compaction retry rebuilds stats after a committed replacement', async () => { + test('later full compaction claims fresh state after a committed replacement', async () => { const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); const documents = [ serializeBucketData(BUCKET, [makeOp(1, 'A', 'a', ctx, sourceTableId)]), @@ -1884,13 +1884,26 @@ bucket_definitions: } }); - const compactor = bucketStorage.createMongoCompactor({ maxOpId: 4n, compactBuckets: [BUCKET] }); + const concurrentDocument = serializeBucketData(BUCKET, [makeOp(5, 'E', 'e', ctx, sourceTableId)]); + const compactor = bucketStorage.createMongoCompactor({ maxOpId: 5n, compactBuckets: [BUCKET] }); const originalFlush = (compactor as any).flushCompactionGroup.bind(compactor); let injectedFailure = false; vi.spyOn(compactor as any, 'flushCompactionGroup').mockImplementation(async (...args: any[]) => { const result = await originalFlush(...args); if (!injectedFailure) { injectedFailure = true; + await insertDocs(collection, [concurrentDocument]); + await bucketStateCollection.updateOne( + { _id: { d: ctx.definitionId, b: BUCKET } }, + { + $set: { last_op: 5n }, + $inc: { + 'bucket_stats.count': concurrentDocument.count, + 'bucket_stats.bytes': BigInt(concurrentDocument.size), + 'bucket_stats.chunks': 1 + } + } + ); throw new ObjectStorageError('failure after committed full-compaction replacement', { cause: new Error('socket reset'), retryable: true @@ -1899,9 +1912,18 @@ bucket_definitions: return result; }); - await expect(compactor.compact()).resolves.toBe(1); + await expect(compactor.compact()).rejects.toThrow('failure after committed full-compaction replacement'); expect(injectedFailure).toBe(true); + const interruptedState = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(interruptedState?.last_op).toBe(5n); + expect(interruptedState?.compacted_state).toBeUndefined(); + expect(interruptedState?.compact_lease).toBeUndefined(); + + await expect(bucketStorage.createMongoCompactor({ maxOpId: 5n, compactBuckets: [BUCKET] }).compact()).resolves.toBe( + 1 + ); + const currentDocuments = await collection.find({ '_id.b': BUCKET }).toArray(); expect(currentDocuments).toHaveLength(1); const expectedStats = { @@ -1911,33 +1933,10 @@ bucket_definitions: }; const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); expect(state?.bucket_stats).toEqual(expectedStats); - expect(state?.compacted_state).toMatchObject({ op_id: 4n, ...expectedStats }); - }); - - test('chunk compaction retry rebuilds state after a committed partial merge', async () => { - const { bucketStorage, collection, bucketStateCollection, ctx } = await setupCompactedTail(); - const compactor = bucketStorage.createMongoCompactor({ - maxOpId: 4n, - compactChunksOnly: true - }); - const originalFlush = (compactor as any).flushCompactionGroup.bind(compactor); - let injectedFailure = false; - vi.spyOn(compactor as any, 'flushCompactionGroup').mockImplementation(async (...args: any[]) => { - const result = await originalFlush(...args); - if (!injectedFailure) { - injectedFailure = true; - throw new ObjectStorageError('failure after committed chunk merge', { - cause: new Error('socket reset'), - retryable: true - }); - } - return result; - }); - - await expect(compactor.compact()).resolves.toBe(1); - - expect(injectedFailure).toBe(true); - await expectRecoveredCompactedTail(collection, bucketStateCollection, ctx.definitionId); + expect(state?.compacted_state).toMatchObject({ op_id: 5n, ...expectedStats }); + expect(state?.last_full_compact?.op_id).toBe(5n); + expect(state?.first_uncompacted_write).toBeUndefined(); + expect(state?.next_compact_check).toBeUndefined(); }); test('chunk compaction treats a missing cached op as a resume hint', async () => { diff --git a/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts b/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts index 34c3400a3..03d66c92a 100644 --- a/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts @@ -29,7 +29,7 @@ function memoryS3Factory(options: { inlineThresholdBytes?: number } = {}) { } describe('S3 compaction storage lifecycle', () => { - test('retries transient object storage failures', async () => { + test('a later compaction recovers from a transient object storage failure', async () => { const { memoryStorage, factory: factoryGen } = memoryS3Factory(); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); @@ -63,12 +63,14 @@ describe('S3 compaction storage lifecycle', () => { const checkpoint = await bucketStorage.getCheckpoint(); const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n); - await compactActive(factory, { + const compactOptions = { maxOpId: checkpoint.checkpoint, compactBuckets: [request.bucket], minBucketChanges: 1, minChangeRatio: 0 - }); + }; + await expect(compactActive(factory, compactOptions)).rejects.toThrow('temporary object storage failure'); + await compactActive(factory, compactOptions); expect(injectedFailure).toBe(true); const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request])); From 94847a0aece13e881aa95e8c00a9ff7237728aa3 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 17 Aug 2026 12:31:16 +0200 Subject: [PATCH 36/38] Avoid rework if compaction target is covered. --- .../implementation/v3/MongoCompactorV3.ts | 40 ++++++++++++++----- .../implementation/v3/compact-utils.ts | 3 +- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index c5b0e3cd9..ca1400570 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -126,6 +126,9 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC if (lease == null || lease.state.first_uncompacted_write == null) { continue; } + if (this.isCompactionTargetCovered(lease.state, CompactionKind.Full)) { + continue; + } const decision = chooseCompactionKind(lease.state, lease.startedAt, this); await this.compactClaimedBucket(lease, CompactionKind.Full, decision); } @@ -203,9 +206,9 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC forceKind == null ? claimedDecision.kind : forcedCompactionKind(lease.state, forceKind, this); if (claimedKind == null) { await this.rescheduleClaimedBucket(lease, claimedDecision, rescheduleNotBefore); - } else if (claimedKind == CompactionKind.Full && this.isFullCompactionTargetCovered(lease.state)) { - // Outstanding writes may all be above this run's fixed op limit. - // Keep them scheduled without rescanning an already-compacted prefix. + } else if (this.isCompactionTargetCovered(lease.state, claimedKind)) { + // The run cannot advance this kind's watermark without regressing + // already-published progress. Keep any newer work scheduled. await this.rescheduleClaimedBucket(lease, claimedDecision, rescheduleNotBefore); } else { await this.compactClaimedBucket(lease, claimedKind, claimedDecision, rescheduleNotBefore); @@ -288,7 +291,13 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC decision: CompactionDecision, rescheduleNotBefore?: Date ) { - const context = new CompactionContext(lease, kind, decision, rescheduleNotBefore); + const context = new CompactionContext( + lease, + kind, + decision, + rescheduleNotBefore, + this.compactionTarget(lease.state) + ); lease.startRenewal(); await this.compactSingleBucket(context); } @@ -301,13 +310,22 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC return notBefore == null ? nextCompactCheck : { $max: [nextCompactCheck, notBefore] }; } - private compactMaxOpId(context: CompactionContext): InternalOpId { - return this.maxOpIdCap == null || context.lastOp < this.maxOpIdCap ? context.lastOp : this.maxOpIdCap; + private compactionTarget(state: BucketStateDocumentV3): InternalOpId { + return this.maxOpIdCap == null || state.last_op < this.maxOpIdCap ? state.last_op : this.maxOpIdCap; } - private isFullCompactionTargetCovered(state: BucketStateDocumentV3): boolean { - const maxOpId = this.maxOpIdCap == null || state.last_op < this.maxOpIdCap ? state.last_op : this.maxOpIdCap; - return state.last_full_compact != null && state.last_full_compact.op_id >= maxOpId; + private isCompactionTargetCovered(state: BucketStateDocumentV3, kind: CompactionKind): boolean { + const target = this.compactionTarget(state); + if (kind == CompactionKind.Chunks) { + return state.compacted_state != null && state.compacted_state.op_id >= target; + } + if (state.last_full_compact != null && state.last_full_compact.op_id >= target) { + return true; + } + // A full compact may change counts before the checksum-cache boundary. + // Wait for the safe target to catch up instead of publishing an older or + // stale cache. At the same boundary, full coverage can still advance. + return state.compacted_state != null && state.compacted_state.op_id > target; } private get objectStorageLifecycle(): ObjectStorageLifecycle { @@ -347,7 +365,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC context.state.compacted_state?.op_id != null && context.state.compacted_state.op_id > 0n ? bucketContext.docId(context.state.compacted_state.op_id - 1n) : bucketContext.minId; - const upperBound = bucketContext.docId(this.compactMaxOpId(context) + 1n); + const upperBound = bucketContext.docId(context.targetOp + 1n); let compactedOpId: bigint | null = null; let overlappingCompactedChunk: BucketStatsWithChecksum | undefined; @@ -684,7 +702,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC const collection = this.db.bucketData(this.group_id, resolvedDefinitionId); const dataContext = { replicationStreamId: this.group_id, definitionId: resolvedDefinitionId }; const lowerBound = bucketContext.minId; - let upperBound = bucketContext.docId(this.compactMaxOpId(context) + 1n); + let upperBound = bucketContext.docId(context.targetOp + 1n); let totalOpCount = 0; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/compact-utils.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/compact-utils.ts index 1366486cc..51d754ee9 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/compact-utils.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/compact-utils.ts @@ -59,7 +59,8 @@ export class CompactionContext { readonly lease: CompactionLease, readonly kind: CompactionKind, readonly decision: CompactionDecision, - readonly rescheduleNotBefore: Date | undefined + readonly rescheduleNotBefore: Date | undefined, + readonly targetOp: InternalOpId ) {} get state() { From 7d916725609b702f80908e664bb88a2015153800 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 17 Aug 2026 12:43:41 +0200 Subject: [PATCH 37/38] Simplify chunked compaction retries. --- .../implementation/v3/MongoCompactorV3.ts | 94 +++++++------------ .../test/src/storage_compacting.test.ts | 10 +- 2 files changed, 40 insertions(+), 64 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index ca1400570..58cfcc159 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -359,13 +359,18 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC }); const collection = this.db.bucketData(this.group_id, resolvedDefinitionId); const dataContext = { replicationStreamId: this.group_id, definitionId: resolvedDefinitionId }; + let previousCompactedState = context.state.compacted_state; + // A zero boundary represents an empty prefix, so there is no stored chunk + // whose statistics need to be carried into this pass. + if (previousCompactedState?.op_id === 0n) { + previousCompactedState = undefined; + } // Include the last previously compacted chunk as well as new chunks. It // is the only old chunk which can become mergeable with the new tail. let lowerBound = - context.state.compacted_state?.op_id != null && context.state.compacted_state.op_id > 0n - ? bucketContext.docId(context.state.compacted_state.op_id - 1n) - : bucketContext.minId; + previousCompactedState != null ? bucketContext.docId(previousCompactedState.op_id - 1n) : bucketContext.minId; const upperBound = bucketContext.docId(context.targetOp + 1n); + let cachedBoundaryToVerify = previousCompactedState?.op_id; let compactedOpId: bigint | null = null; let overlappingCompactedChunk: BucketStatsWithChecksum | undefined; @@ -410,6 +415,20 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC } ); + if (cachedBoundaryToVerify != null) { + const cachedBoundary = cachedBoundaryToVerify; + cachedBoundaryToVerify = undefined; + if (batch.documents[0]?._id.o !== cachedBoundary) { + // A previous attempt may have replaced the cached boundary before + // finalizing bucket state. Keep the persisted cache available to + // readers, but ignore it in this attempt and calculate its + // replacement through the normal scan from the bucket beginning. + previousCompactedState = undefined; + lowerBound = bucketContext.minId; + continue; + } + } + if (batch.documents.length == 0) { break; } @@ -417,7 +436,7 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC for (const doc of batch.documents) { compactedOpId = maxOpId(compactedOpId, doc._id.o); const documentStats = statsForDocument(doc); - if (context.state.compacted_state?.op_id === doc._id.o) { + if (previousCompactedState?.op_id === doc._id.o) { overlappingCompactedChunk = documentStats; } @@ -449,34 +468,18 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC return; } - const previousCompactedState = context.state.compacted_state; - let result: CompactionResult; - if (previousCompactedState != null && overlappingCompactedChunk == null) { - // A previous attempt may have committed document replacements without - // finalizing bucket state. When the exact cached chunk no longer exists, - // the cached prefix cannot be combined with the current tail. Rebuild the - // metadata from the authoritative documents while keeping the old op id - // as a conservative resume hint. - result = await this.readAuthoritativeCompactionResult(context, bucketContext, compactedOpId); - } else { - const compactedState = - previousCompactedState == null - ? compactedTail - : combineChunkStats(previousCompactedState, compactedTail, overlappingCompactedChunk!); - const tailStats = - compactedOpId == context.lastOp - ? undefined - : await this.readBucketStats( - bucket, - resolvedDefinitionId, - context.lastOp, - bucketContext.docId(compactedOpId) - ); - result = { - compactedState, - bucketStats: tailStats == null ? compactedState : combineAdjacentStats(compactedState, tailStats) - }; - } + const compactedState = + previousCompactedState == null + ? compactedTail + : combineChunkStats(previousCompactedState, compactedTail, overlappingCompactedChunk!); + const tailStats = + compactedOpId == context.lastOp + ? undefined + : await this.readBucketStats(bucket, resolvedDefinitionId, context.lastOp, bucketContext.docId(compactedOpId)); + const result: CompactionResult = { + compactedState, + bucketStats: tailStats == null ? compactedState : combineAdjacentStats(compactedState, tailStats) + }; await this.finalizeCompactedBucket({ context, compactedOpId, compactionResult: result, puts: 0 }); this.compactedBucketCount++; @@ -617,33 +620,6 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC ); } - /** - * Recover chunk statistics from the data that is currently stored. - * - * Chunk compaction normally calculates statistics while processing its - * working range. If a previous attempt replaced the cached boundary before - * failing, that range no longer contains enough information to update the - * older cached prefix. In that case, rebuild the prefix and include any - * untouched data after an operation limit in the bucket total. - */ - private async readAuthoritativeCompactionResult( - context: CompactionContext, - bucketContext: BucketDataContextV3, - compactedOpId: InternalOpId - ): Promise { - const { bucket, definitionId } = bucketContext.key; - const [compactedState, tailStats] = await Promise.all([ - this.readBucketStats(bucket, definitionId, compactedOpId), - compactedOpId == context.lastOp - ? Promise.resolve(undefined) - : this.readBucketStats(bucket, definitionId, context.lastOp, bucketContext.docId(compactedOpId)) - ]); - return { - compactedState, - bucketStats: tailStats == null ? compactedState : combineAdjacentStats(compactedState, tailStats) - }; - } - /** * Read bucket stats directly from bucket_data documents. */ diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index c68b99b2e..78c2e5012 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -1774,7 +1774,6 @@ bucket_definitions: async function expectRecoveredCompactedTail(collection: any, bucketStateCollection: any, definitionId: string) { const documents = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray(); - expect(documents).toHaveLength(2); expect(documents.flatMap((document: BucketDataDocumentV3) => document.ops!.map((op) => op.o))).toEqual([ 1n, 2n, @@ -1789,9 +1788,9 @@ bucket_definitions: checksum: 70n, count: 4, bytes, - chunks: 2 + chunks: documents.length }); - expect(state?.bucket_stats).toEqual({ count: 4, bytes, chunks: 2 }); + expect(state?.bucket_stats).toEqual({ count: 4, bytes, chunks: documents.length }); } test('initial compaction merges small chunks and refreshes bucket metadata', async () => { @@ -1968,6 +1967,7 @@ bucket_definitions: ); const currentDocuments = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray(); + expect(currentDocuments.flatMap((document) => document.ops!.map((op) => op.o))).toEqual([1n, 3n, 4n]); const bytes = BigInt(currentDocuments.reduce((total, document) => total + document.size, 0)); const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); expect(state?.compacted_state).toMatchObject({ @@ -1975,9 +1975,9 @@ bucket_definitions: checksum: 56n, count: 3, bytes, - chunks: 2 + chunks: currentDocuments.length }); - expect(state?.bucket_stats).toEqual({ count: 3, bytes, chunks: 2 }); + expect(state?.bucket_stats).toEqual({ count: 3, bytes, chunks: currentDocuments.length }); }); test('later chunk compaction rebuilds state after a committed partial merge', async () => { From b5a34182a7158da6d841c94dd2cb418997090b6a Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 17 Aug 2026 13:14:19 +0200 Subject: [PATCH 38/38] Don't reschedule a batch when aborting. --- .../implementation/v3/MongoCompactorV3.ts | 5 ++ .../test/src/storage_compacting.test.ts | 51 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 58cfcc159..38998b9e6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -214,6 +214,11 @@ export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalC await this.compactClaimedBucket(lease, claimedKind, claimedDecision, rescheduleNotBefore); } } catch (error) { + if (this.signal?.aborted) { + // When aborted, stop completely, rather than logging and re-scheduling individual buckets. + // The lease on the current bucket is still released automatically. + throw error; + } await this.rescheduleFailedBucket(state, rescheduleNotBefore, error); } } diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 78c2e5012..258acf9b1 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -7,6 +7,7 @@ import { CompactionLease } from '@module/storage/implementation/v3/CompactionLea import { BucketDataDocumentV3 } from '@module/storage/implementation/v3/models.js'; import { ObjectStorageError } from '@module/storage/implementation/v3/object-storage/ObjectStorage.js'; import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; +import { logger as defaultLogger } from '@powersync/lib-services-framework'; import { addChecksums, CheckpointChecksumInvalidatedError, @@ -452,6 +453,56 @@ bucket_definitions: expect(goodState?.compacted_state?.op_id).toBe(2n); }); + test('aborting scheduled compaction does not reschedule the remaining batch', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3Storage(); + const buckets = ['first[]', 'second[]', 'third[]']; + const documents = buckets.map((bucket) => + serializeBucketData(bucket, [makeOp(2, bucket, bucket, { ...ctx, bucket }, sourceTableId)]) + ); + await insertDocs(collection, documents); + await bucketStateCollection.insertMany( + documents.map((document, index) => ({ + _id: { d: ctx.definitionId, b: buckets[index] }, + last_op: 2n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { count: document.count, bytes: BigInt(document.size), chunks: 1 } + })) + ); + + const abortController = new AbortController(); + const testLogger = defaultLogger.child({}); + vi.spyOn(testLogger, 'info').mockImplementation((message: unknown) => { + if (typeof message == 'string' && message.startsWith('Compacted bucket chunks ')) { + abortController.abort(); + } + return testLogger; + }); + const errorLog = vi.spyOn(testLogger, 'error'); + + await expect( + bucketStorage + .createMongoCompactor({ + maxOpId: 2n, + compactChunksOnly: true, + signal: abortController.signal, + logger: testLogger + }) + .compact() + ).rejects.toThrow(); + + const states = await bucketStateCollection.find({ '_id.b': { $in: buckets } }).toArray(); + const completed = states.filter((state) => state.compacted_state != null); + const remaining = states.filter((state) => state.compacted_state == null); + expect(completed).toHaveLength(1); + expect(remaining).toHaveLength(2); + for (const state of remaining) { + expect(state.next_compact_check).toEqual(new Date(0)); + expect(state.compact_lease).toBeUndefined(); + } + expect(errorLog).not.toHaveBeenCalled(); + }); + test('1. ops[] ordering - preserves caller ordering (no implicit sort)', () => { const ops = [ makeBucketDataDoc({ o: 5n, data: '{"id":"c"}' }),