diff --git a/.changeset/wacky-impalas-vanish.md b/.changeset/wacky-impalas-vanish.md new file mode 100644 index 000000000..f9e4afd45 --- /dev/null +++ b/.changeset/wacky-impalas-vanish.md @@ -0,0 +1,12 @@ +--- +'@powersync/service-module-mongodb-storage': minor +'@powersync/service-core': minor +'@powersync/service-module-postgres-storage': patch +'@powersync/service-core-tests': patch +'@powersync/service-module-postgres': patch +'@powersync/service-module-mongodb': patch +'@powersync/service-module-convex': patch +'@powersync/service-module-mysql': patch +--- + +Restructure MongoDB V3 bucket compacting. diff --git a/AGENTS.md b/AGENTS.md index 6363cbaf8..005732503 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,3 +66,12 @@ After loading the relevant spec context, inspect the closest existing implementa - Import real test storage factories from storage modules, such as `@powersync/service-module-mongodb-storage` and `@powersync/service-module-postgres-storage`, and use `describeWithStorage`-style coverage where practical. - Add a stream test context for new modules, following existing examples such as `WalStreamTestContext`, `ChangeStreamTestContext`, `BinlogStreamTestContext`, `CDCStreamTestContext`, or `ConvexStreamTestContext`. - For when to use spies versus mocks, follow the General Workflow testing guidance above. + +### Backwards-compatibility + +The NPM packages here do not follow semver: We do not guarantee any backwards-compatibility on package APIs. Backwards-compatibility is only relevant for the service itself. + +For storage: + +1. We preserve backwards-compatibility for Postres storage and MongoDB storage with storage_version: 1 and 2, and future even versions. +2. We do not preserve backwards-compatibility for MongoDB storage with storage_version: 3, or any future odd version numbers. diff --git a/docs/storage/v3-compaction-design.md b/docs/storage/v3-compaction-design.md new file mode 100644 index 000000000..372fc93fc --- /dev/null +++ b/docs/storage/v3-compaction-design.md @@ -0,0 +1,87 @@ +# V3 Compaction Design + +This describes the design of compaction scheduling in MongoDB storage V3. For details on what compaction means on a protocol level, see [./compacting-operations.md](./compacting-operations.md). + +## Goals + +Compaction should: + +- make background work proportional to modified buckets and their modified data, rather than proportional to the overall number of buckets and/or operations in a stream. +- support frequent, resumable, concurrent runs; +- avoid turning regular small writes into repeated full rewrites; and +- ensure that every bucket with outstanding work eventually receives a full compact. + +The design makes the bucket-state collection a persistent work queue. It deliberately replaces the V1-style dirty-operation estimate with explicit scheduling and state captured at the last compact. + +## Scheduling + +In earlier versions, compaction required a scheduled job that would either: + +1. Iterate through all buckets, filter them according to stats in the bucket-state collection, then compact if needed. This was not safe for interruption. +2. Iterate through all buckets with `estimate_since_compact.count >= 10` or similar indexed condition, perform additional filtering, then compact. This had better resumability, but could repeatedly re-compact the same busy buckets, and fail to keep up with incoming changes on others. + +For the scheduling approach here, we instead focus on a single new field: `next_compact_check`. This field is indexed and populated both during replication and during compaction. It supports multiple different scenarios: + +1. New data was added to the bucket while replicating, which may or may not need a compact. This schedules a _check_ on the bucket. +2. A bucket was checked for compacting, but does not meet the threshold to compact just yet. Re-schedule another check for later, when the thresholds may be met. +3. A bucket was partially compacted, and may need a full compact later. Re-schedule the full compact. + +This lets an index on `next_compact_check` act as a time-prioritized compact queue. We can incrementally process this queue without reading or rewriting bucket data when a check finds no compaction work. It also keeps the logic of when to compact out of the replication worker - the replication worker only has to schedule a compact check. + +Workers may inspect a bounded batch before taking a lease. Checks already known to be no-ops are rescheduled together from their unleased snapshots. Each reschedule is conditional on the state still matching that snapshot and no lease being held, so concurrent writes or compactors make the reschedule a no-op rather than losing work. + +## Compact types + +We use two separate compact types: + +1. Full compact. This is similar to a v1 storage compact: Iterate through the bucket, replace duplicate operations with MOVE operations; squash a leading sequence of MOVE/REMOVE operations into a CLEAR operation. Additionally, this merges small chunks into larger ones if applicable. +2. Chunk compaction. This only merges small chunks into larger ones, which can be much faster. We can compute whether chunks should be merged by just reading the metadata, avoiding reading the individual operations unless we need to merge. This can also incrementally continue from the last position, instead of re-reading the entire bucket. + +Chunk compaction also replaces the separate "checksum pre-calculation" operation in MongoDB v1 storage, as a similar "fast to calculate" job. + +Both types calculate and persist checksum state for their compacted data. When using S3 storage, the benefit is significantly reduced. However, S3 storage is still opt-in, and this is cheap to calculate together with compacting, so we keep the logic for now. + +Chunk compaction is important to maintain checksum and data reading performance over large buckets. + +Full compact is required to keep bucket sizes low if the same source rows are repeatedly modified. It is also required to "expire" historical data that should not be exposed to users indefinitely. + +The delay before a full compact is inversely related to the amount of work since the previous full compact, and is capped by a maximum interval. This avoids frequent full rewrites for a small amount of new work while ensuring that outstanding work is eventually compacted. + +## Bucket stats + +To assist with deciding whether to perform a full compact, a chunk compact, or no compact, we store current aggregate counts and sizes, a cached snapshot at the latest compact, and the figures from the last full compact needed to schedule the next one. + +To allow configuring minimum and maximum intervals between compacts, we also store a scheduling timestamp for outstanding full-compaction work, the latest compact, and the most recent full compact. Writers initially set that scheduling timestamp from the first uncompacted write. A full compact that leaves a capped or concurrently-written tail advances it to the completion time, starting a fresh scheduling window for the remaining work. + +## Concurrency + +To allow running concurrent compact jobs on different buckets, we store a renewable lease for each bucket. The compact job checks out a lease on a bucket before starting any compact work. The lease helps to avoid redundant work, but it is not the only correctness mechanism: the individual compact operations are also designed to be safe under concurrency. + +## Statistics during concurrent writes + +Compaction must not hold a long transaction over bucket state, because replication must remain writable. At the same time, bucket stats must remain correct if there are new writes to a bucket while we compact it. + +To cater for this, the compact process calculates the delta of statistics while compacting, then applies that to the total state after compacting. If there are no concurrent modifications while compacting, the aggregate state converges on the result for the compacted range. + +Some care needs to be taken to take into account the "tail" of a bucket that exists while compacting, but cannot be included in the compact job. + +`next_compact_check` and `first_uncompacted_write` are also affected by this. A full compact clears them only when it reaches the claimed bucket head and no later writes are visible during finalization. If a configured operation limit or concurrent writes leave a tail, the compact records statistics for the prefix it covered and advances `first_uncompacted_write` to the database completion time. This prevents common concurrent writes from immediately triggering another full scan while keeping the tail scheduled. Since the timestamp is only used for scheduling, it does not have to equal the oldest tail write exactly. + +## Initial replication + +Initial replication uses the same scheduled work model, but forces chunk compaction and includes the first chunk-compaction interval of scheduled work. This makes the initial pass immediate and resumable without introducing a separate selection model. + +The initial pass has a fixed boundary and does not chase writes that arrive while it runs. Chunk compaction performed concurrently during replication therefore reduces the work remaining for the post-replication pass. + +## Compaction jobs + +This design gives us flexibility in how compaction could be run: + +1. Scheduled job running once per day - same as before. +2. Scheduled job running once per hour or even every 5 minutes. Actual work to perform is throttled in the job itself, and does not depend on job scheduling anymore. +3. As a tweak to the above, allow concurrent compaction jobs if the previous one has not finished. This would effectively increase the number of compaction jobs as the backlog increases. +4. Run a single process continuously polling for new buckets to compact. +5. Run multiple processes continously polling for new buckets to compact. +6. Use auto-scaling to dynamically configure the number of compaction processes depending on the backlog size. + +Note that the options 4-6 are technically feasible, but not implemented yet. diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index 83477cb97..57e2a938b 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -140,7 +140,7 @@ export class ConvexStream { const { lastOpId } = await this.initialReplication(status.snapshotLsn); if (lastOpId != null) { - await this.storage.populatePersistentChecksumCache({ + await this.storage.compactInitialReplication({ signal: this.abortSignal, maxOpId: lastOpId }); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index 269512af6..9e8aa0b27 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -173,7 +173,7 @@ function createFakeStorage(options?: { }; Object.assign(storage, { clear: vi.fn(async () => undefined), - populatePersistentChecksumCache: vi.fn(async () => ({ buckets: 0 })), + compactInitialReplication: vi.fn(async () => ({ buckets: 0 })), createWriter: vi.fn(async (_options: any) => batch), startBatch: vi.fn(async (_options: any, callback: (batch: any) => Promise) => { await callback(batch); diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index 34eeaa95c..789313f1e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -1,64 +1,27 @@ import * as timers from 'node:timers/promises'; -import { isMongoServerError, mongo, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb'; -import { - logger as defaultLogger, - Logger, - ReplicationAssertionError, - ServiceAssertionError -} from '@powersync/lib-services-framework'; -import { InternalOpId, isPartialChecksum, PopulateChecksumCacheResults, storage } from '@powersync/service-core'; -import { BucketDefinitionId } from '@powersync/service-sync-rules'; +import { isMongoServerError, mongo } from '@powersync/lib-service-mongodb'; +import { logger as defaultLogger, Logger, ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; import { BucketKey } from './common/BucketDataDoc.js'; import type { VersionedPowerSyncMongo } from './db.js'; -import { BucketStateDocumentBase } from './models.js'; import type { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js'; import { isRetryableObjectStorageError } from './v3/object-storage/ObjectStorage.js'; -export interface CurrentBucketState { - /** Bucket name */ - bucket: string; - definitionId: BucketDefinitionId; +export interface MongoCompactOptions extends storage.CompactOptions { /** - * Rows seen in the bucket, with the last op_id of each. + * Only merge adjacent V3 bucket-data chunks. This is used after initial + * replication, where reading every operation would defeat the purpose of + * the lightweight pass. */ - seen: Map; - /** - * Estimated memory usage of the seen Map. - */ - trackingSize: number; - /** - * Last (lowest) seen op_id that is not a PUT. - */ - lastNotPut: InternalOpId | null; - /** - * Number of REMOVE/MOVE operations seen since lastNotPut. - */ - opsSincePut: number; - /** - * Incrementally-updated checksum, up to maxOpId. - */ - checksum: number; - /** - * Op count for the checksum. - */ - opCount: number; - /** - * Byte size of ops covered by the checksum. - */ - opBytes: number; + compactChunksOnly?: boolean; } -export interface MongoCompactOptions extends storage.CompactOptions {} - const DEFAULT_CLEAR_BATCH_LIMIT = 5000; const DEFAULT_MOVE_BATCH_LIMIT = 2000; const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000; const DEFAULT_MOVE_BATCH_BYTE_LIMIT = 16 * 1024 * 1024; -const DEFAULT_MIN_BUCKET_CHANGES = 10; -const DEFAULT_MIN_CHANGE_RATIO = 0.1; -const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000; /** This default is primarily for tests. */ const DEFAULT_MEMORY_LIMIT_MB = 64; const COMPACTION_RETRY_LIMIT = 3; @@ -71,28 +34,30 @@ export class ConcurrentCompactionError extends Error { } } -export interface DirtyBucket { - bucket: string; - definitionId: BucketDefinitionId | null; - estimatedCount: number; - dirtyRatio?: number; +/** + * A worker whose bucket lease has been replaced must stop immediately. Unlike + * a transactional replacement conflict, retrying it with the same compactor + * instance would still use the stale lease and could race the new owner. + */ +export class CompactionLeaseLostError extends ConcurrentCompactionError { + constructor(message: string) { + super(message); + this.name = 'CompactionLeaseLostError'; + } } export abstract class MongoCompactor { - protected bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; - protected readonly idLimitBytes: number; protected readonly moveBatchLimit: number; protected readonly moveBatchQueryLimit: number; protected readonly moveBatchByteLimit: number; protected readonly clearBatchLimit: number; - protected readonly minBucketChanges: number; - protected readonly minChangeRatio: number; - protected readonly maxOpId: bigint; protected readonly buckets: string[] | undefined; protected readonly deleteCheckpointRequestsBefore: Date | undefined; protected readonly signal?: AbortSignal; protected readonly group_id: number; + protected readonly compactChunksOnly: boolean; + protected compactedBucketCount = 0; protected readonly logger: Logger; @@ -110,35 +75,16 @@ export abstract class MongoCompactor { if (this.clearBatchLimit < 2) { throw new ReplicationAssertionError('clearBatchLimit must be >= 2'); } - this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES; - this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO; - this.maxOpId = options.maxOpId ?? 0n; this.buckets = options.compactBuckets; this.deleteCheckpointRequestsBefore = options.deleteCheckpointRequestsBefore; this.signal = options.signal; + this.compactChunksOnly = options.compactChunksOnly ?? false; this.logger = options.logger ?? defaultLogger; } - /** - * Compact buckets by converting operations into MOVE and/or CLEAR operations. - * - * See /docs/storage/compacting-operations.md for details. - */ - async compact() { - await this.deleteOldCheckpointRequests(); - - if (this.buckets) { - for (const bucket of this.buckets) { - // We can make this more efficient later on by iterating through the buckets in a single query. - // That makes batching more tricky, so we leave for later. - await this.compactSingleBucketRetried(bucket); - } - } else { - await this.compactDirtyBuckets(); - } - } + abstract compact(): Promise; - private async deleteOldCheckpointRequests() { + protected async deleteOldCheckpointRequests() { if (this.deleteCheckpointRequestsBefore == null) { return; } @@ -157,177 +103,6 @@ export abstract class MongoCompactor { }); } - /** - * Subset of compact, only populating checksums where relevant. - */ - async populateChecksums(options: { minBucketChanges: number }): Promise { - let count = 0; - // Paginate through dirty buckets in batches until no more buckets meet the criteria. - while (true) { - this.signal?.throwIfAborted(); - const buckets = await this.dirtyBucketBatchForChecksums(options); - if (buckets.length == 0) { - break; - } - this.signal?.throwIfAborted(); - - const start = Date.now(); - // Filter batch by estimated bucket size, to reduce possibility of timeouts. - const checkBuckets: typeof buckets = []; - let totalCountEstimate = 0; - for (const bucket of buckets) { - checkBuckets.push(bucket); - totalCountEstimate += bucket.estimatedCount; - if (totalCountEstimate > 50_000) { - break; - } - } - this.logger.info( - `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}` - ); - await this.updateChecksumsBatch(checkBuckets); - this.logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`); - count += checkBuckets.length; - } - return { buckets: count }; - } - - protected async *dirtyBucketBatchesForCollection( - collection: mongo.Collection, - lastId: TCollectionBucketState['_id'], - maxId: TCollectionBucketState['_id'], - options: { - minBucketChanges: number; - minChangeRatio: number; - }, - getDefinitionId: (state: TCollectionBucketState) => BucketDefinitionId | null - ): AsyncGenerator { - // Paginate through the bucket state collection using cursor-based scanning. - while (true) { - // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, use an aggregation pipeline - // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria. - const [result] = await collection - .aggregate<{ - buckets: TCollectionBucketState[]; - cursor: Pick[]; - }>( - [ - { - $match: { - _id: { $gt: lastId, $lt: maxId } - } - }, - { - $sort: { _id: 1 } - }, - { - // Scan a fixed number of docs each query so sparse matches don't block progress. - $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE - }, - { - $facet: { - buckets: [ - { - $match: { - 'estimate_since_compact.count': { $gte: options.minBucketChanges } - } - }, - { - $project: { - _id: 1, - estimate_since_compact: 1, - compacted_state: 1 - } - } - ], - // This is used for the next query. - cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }] - } - } - ], - { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } - ) - .toArray(); - - const cursor = result?.cursor?.[0]; - if (cursor == null) { - break; - } - lastId = cursor._id; - - const mapped = (result?.buckets ?? []).map((bucketState) => { - // The numbers, specifically the bytes, could be a bigint. Convert to Number to allow calculating ratios. - // BigInt precision is not needed here since this is only an estimate. - const updatedCount = bucketState.estimate_since_compact?.count ?? 0; - const totalCount = (bucketState.compacted_state?.count ?? 0) + updatedCount; - const updatedBytes = Number(bucketState.estimate_since_compact?.bytes ?? 0); - const totalBytes = Number(bucketState.compacted_state?.bytes ?? 0) + updatedBytes; - const dirtyChangeNumber = totalCount > 0 ? updatedCount / totalCount : 0; - const dirtyChangeBytes = totalBytes > 0 ? updatedBytes / totalBytes : 0; - return { - bucket: bucketState._id.b, - definitionId: getDefinitionId(bucketState), - estimatedCount: totalCount, - dirtyRatio: Math.max(dirtyChangeNumber, dirtyChangeBytes) - }; - }); - - yield mapped.filter( - (bucket) => bucket.estimatedCount >= options.minBucketChanges && bucket.dirtyRatio >= options.minChangeRatio - ); - } - } - - protected async dirtyBucketBatchForChecksumsForCollection( - collection: mongo.Collection, - filter: mongo.Filter, - getDefinitionId: (state: mongo.WithId) => BucketDefinitionId | null - ): Promise { - const dirtyBuckets = await collection - .find(filter, { - projection: { - _id: 1, - estimate_since_compact: 1, - compacted_state: 1 - }, - sort: { - 'estimate_since_compact.count': -1 - }, - limit: 200, - maxTimeMS: MONGO_OPERATION_TIMEOUT_MS - }) - .toArray(); - - return dirtyBuckets.map((bucket) => ({ - bucket: bucket._id.b, - definitionId: getDefinitionId(bucket), - estimatedCount: Number(bucket.estimate_since_compact!.count) + Number(bucket.compacted_state?.count ?? 0) - })); - } - - public abstract dirtyBucketBatches(options: { - minBucketChanges: number; - minChangeRatio: number; - }): AsyncGenerator; - - public abstract dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise; - - protected async compactDirtyBuckets() { - for await (const buckets of this.dirtyBucketBatches({ - minBucketChanges: this.minBucketChanges, - minChangeRatio: this.minChangeRatio - })) { - this.signal?.throwIfAborted(); - if (buckets.length == 0) { - continue; - } - - for (const { bucket, definitionId } of buckets) { - await this.compactSingleBucketRetried(bucket, definitionId); - } - } - } - /** * Compaction for a single bucket, with retries on failure. * @@ -336,14 +111,12 @@ export abstract class MongoCompactor { * storage paths are also prepared with lifecycle markers, so a retry can * safely overwrite or eventually clean up uploads from the failed attempt. */ - protected async compactSingleBucketRetried(bucket: string, definitionId: BucketDefinitionId | null = null) { + protected async retryCompaction(bucket: string, compact: () => Promise) { let retryCount = 0; while (true) { this.signal?.throwIfAborted(); - // Do not carry queued bucket state writes from a failed attempt into the rescan. - this.bucketStateUpdates = []; try { - await this.compactSingleBucket(bucket, definitionId); + await compact(); return; } catch (e) { if (this.signal?.aborted) { @@ -372,101 +145,6 @@ export abstract class MongoCompactor { } } } - - protected abstract compactSingleBucket(bucket: string, definitionId?: BucketDefinitionId | null): Promise; - - protected collectBucketStateUpdates( - state: CurrentBucketState, - compactedOpId: InternalOpId - ): mongo.AnyBulkWriteOperation { - if (state.opCount < 0) { - throw new ServiceAssertionError( - `Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}` - ); - } - return { - updateOne: { - filter: this.bucketStateFilter(state.bucket, state.definitionId), - update: { - $set: { - compacted_state: { - op_id: compactedOpId, - count: state.opCount, - checksum: BigInt(state.checksum), - bytes: state.opBytes - }, - estimate_since_compact: { - // There could have been a whole bunch of new operations added to the bucket while compacting, - // which we don't currently cater for. We could potentially query for that, but that adds overhead. - count: 0, - bytes: 0 - } - } - } satisfies mongo.UpdateFilter, - // We generally expect this to have been created before. - // We don't create new ones here, to avoid issues with the unique index on bucket_updates. - upsert: false - } - }; - } - - protected updateBucketChecksums(state: CurrentBucketState, compactedOpId: InternalOpId) { - this.bucketStateUpdates.push(this.collectBucketStateUpdates(state, compactedOpId)); - } - - protected async flushBucketStateUpdates() { - if (this.bucketStateUpdates.length > 0) { - this.logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`); - await this.writeBucketStateUpdates(); - this.bucketStateUpdates = []; - } - } - - protected async updateChecksumsBatch(buckets: Pick[]) { - const checksums = await this.computeChecksumsForBuckets(buckets); - const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId])); - - for (const bucketChecksum of checksums.values()) { - if (isPartialChecksum(bucketChecksum)) { - // Should never happen since we don't specify `start`. - throw new ServiceAssertionError(`Full checksum expected, got ${JSON.stringify(bucketChecksum)}`); - } - - this.bucketStateUpdates.push({ - updateOne: { - filter: this.bucketStateFilter( - bucketChecksum.bucket, - definitionIdByBucket.get(bucketChecksum.bucket) ?? null - ), - update: { - $set: { - compacted_state: { - op_id: this.maxOpId, - count: bucketChecksum.count, - checksum: BigInt(bucketChecksum.checksum), - bytes: null - }, - estimate_since_compact: { - count: 0, - bytes: 0 - } - } - } satisfies mongo.UpdateFilter, - // We don't create new ones here - it gets tricky to get the last_op right with the unique index on - // bucket_updates. - upsert: false - } - }); - } - - await this.flushBucketStateUpdates(); - } - - protected abstract writeBucketStateUpdates(): Promise; - protected abstract computeChecksumsForBuckets( - buckets: Pick[] - ): Promise; - protected abstract bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document; } export interface BucketDataCollectionContext { @@ -475,6 +153,9 @@ export interface BucketDataCollectionContext } function compactionRetryReason(error: unknown): string | null { + if (error instanceof CompactionLeaseLostError) { + return null; + } if (error instanceof ConcurrentCompactionError) { return 'concurrent compaction'; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 9183b7814..2df749088 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -11,14 +11,15 @@ import { BroadcastIterable, CHECKPOINT_INVALIDATE_ALL, CheckpointChanges, + CompactInitialReplicationOptions, + CompactInitialReplicationResults, GetCheckpointChangesOptions, InternalOpId, mergeAsyncIterables, - PopulateChecksumCacheOptions, - PopulateChecksumCacheResults, ReplicationCheckpoint, ReplicationStreamStorageIds, storage, + SyncRuleState, utils, WatchWriteCheckpointOptions } from '@powersync/service-core'; @@ -367,43 +368,29 @@ export abstract class MongoSyncBucketStorage async compact(options?: storage.CompactOptions) { let maxOpId = options?.maxOpId; if (maxOpId == null) { + // For PROCESSING streams, this will be undefined. const checkpoint = await this.getCheckpointInternal(); maxOpId = checkpoint?.checkpoint ?? undefined; } await this.createMongoCompactor({ ...options, maxOpId, logger: this.logger }).compact(); - if (maxOpId != null && options?.compactParameterData) { + if (maxOpId != null && options?.compactParameterData && this.replicationStream.state == SyncRuleState.ACTIVE) { await this.createMongoParameterCompactor(maxOpId, options).compact(); } } + abstract compactInitialReplication( + options: CompactInitialReplicationOptions + ): Promise; + /** * The highest op id persisted for this stream, whether or not covered by a checkpoint. * - * Used as the default `maxOpId` for {@link populatePersistentChecksumCache}, which runs after + * Used as the default `maxOpId` for {@link compactInitialReplication}, which runs after * initial replication but before the first checkpoint exists. */ protected abstract fetchPersistedOpHead(): Promise; - async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise { - this.logger.info(`Populating persistent checksum cache...`); - const start = Date.now(); - const maxOpId = options.maxOpId ?? (await this.fetchPersistedOpHead()) ?? undefined; - const compactor = this.createMongoCompactor({ - ...options, - maxOpId, - memoryLimitMB: 0, - logger: this.logger - }); - - const result = await compactor.populateChecksums({ - minBucketChanges: options.minBucketChanges ?? 10 - }); - const duration = Date.now() - start; - this.logger.info(`Populated persistent checksum cache in ${(duration / 1000).toFixed(1)}s`); - return result; - } - private async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable { if (signal.aborted) { return; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts index 928c04224..527eeda9d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts @@ -5,6 +5,8 @@ import { ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framewo import { storage } from '@powersync/service-core'; import { VersionedPowerSyncMongo } from './db.js'; +const LOCK_DURATION_MS = 60 * 1000; + /** * Manages a lock on a replication stream document, so that only one process * processes that replication stream at a time. @@ -22,15 +24,20 @@ export class MongoSyncRulesLock implements storage.ReplicationLock { ): Promise { const lockId = crypto.randomBytes(8).toString('hex'); const doc = await db.sync_rules.findOneAndUpdate( - { _id: sync_rules.replicationStreamId, $or: [{ lock: null }, { 'lock.expires_at': { $lt: new Date() } }] }, { - $set: { - lock: { - id: lockId, - expires_at: new Date(Date.now() + 60 * 1000) + _id: sync_rules.replicationStreamId, + $or: [{ lock: null }, { $expr: { $lt: ['$lock.expires_at', '$$NOW'] } }] + }, + [ + { + $set: { + lock: { + id: lockId, + expires_at: { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: LOCK_DURATION_MS } } + } } } - }, + ], { projection: { lock: 1 }, returnDocument: 'before', @@ -96,9 +103,15 @@ export class MongoSyncRulesLock implements storage.ReplicationLock { _id: this.sync_rules_id, 'lock.id': this.lock_id }, - { - $set: { 'lock.expires_at': new Date(Date.now() + 60 * 1000) } - }, + [ + { + $set: { + 'lock.expires_at': { + $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: LOCK_DURATION_MS } + } + } + } + ], { returnDocument: 'after' } ); if (result == null) { diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts index 3c2fa9327..d403fe680 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts @@ -234,11 +234,38 @@ export abstract class PersistedBatch { bucket, lastOp: op_id, incrementCount: 1, - incrementBytes: bytes + incrementBytes: bytes, + incrementChunks: 0 }); } } + /** + * V3 persists operations in chunks. Keep this separate from incrementBucket: + * operation counts are known while evaluating rows, while chunk counts and + * exact persisted bytes are only known after the writer has chunked a flush. + */ + protected incrementBucketPersistedChunk(definitionId: BucketDefinitionId, bucket: string, bytes: number) { + const key = `${definitionId ?? ''}:${bucket}`; + const existingState = this.bucketStates.get(key); + if (existingState != null) { + existingState.incrementChunks += 1; + existingState.incrementBytes += bytes; + } + } + + /** + * V3's compactor calculates byte deltas from persisted chunk metadata. + * Replace the writer's per-operation estimate with those exact sizes before + * flushing the corresponding bucket state. + */ + protected resetBucketPersistedBytes(definitionId: BucketDefinitionId, bucket: string) { + const state = this.bucketStates.get(`${definitionId ?? ''}:${bucket}`); + if (state != null) { + state.incrementBytes = 0; + } + } + protected addBucketDataPut(options: { op_id: InternalOpId; bucketKey: BucketKey; @@ -377,4 +404,5 @@ export interface BucketStateUpdate { lastOp: InternalOpId; incrementCount: number; incrementBytes: number; + incrementChunks: number; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index 360424b9b..f5dffb796 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -1,10 +1,18 @@ -import { mongo } from '@powersync/lib-service-mongodb'; -import { ReplicationAssertionError } from '@powersync/lib-services-framework'; -import { addChecksums, storage, utils } from '@powersync/service-core'; +import { mongo, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb'; +import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; +import { + addChecksums, + CompactInitialReplicationResults, + InternalOpId, + isPartialChecksum, + storage, + SyncRuleState, + utils +} from '@powersync/service-core'; import { BucketDefinitionId } from '@powersync/service-sync-rules'; import { BucketDataDoc } from '../common/BucketDataDoc.js'; import { BucketStateDocumentBase, LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js'; -import { CurrentBucketState, DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; +import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; import { cacheKey } from '../OperationBatch.js'; import { BucketDataDocumentV1, BucketStateDocumentV1 } from './models.js'; import type { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js'; @@ -13,12 +21,79 @@ import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; type CompactClearProperties = 'op' | 'checksum' | 'target_op'; +const DEFAULT_MIN_BUCKET_CHANGES = 10; +const DEFAULT_MIN_CHANGE_RATIO = 0.1; +const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000; + +interface CurrentBucketState { + bucket: string; + definitionId: BucketDefinitionId; + seen: Map; + trackingSize: number; + lastNotPut: InternalOpId | null; + opsSincePut: number; + checksum: number; + opCount: number; + opBytes: number; +} + +interface DirtyBucket { + bucket: string; + definitionId: BucketDefinitionId | null; + estimatedCount: number; + dirtyRatio?: number; +} + export class MongoCompactorV1 extends MongoCompactor { // Override types to the more specific ones declare protected readonly db: VersionedPowerSyncMongoV1; declare protected readonly storage: MongoSyncBucketStorageV1; private updates: mongo.AnyBulkWriteOperation[] = []; + private bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; + private readonly minBucketChanges: number; + private readonly minChangeRatio: number; + private readonly maxOpId: bigint; + + constructor( + bucketStorage: MongoSyncBucketStorageV1, + db: VersionedPowerSyncMongoV1, + private options: MongoCompactOptions + ) { + super(bucketStorage, db, options); + this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES; + this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO; + this.maxOpId = options.maxOpId ?? 0n; + } + + /** + * Compact buckets by converting operations into MOVE and/or CLEAR operations. + * + * See /docs/storage/compacting-operations.md for details. + */ + override async compact(): Promise { + if (this.options?.incrementalOnly) { + // Not supported for V1 + this.logger.info('Incremental compacting is not supported on MongoDB storage V1/V2'); + return 0; + } else if (this.storage.replicationStream.state != SyncRuleState.ACTIVE) { + this.logger.info(`Skipping compacting of replication stream in ${this.storage.replicationStream.state} state.`); + return 0; + } + await this.deleteOldCheckpointRequests(); + + if (this.buckets) { + for (const bucket of this.buckets) { + // We can make this more efficient later on by iterating through the buckets in a single query. + // That makes batching more tricky, so we leave for later. + await this.compactSingleBucketRetried(bucket); + } + } else { + await this.compactDirtyBuckets(); + } + + return this.compactedBucketCount; + } public async *dirtyBucketBatches(options: { minBucketChanges: number; @@ -34,8 +109,7 @@ export class MongoCompactorV1 extends MongoCompactor { this.db.bucketStateV1, { g: this.group_id, b: new mongo.MinKey() as any }, { g: this.group_id, b: new mongo.MaxKey() as any }, - options, - () => null + options ); } @@ -45,14 +119,225 @@ export class MongoCompactorV1 extends MongoCompactor { } // Unlike dirtyBucketBatches, this path is resumable after restart because populateChecksums resets // estimate_since_compact as it progresses. - return this.dirtyBucketBatchForChecksumsForCollection( - this.db.bucketStateV1, - { - '_id.g': this.group_id, - 'estimate_since_compact.count': { $gte: options.minBucketChanges } - }, - () => null - ); + return this.dirtyBucketBatchForChecksumsForCollection({ + '_id.g': this.group_id, + 'estimate_since_compact.count': { $gte: options.minBucketChanges } + }); + } + + private async *dirtyBucketBatchesForCollection( + collection: mongo.Collection, + lastId: TBucketState['_id'], + maxId: TBucketState['_id'], + options: { minBucketChanges: number; minChangeRatio: number } + ): AsyncGenerator { + // Paginate through the bucket state collection using cursor-based scanning. + while (true) { + // To avoid timeouts from too many buckets not meeting the minimum-change + // criteria, scan a fixed batch and only return matching buckets. + const [result] = await collection + .aggregate<{ buckets: TBucketState[]; cursor: Pick[] }>( + [ + { $match: { _id: { $gt: lastId, $lt: maxId } } }, + { $sort: { _id: 1 } }, + // Scan a fixed number of docs each query so sparse matches don't block progress. + { $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE }, + { + $facet: { + buckets: [ + { $match: { 'estimate_since_compact.count': { $gte: options.minBucketChanges } } }, + { $project: { _id: 1, estimate_since_compact: 1, compacted_state: 1 } } + ], + cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }] + } + } + ], + { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } + ) + .toArray(); + + const cursor = result?.cursor?.[0]; + if (cursor == null) { + break; + } + lastId = cursor._id; + + const dirtyBuckets = (result?.buckets ?? []).map((bucketState) => { + // BigInt precision is not needed here since this is only an estimate. + const updatedCount = bucketState.estimate_since_compact?.count ?? 0; + const totalCount = (bucketState.compacted_state?.count ?? 0) + updatedCount; + const updatedBytes = Number(bucketState.estimate_since_compact?.bytes ?? 0); + const totalBytes = Number(bucketState.compacted_state?.bytes ?? 0) + updatedBytes; + return { + bucket: bucketState._id.b, + definitionId: null, + estimatedCount: totalCount, + dirtyRatio: Math.max( + totalCount > 0 ? updatedCount / totalCount : 0, + totalBytes > 0 ? updatedBytes / totalBytes : 0 + ) + }; + }); + + yield dirtyBuckets.filter( + (bucket) => bucket.estimatedCount >= options.minBucketChanges && bucket.dirtyRatio >= options.minChangeRatio + ); + } + } + + private async dirtyBucketBatchForChecksumsForCollection( + filter: mongo.Filter + ): Promise { + const dirtyBuckets = await this.db.bucketStateV1 + .find(filter, { + projection: { _id: 1, estimate_since_compact: 1, compacted_state: 1 }, + sort: { 'estimate_since_compact.count': -1 }, + limit: 200, + maxTimeMS: MONGO_OPERATION_TIMEOUT_MS + }) + .toArray(); + + return dirtyBuckets.map((bucket) => ({ + bucket: bucket._id.b, + definitionId: null, + estimatedCount: Number(bucket.estimate_since_compact!.count) + Number(bucket.compacted_state?.count ?? 0) + })); + } + + private async compactSingleBucketRetried(bucket: string, _definitionId: BucketDefinitionId | null = null) { + await this.retryCompaction(bucket, () => this.compactSingleBucket(bucket)); + this.compactedBucketCount++; + } + + private async compactDirtyBuckets() { + for await (const buckets of this.dirtyBucketBatches({ + minBucketChanges: this.minBucketChanges, + minChangeRatio: this.minChangeRatio + })) { + this.signal?.throwIfAborted(); + for (const { bucket, definitionId } of buckets) { + await this.compactSingleBucketRetried(bucket, definitionId); + } + } + } + + /** + * Subset of compact, only populating checksums where relevant. + */ + async populateChecksums(options: { minBucketChanges: number }): Promise { + let count = 0; + // Paginate through dirty buckets in batches until no more buckets meet the criteria. + while (true) { + this.signal?.throwIfAborted(); + const buckets = await this.dirtyBucketBatchForChecksums(options); + if (buckets.length == 0) { + break; + } + this.signal?.throwIfAborted(); + + const start = Date.now(); + // Filter batch by estimated bucket size, to reduce possibility of timeouts. + const checkBuckets: typeof buckets = []; + let totalCountEstimate = 0; + for (const bucket of buckets) { + checkBuckets.push(bucket); + totalCountEstimate += bucket.estimatedCount; + if (totalCountEstimate > 50_000) { + break; + } + } + this.logger.info( + `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}` + ); + await this.updateChecksumsBatch(checkBuckets); + this.logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`); + count += checkBuckets.length; + } + return { buckets: count }; + } + + private collectBucketStateUpdate( + state: CurrentBucketState, + compactedOpId: bigint + ): mongo.AnyBulkWriteOperation { + if (state.opCount < 0) { + throw new ServiceAssertionError( + `Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}` + ); + } + return { + updateOne: { + filter: this.bucketStateFilter(state.bucket), + update: { + $set: { + compacted_state: { + op_id: compactedOpId, + count: state.opCount, + checksum: BigInt(state.checksum), + bytes: state.opBytes + }, + estimate_since_compact: { + // There could have been a whole bunch of new operations added to the bucket while compacting, + // which we don't currently cater for. We could potentially query for that, but that adds overhead. + count: 0, + bytes: 0 + } + } + } satisfies mongo.UpdateFilter, + // We generally expect this to have been created before. + // We don't create new ones here, to avoid issues with the unique index on bucket_updates. + upsert: false + } + }; + } + + private updateBucketChecksums(state: CurrentBucketState, compactedOpId: bigint) { + this.bucketStateUpdates.push(this.collectBucketStateUpdate(state, compactedOpId)); + } + + private async flushBucketStateUpdates() { + if (this.bucketStateUpdates.length > 0) { + this.logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`); + await this.writeBucketStateUpdates(); + this.bucketStateUpdates = []; + } + } + + private async updateChecksumsBatch(buckets: Pick[]) { + const checksums = await this.computeChecksumsForBuckets(buckets); + const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId])); + + for (const bucketChecksum of checksums.values()) { + if (isPartialChecksum(bucketChecksum)) { + // Should never happen since we don't specify `start`. + throw new ServiceAssertionError(`Full checksum expected, got ${JSON.stringify(bucketChecksum)}`); + } + + this.bucketStateUpdates.push({ + updateOne: { + filter: this.bucketStateFilter(bucketChecksum.bucket), + update: { + $set: { + compacted_state: { + op_id: this.maxOpId, + count: bucketChecksum.count, + checksum: BigInt(bucketChecksum.checksum), + bytes: null + }, + estimate_since_compact: { + count: 0, + bytes: 0 + } + } + } satisfies mongo.UpdateFilter, + // We don't create new ones here - it gets tricky to get the last_op right with the unique index on + // bucket_updates. + upsert: false + } + }); + } + + await this.flushBucketStateUpdates(); } protected async writeBucketStateUpdates(): Promise { @@ -74,10 +359,7 @@ export class MongoCompactorV1 extends MongoCompactor { ); } - protected bucketStateFilter( - bucket: string, - _definitionId: BucketDefinitionId | null - ): mongo.Filter { + private bucketStateFilter(bucket: string): mongo.Filter { return { _id: { g: this.group_id, @@ -95,8 +377,9 @@ export class MongoCompactorV1 extends MongoCompactor { } protected async compactSingleBucket(bucket: string) { - // Do not carry queued writes from a failed attempt into the rescan + // Do not carry queued writes from a failed attempt into the rescan. this.updates = []; + this.bucketStateUpdates = []; const idLimitBytes = this.idLimitBytes; const bucketContext = this.getBucketDataContext(bucket); diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 89fec0c47..06e412b2c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -3,6 +3,8 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { ServiceAssertionError } from '@powersync/lib-services-framework'; import { CheckpointChanges, + CompactInitialReplicationOptions, + CompactInitialReplicationResults, deserializeParameterLookup, GetCheckpointChangesOptions, InternalOpId, @@ -29,7 +31,7 @@ import { MongoBucketStorage } from '../../MongoBucketStorage.js'; import { MongoSyncBucketStorageCheckpoint } from '../common/MongoSyncBucketStorageCheckpoint.js'; import { SourceKey } from '../models.js'; import { MongoChecksums } from '../MongoChecksums.js'; -import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; +import { MongoCompactOptions } from '../MongoCompactor.js'; import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js'; import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js'; @@ -186,10 +188,31 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { }); } - createMongoCompactor(options: MongoCompactOptions): MongoCompactor { + createMongoCompactor(options: MongoCompactOptions): MongoCompactorV1 { return new MongoCompactorV1(this, this.db, options); } + async compactInitialReplication( + options: CompactInitialReplicationOptions + ): Promise { + this.logger.info(`Compacting after initial replication...`); + const start = Date.now(); + const maxOpId = options.maxOpId ?? (await this.fetchPersistedOpHead()) ?? undefined; + const compactor = this.createMongoCompactor({ + ...options, + maxOpId, + memoryLimitMB: 0, + logger: this.logger + }); + + const result = await compactor.populateChecksums({ + minBucketChanges: options.minBucketChanges ?? 10 + }); + const duration = Date.now() - start; + this.logger.info(`Compacted after initial replication in ${(duration / 1000).toFixed(1)}s`); + return result; + } + protected createMongoParameterCompactor( checkpoint: InternalOpId, options: storage.CompactOptions diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts new file mode 100644 index 000000000..546bccd39 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/CompactionLease.ts @@ -0,0 +1,152 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { InternalOpId } from '@powersync/service-core'; +import { CompactionLeaseLostError } from '../MongoCompactor.js'; +import { BucketStateDocumentV3 } from './models.js'; + +const LEASE_RENEW_INTERVAL_MS = 60 * 1000; + +/** + * Filters for buckets that have no lease or an expired lease. + */ +export const AVAILABLE_LEASE_EXPR = { + // $$NOW is evaluated by MongoDB, avoiding lease expiry races due + // to clocks on separate compact workers. + $expr: { + $or: [{ $eq: [{ $type: '$compact_lease' }, 'missing'] }, { $lte: ['$compact_lease.expires_at', '$$NOW'] }] + } +}; + +/** + * Owns one V3 bucket-compaction lease, including its server-time renewal and + * the owner-fenced operations which release it. + * + * This is intended as a way to reduce redundant work if multiple jobs attempt to + * compact the same bucket concurrently, but it's not an absolute safety guarantee. + * The individual operations must still be designed to be safe with concurrent compacting. + */ +export class CompactionLease implements AsyncDisposable { + readonly startedAt: Date; + readonly lastOp: InternalOpId; + + private timer: NodeJS.Timeout | undefined; + private renewalInFlight = false; + private renewalError: unknown; + private finalizing = false; + private finished = false; + + private constructor( + private readonly collection: mongo.Collection, + readonly state: BucketStateDocumentV3, + readonly id: mongo.ObjectId, + private readonly durationMs: number + ) { + this.startedAt = new Date(state.compact_lease!.expires_at.getTime() - durationMs); + this.lastOp = state.last_op; + } + + static async claim( + collection: mongo.Collection, + filter: mongo.Filter, + sort: mongo.Sort | undefined, + durationMs: number + ): Promise { + const id = new mongo.ObjectId(); + const state = await collection.findOneAndUpdate( + { + $and: [filter, AVAILABLE_LEASE_EXPR] + }, + [ + { + $set: { + compact_lease: { + id, + expires_at: { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: durationMs } } + } + } + } + ], + { sort, returnDocument: 'after' } + ); + return state == null ? null : new CompactionLease(collection, state, id, durationMs); + } + + startRenewal() { + const interval = Math.max(1, Math.min(LEASE_RENEW_INTERVAL_MS, Math.floor(this.durationMs / 2))); + this.timer = setInterval(() => { + if (this.renewalInFlight) { + return; + } + this.renewalInFlight = true; + void this.renew() + .catch((error) => { + this.renewalError = error; + }) + .finally(() => { + this.renewalInFlight = false; + }); + }, interval); + this.timer.unref(); + } + + async throwIfLost() { + if (this.renewalError != null) { + throw this.renewalError; + } + } + + async reschedule(nextCompactCheck: mongo.Document) { + await this.finish([{ $set: { next_compact_check: nextCompactCheck } }, { $unset: 'compact_lease' }]); + } + + async finalize(update: mongo.Document) { + await this.finish([{ $set: update }, { $unset: 'compact_lease' }]); + } + + async [Symbol.asyncDispose]() { + if (this.timer != null) { + clearInterval(this.timer); + this.timer = undefined; + } + if (this.finished) { + return; + } + this.finalizing = true; + await this.collection.updateOne(this.filter, { $unset: { compact_lease: '' } }); + this.finished = true; + } + + private get filter(): mongo.Filter { + return { _id: this.state._id, 'compact_lease.id': this.id }; + } + + private async renew() { + const result = await this.collection.updateOne(this.filter, [ + { + $set: { + 'compact_lease.expires_at': { + $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.durationMs } + } + } + } + ]); + // A finalization in progress owns the authoritative fenced result. + if (result.matchedCount != 1 && !this.finalizing) { + throw new CompactionLeaseLostError(`Lost compaction lease for bucket ${this.state._id.b}`); + } + // A successful renewal confirms that a transient failure has passed. A + // lease-loss error remains sticky: it means another worker may own it. + if (!(this.renewalError instanceof CompactionLeaseLostError)) { + this.renewalError = undefined; + } + } + + private async finish(update: mongo.Document[]) { + await this.throwIfLost(); + this.finalizing = true; + const result = await this.collection.updateOne(this.filter, update); + if (result.matchedCount != 1) { + throw new CompactionLeaseLostError(`Lost compaction lease for bucket ${this.state._id.b}`); + } + this.finished = true; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index cd4434968..38998b9e6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -1,75 +1,84 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { logger, ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; -import { addChecksums, InternalOpId, storage, utils } from '@powersync/service-core'; +import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; +import { addChecksums, formatBytes, InternalOpId, storage, utils } from '@powersync/service-core'; import { BucketDefinitionId } from '@powersync/service-sync-rules'; import { BucketDataDoc } from '../common/BucketDataDoc.js'; -import { BucketDataKey, BucketStateDocumentBase } from '../models.js'; -import { ConcurrentCompactionError, DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; +import { BucketDataKey } from '../models.js'; +import { ConcurrentCompactionError, MongoCompactor } from '../MongoCompactor.js'; import { cacheKey } from '../OperationBatch.js'; import { loadBucketDataDocument, maxOpId, serializeBucketData } from './bucket-format.js'; import { BucketDataContextV3 } from './BucketDataContextV3.js'; import { DEFAULT_MAX_DOC_SIZE_BYTES } from './chunking.js'; +import { + applyStatsReplacement, + bucketStats, + BucketStatsWithChecksum, + chooseCompactionKind, + combineAdjacentStats, + combineChunkStats, + CompactIntervalConfig, + CompactionContext, + CompactionDecision, + CompactionKind, + CompactionResult, + CompactTargetConfig, + emptyBucketStats, + firstUncompactedWrite, + forcedCompactionKind, + PendingCompactionGroup, + readCompactionBatch, + ScheduledCompactionOptions, + statsForDocument, + statsForDocuments, + unclaimedSnapshotFilter +} from './compact-utils.js'; +import { DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS } from './compaction-constants.js'; +import { AVAILABLE_LEASE_EXPR, CompactionLease } from './CompactionLease.js'; import { BucketDataDocumentV3, BucketStateDocumentV3 } from './models.js'; -import { DefinitionChecksumOperations, MongoChecksumsV3 } from './MongoChecksumsV3.js'; import type { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js'; import { BucketDataObjectStorage, hydrateBucketDataDocuments } from './object-storage/BucketDataObjectStorage.js'; import { ObjectStorageLifecycle, PreparedObjectStorageUpload } from './object-storage/ObjectStorageLifecycle.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; -interface PendingCompactionGroup { - /** - * Input documents are ordered from oldest to newest, matching `ops`. - * Keeping the inputs intact lets unchanged singletons retain their object. - */ - inputs: BucketDataDocumentV3[]; - ops: BucketDataDoc[]; - changed: boolean; - targetOp: InternalOpId | null; -} +const DEFAULT_MIN_COMPACT_FULL_INTERVAL_MS = 2 * 60 * 60 * 1000; +const DEFAULT_MAX_COMPACT_FULL_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000; +const DEFAULT_COMPACT_LEASE_DURATION_MS = 10 * 60 * 1000; +const SCHEDULED_COMPACTION_BATCH_SIZE = 100; -/** - * Read one bounded prefix from a descending compaction cursor. - * - * The document that would cross the byte limit is deliberately not returned: - * pagination resumes below the last returned `_id`, so that document remains - * eligible for the next query. The first document is always accepted to ensure - * progress when a single document exceeds the configured byte limit. - * - * `hasMore` is conservative when the document limit is reached. An extra empty - * query is preferable to exhausting the cursor just to determine whether the - * limited MongoDB query contained another document. - */ -async function readCompactionBatch( - cursor: mongo.AggregationCursor, - options: { byteLimit: number; documentLimit: number } -): Promise<{ documents: BucketDataDocumentV3[]; hasMore: boolean }> { - const documents: BucketDataDocumentV3[] = []; - let cumulativeBytes = 0; - - try { - for await (const document of cursor) { - if (documents.length > 0 && cumulativeBytes + document.size > options.byteLimit) { - return { documents, hasMore: true }; - } +interface CompactionGroupResult { + documentId: BucketDataKey; + stats: BucketStatsWithChecksum; +} - documents.push(document); - cumulativeBytes += document.size; +interface CompactionStatsReplacement { + before: BucketStatsWithChecksum; + after: BucketStatsWithChecksum; +} - if (documents.length >= options.documentLimit) { - return { documents, hasMore: true }; - } - } - return { documents, hasMore: false }; - } finally { - await cursor.close(); - } +interface ClearCompactionResult extends CompactionStatsReplacement { + opCountDiff: number; } -export class MongoCompactorV3 extends MongoCompactor { +export class MongoCompactorV3 extends MongoCompactor implements CompactIntervalConfig, CompactTargetConfig { declare protected readonly db: VersionedPowerSyncMongoV3; declare protected readonly storage: MongoSyncBucketStorageV3; - override async compact(): Promise { + readonly minCompactChunkIntervalMs: number; + readonly minCompactFullIntervalMs: number; + readonly maxCompactFullIntervalMs: number; + readonly compactLeaseDurationMs: number; + readonly maxOpIdCap: InternalOpId | undefined; + + constructor(bucketStorage: MongoSyncBucketStorageV3, db: VersionedPowerSyncMongoV3, options: storage.CompactOptions) { + super(bucketStorage, db, options); + this.minCompactChunkIntervalMs = options.minCompactChunkIntervalMs ?? DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS; + this.minCompactFullIntervalMs = options.minCompactFullIntervalMs ?? DEFAULT_MIN_COMPACT_FULL_INTERVAL_MS; + this.maxCompactFullIntervalMs = options.maxCompactFullIntervalMs ?? DEFAULT_MAX_COMPACT_FULL_INTERVAL_MS; + this.compactLeaseDurationMs = options.compactLeaseDurationMs ?? DEFAULT_COMPACT_LEASE_DURATION_MS; + this.maxOpIdCap = options.maxOpId; + } + + override async compact(): Promise { if (this.storage.objectStorage) { // Clean these before compacting - should be quick in most cases. try { @@ -79,158 +88,619 @@ export class MongoCompactorV3 extends MongoCompactor { this.logger.error(`Failed to clean up object storage deletion markers before compaction`, e); } } - await super.compact(); + await this.deleteOldCheckpointRequests(); + + if (this.buckets != null) { + await this.compactExplicitBuckets(this.buckets); + } else if (this.compactChunksOnly) { + // Writers defer their first chunk-compaction check by this fixed default. + // Include that interval so this synchronous initial-replication pass + // processes the work that existed when it started. + await this.compactScheduledBuckets({ + dueAheadMs: DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS, + forceKind: CompactionKind.Chunks + }); + } else { + await this.compactScheduledBuckets(); + } if (this.storage.objectStorage) { // Cleanup for any produced during compacting. // Note that markers only expire after a delay, so this may skip many produced during this compact // run. However, during long compact runs, this may also have many ones it can clean up. await this.objectStorageLifecycle.cleanup(this.logger); } + return this.compactedBucketCount; } - private get objectStorageLifecycle(): ObjectStorageLifecycle { - if (!this.storage.objectStorage) { - throw new Error('Object storage is not configured'); + /** An explicit compact request always runs a full compact for its buckets. */ + private async compactExplicitBuckets(buckets: string[]) { + for (const bucket of buckets) { + // This is not a super efficient query, but this is not a common use case. + // May be optimized later. + const states = await this.db + .bucketState(this.group_id) + .find({ '_id.b': bucket }, { projection: { _id: 1 } }) + .toArray(); + for (const state of states) { + await using lease = await this.claimBucket({ _id: state._id }); + if (lease == null || lease.state.first_uncompacted_write == null) { + continue; + } + if (this.isCompactionTargetCovered(lease.state, CompactionKind.Full)) { + continue; + } + const decision = chooseCompactionKind(lease.state, lease.startedAt, this); + await this.compactClaimedBucket(lease, CompactionKind.Full, decision); + } } - return new ObjectStorageLifecycle(this.db, this.group_id, this.storage.objectStorage); } - public async *dirtyBucketBatches(options: { - minBucketChanges: number; - minChangeRatio: number; - }): AsyncGenerator { - if (options.minBucketChanges <= 0) { - throw new ReplicationAssertionError('minBucketChanges must be >= 1'); + /** + * Process scheduled work in bounded batches. + * + * Batching specifically help to cover cases of many buckets where no compaction is required: + * Instead of sequentially claiming and then rescheduling a bucket, this handles it in bulk. + * + * Buckets that do need compaction are still claimed and processed sequentially. + * + * Any concurrent workers may read the same batch. Rescheduling filters out buckets handled + * by a concurrent worker or replication write, while buckets that do need compaction are + * filtered out when claiming a compaction lease. + * + * We filter scheduled jobs by the job start date, so that the same bucket is not compacted + * multiple times in one run. Reschedules fall beyond the fixed boundary. For the run after + * initial replication, dueAheadMs extends that boundary to include the first deferred interval. + */ + private async compactScheduledBuckets(options: ScheduledCompactionOptions = {}) { + // Writers derive next_compact_check from MongoDB's $$NOW. Use the same + // clock for the fixed job boundary so clock skew cannot exclude work at + // the exact initial-replication interval. + const [{ now: jobStartedAt }] = await this.db.db + .aggregate<{ now: Date }>([{ $documents: [{}] }, { $project: { _id: 0, now: '$$NOW' } }]) + .toArray(); + const dueBefore = new Date(jobStartedAt.getTime() + (options.dueAheadMs ?? 0)); + const forceKind = options.forceKind; + const rescheduleNotBefore = new Date(dueBefore.getTime() + 1); + while (true) { + this.signal?.throwIfAborted(); + const states = await this.findScheduledBucketBatch(dueBefore); + if (states.length == 0) { + break; + } + + const scheduled: { + state: BucketStateDocumentV3; + decision: CompactionDecision; + forcedKind: CompactionKind | null; + }[] = []; + for (const state of states) { + try { + scheduled.push({ + state, + decision: chooseCompactionKind(state, jobStartedAt, this), + forcedKind: forcedCompactionKind(state, forceKind, this) + }); + } catch (error) { + await this.rescheduleFailedBucket(state, rescheduleNotBefore, error); + } + } + const noOpStates = scheduled.filter( + ({ state, decision, forcedKind }) => + state.compact_lease == null && (forceKind == null ? decision.kind : forcedKind) == null + ); + await this.rescheduleUnclaimedBuckets(noOpStates, rescheduleNotBefore); + + for (const { state, decision, forcedKind } of scheduled) { + const kind = forceKind == null ? decision.kind : forcedKind; + if (state.compact_lease == null && kind == null) { + continue; + } + + try { + await using lease = await this.claimBucket({ _id: state._id, next_compact_check: { $lte: dueBefore } }); + if (lease == null) { + continue; + } + const claimedDecision = chooseCompactionKind(lease.state, lease.startedAt, this); + const claimedKind = + forceKind == null ? claimedDecision.kind : forcedCompactionKind(lease.state, forceKind, this); + if (claimedKind == null) { + await this.rescheduleClaimedBucket(lease, claimedDecision, rescheduleNotBefore); + } else if (this.isCompactionTargetCovered(lease.state, claimedKind)) { + // The run cannot advance this kind's watermark without regressing + // already-published progress. Keep any newer work scheduled. + await this.rescheduleClaimedBucket(lease, claimedDecision, rescheduleNotBefore); + } else { + await this.compactClaimedBucket(lease, claimedKind, claimedDecision, rescheduleNotBefore); + } + } catch (error) { + if (this.signal?.aborted) { + // When aborted, stop completely, rather than logging and re-scheduling individual buckets. + // The lease on the current bucket is still released automatically. + throw error; + } + await this.rescheduleFailedBucket(state, rescheduleNotBefore, error); + } + } } - const collection = this.db.bucketState(this.group_id) as unknown as mongo.Collection; - yield* this.dirtyBucketBatchesForCollection( - collection, - { d: new mongo.MinKey(), b: new mongo.MinKey() } as unknown as BucketStateDocumentV3['_id'], - { d: new mongo.MaxKey(), b: new mongo.MaxKey() } as unknown as BucketStateDocumentV3['_id'], - options, - (bucketState) => (bucketState as BucketStateDocumentV3)._id.d - ); } - public async dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise { - if (options.minBucketChanges <= 0) { - throw new ReplicationAssertionError('minBucketChanges must be >= 1'); + /** Read a bounded, priority-ordered snapshot of currently claimable scheduled work. */ + private async findScheduledBucketBatch(dueBefore: Date): Promise { + return this.db + .bucketState(this.group_id) + .find({ + next_compact_check: { $lte: dueBefore }, + ...AVAILABLE_LEASE_EXPR + }) + .sort({ next_compact_check: 1 }) + .limit(SCHEDULED_COMPACTION_BATCH_SIZE) + .toArray(); + } + + /** + * Reschedule snapshots that were already known to be no-ops without first + * taking a lease. Every decision input is compared so a concurrent writer + * or compactor simply makes the update a no-op instead of losing work. A + * successful reschedule moves beyond this run's fixed selection boundary. + */ + private async rescheduleUnclaimedBuckets( + states: { state: BucketStateDocumentV3; decision: CompactionDecision }[], + notBefore: Date + ) { + if (states.length == 0) { + return; } - return this.dirtyBucketBatchForChecksumsForCollection( - this.db.bucketState(this.group_id) as unknown as mongo.Collection, - { - 'estimate_since_compact.count': { $gte: options.minBucketChanges } - } as unknown as mongo.Filter, - (bucketState) => (bucketState as BucketStateDocumentV3)._id.d + await this.db.bucketState(this.group_id).bulkWrite( + states.map(({ state, decision }) => ({ + updateOne: { + filter: unclaimedSnapshotFilter(state), + update: [{ $set: { next_compact_check: this.rescheduleAtOrAfter(decision.nextCompactCheck, notBefore) } }] + } + })), + { ordered: false } ); } - protected async writeBucketStateUpdates(): Promise { - await this.db - .bucketState(this.group_id) - .bulkWrite(this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], { - ordered: false - }); + /** + * Isolate a malformed bucket so it cannot prevent other scheduled buckets + * from compacting. The snapshot filter preserves any concurrent write or + * compactor result instead of overwriting its next check. + */ + private async rescheduleFailedBucket(state: BucketStateDocumentV3, notBefore: Date, error: unknown) { + this.logger.error(`Failed to compact scheduled bucket ${state._id.b}; rescheduling it`, error); + try { + await this.db + .bucketState(this.group_id) + .updateOne(unclaimedSnapshotFilter(state), [{ $set: { next_compact_check: notBefore } }]); + } catch (rescheduleError) { + this.logger.error(`Failed to reschedule bucket ${state._id.b} after a compaction error`, rescheduleError); + } } /** - * The compactor operates on persisted definition ids only - never on parsed sources. - * This narrowed view makes the source-resolving checksum methods unreachable here. + * Given a bucket filter, claim a lease on the bucket. The filter should include a filter on _id. + * + * Resolves to null if the bucket is already claimed, not found, or filtered out. */ - private get definitionChecksums(): DefinitionChecksumOperations { - return this.storage.checksums as MongoChecksumsV3; + private async claimBucket( + filter: mongo.Filter, + sort?: mongo.Sort + ): Promise { + return CompactionLease.claim(this.db.bucketState(this.group_id), filter, sort, this.compactLeaseDurationMs); } - protected async computeChecksumsForBuckets( - buckets: Pick[] - ): Promise { - return this.definitionChecksums.computePartialChecksumsDirectByDefinition( - buckets.map(({ bucket, definitionId }) => { - if (definitionId == null) { - throw new ServiceAssertionError(`Missing definitionId for bucket checksum update on bucket ${bucket}`); - } - return { - bucket, - definitionId, - end: this.maxOpId - }; - }) + private async compactClaimedBucket( + lease: CompactionLease, + kind: CompactionKind, + decision: CompactionDecision, + rescheduleNotBefore?: Date + ) { + const context = new CompactionContext( + lease, + kind, + decision, + rescheduleNotBefore, + this.compactionTarget(lease.state) ); + lease.startRenewal(); + await this.compactSingleBucket(context); } - protected bucketStateFilter( - bucket: string, - definitionId: BucketDefinitionId | null - ): mongo.Filter { - if (definitionId == null) { - throw new ServiceAssertionError(`Missing definitionId for V3 bucket state filter on bucket ${bucket}`); + private async rescheduleClaimedBucket(lease: CompactionLease, decision: CompactionDecision, notBefore?: Date) { + await lease.reschedule(this.rescheduleAtOrAfter(decision.nextCompactCheck, notBefore)); + } + + private rescheduleAtOrAfter(nextCompactCheck: mongo.Document, notBefore: Date | undefined): mongo.Document { + return notBefore == null ? nextCompactCheck : { $max: [nextCompactCheck, notBefore] }; + } + + private compactionTarget(state: BucketStateDocumentV3): InternalOpId { + return this.maxOpIdCap == null || state.last_op < this.maxOpIdCap ? state.last_op : this.maxOpIdCap; + } + + private isCompactionTargetCovered(state: BucketStateDocumentV3, kind: CompactionKind): boolean { + const target = this.compactionTarget(state); + if (kind == CompactionKind.Chunks) { + return state.compacted_state != null && state.compacted_state.op_id >= target; } - return { - _id: { - d: definitionId, - b: bucket + if (state.last_full_compact != null && state.last_full_compact.op_id >= target) { + return true; + } + // A full compact may change counts before the checksum-cache boundary. + // Wait for the safe target to catch up instead of publishing an older or + // stale cache. At the same boundary, full coverage can still advance. + return state.compacted_state != null && state.compacted_state.op_id > target; + } + + private get objectStorageLifecycle(): ObjectStorageLifecycle { + if (!this.storage.objectStorage) { + throw new Error('Object storage is not configured'); + } + return new ObjectStorageLifecycle(this.db, this.group_id, this.storage.objectStorage); + } + + private async compactSingleBucket(context: CompactionContext) { + if (context.kind == CompactionKind.Chunks) { + return this.compactSingleBucketChunks(context); + } + + return this.compactSingleBucketFully(context); + } + + /** + * Merge adjacent bucket-data chunks without inspecting their operations + * unless a merge is possible. The metadata contains enough information to + * update the persisted checksum state and to decide whether a group can fit + * in one chunk. + */ + private async compactSingleBucketChunks(context: CompactionContext) { + const bucket = context.state._id.b; + const resolvedDefinitionId = context.state._id.d; + const bucketContext = new BucketDataContextV3(this.db, { + bucket, + definitionId: resolvedDefinitionId, + replicationStreamId: this.group_id + }); + const collection = this.db.bucketData(this.group_id, resolvedDefinitionId); + const dataContext = { replicationStreamId: this.group_id, definitionId: resolvedDefinitionId }; + let previousCompactedState = context.state.compacted_state; + // A zero boundary represents an empty prefix, so there is no stored chunk + // whose statistics need to be carried into this pass. + if (previousCompactedState?.op_id === 0n) { + previousCompactedState = undefined; + } + // Include the last previously compacted chunk as well as new chunks. It + // is the only old chunk which can become mergeable with the new tail. + let lowerBound = + previousCompactedState != null ? bucketContext.docId(previousCompactedState.op_id - 1n) : bucketContext.minId; + const upperBound = bucketContext.docId(context.targetOp + 1n); + let cachedBoundaryToVerify = previousCompactedState?.op_id; + + let compactedOpId: bigint | null = null; + let overlappingCompactedChunk: BucketStatsWithChecksum | undefined; + let compactedTail = emptyBucketStats(); + let pendingChunks: BucketDataDocumentV3[] = []; + let pendingSize = 0; + + while (true) { + this.signal?.throwIfAborted(); + await context.lease.throwIfLost(); + + const batch = await readCompactionBatch( + collection.aggregate( + [ + { + $match: { + _id: { + $gt: lowerBound, + $lt: upperBound + } + } + }, + { $sort: { _id: 1 } }, + { $limit: this.moveBatchQueryLimit }, + { + $project: { + _id: 1, + min_op: 1, + checksum: 1, + count: 1, + size: 1, + target_op: 1, + storage_ref: 1 + } + } + ], + { batchSize: this.moveBatchQueryLimit + 1 } + ), + { + byteLimit: this.moveBatchByteLimit, + documentLimit: this.moveBatchQueryLimit + } + ); + + if (cachedBoundaryToVerify != null) { + const cachedBoundary = cachedBoundaryToVerify; + cachedBoundaryToVerify = undefined; + if (batch.documents[0]?._id.o !== cachedBoundary) { + // A previous attempt may have replaced the cached boundary before + // finalizing bucket state. Keep the persisted cache available to + // readers, but ignore it in this attempt and calculate its + // replacement through the normal scan from the bucket beginning. + previousCompactedState = undefined; + lowerBound = bucketContext.minId; + continue; + } + } + + if (batch.documents.length == 0) { + break; + } + + for (const doc of batch.documents) { + compactedOpId = maxOpId(compactedOpId, doc._id.o); + const documentStats = statsForDocument(doc); + if (previousCompactedState?.op_id === doc._id.o) { + overlappingCompactedChunk = documentStats; + } + + const nextSize = pendingSize + doc.size; + if (pendingChunks.length > 0 && nextSize > DEFAULT_MAX_DOC_SIZE_BYTES) { + const groupStats = await this.flushChunkMerge(bucket, pendingChunks, collection, dataContext, bucketContext); + compactedTail = combineAdjacentStats(compactedTail, groupStats); + pendingChunks = []; + pendingSize = 0; + } + + pendingChunks.push(doc); + pendingSize += doc.size; + } + + lowerBound = batch.documents[batch.documents.length - 1]._id; + if (!batch.hasMore) { + break; } + } + + if (pendingChunks.length > 0) { + const groupStats = await this.flushChunkMerge(bucket, pendingChunks, collection, dataContext, bucketContext); + compactedTail = combineAdjacentStats(compactedTail, groupStats); + } + + if (compactedOpId == null) { + await this.finalizeSkippedBucket(context); + return; + } + + const compactedState = + previousCompactedState == null + ? compactedTail + : combineChunkStats(previousCompactedState, compactedTail, overlappingCompactedChunk!); + const tailStats = + compactedOpId == context.lastOp + ? undefined + : await this.readBucketStats(bucket, resolvedDefinitionId, context.lastOp, bucketContext.docId(compactedOpId)); + const result: CompactionResult = { + compactedState, + bucketStats: tailStats == null ? compactedState : combineAdjacentStats(compactedState, tailStats) }; + + await this.finalizeCompactedBucket({ context, compactedOpId, compactionResult: result, puts: 0 }); + this.compactedBucketCount++; + this.logger.info( + `Compacted bucket chunks ${bucket}: ${result.bucketStats.count} ops, ${result.bucketStats.chunks} chunks, ${formatBytes(result.bucketStats.bytes)}` + ); } - private async getBucketDataContext( + private async flushChunkMerge( bucket: string, - definitionId: BucketDefinitionId | null - ): Promise { - let resolvedDefinitionId = definitionId; - - if (resolvedDefinitionId == null) { - const allDefinitionIds = this.storage.storageIds.bucketDefinitionIds; - if (allDefinitionIds.length > 0) { - const potentialIds = allDefinitionIds.map((id) => ({ d: id, b: bucket })); - const bucketState = await this.db.bucketState(this.group_id).findOne({ - _id: { $in: potentialIds } - }); - if (bucketState != null) { - resolvedDefinitionId = bucketState._id.d; - } + inputs: BucketDataDocumentV3[], + collection: mongo.Collection, + context: { replicationStreamId: number; definitionId: string }, + bucketContext: BucketDataContextV3 + ): Promise { + if (inputs.length == 1) { + return statsForDocument(inputs[0]); + } + + // The metadata scan deliberately excluded ops. Read inline payloads only + // for this merge group; object-storage payloads are fetched below using + // the same rule. + const inlineInputs = inputs.filter((input) => input.storage_ref == null); + if (inlineInputs.length > 0) { + const inlineDocuments = await collection + .find({ _id: { $in: inlineInputs.map((input) => input._id) } }, { projection: { _id: 1, ops: 1 } }) + .toArray(); + const opsById = new Map(inlineDocuments.map((document) => [document._id.o.toString(), document.ops])); + for (const input of inlineInputs) { + input.ops = opsById.get(input._id.o.toString()); } } + await hydrateBucketDataDocuments(inputs, this.storage.objectStorage, { signal: this.signal }); + + const operations = inputs.flatMap((input) => Array.from(loadBucketDataDocument(context, input))); + const targetOp = inputs.reduce( + (maxTarget, input) => maxOpId(maxTarget, input.target_op), + null + ); + const result = await this.flushCompactionGroup( + bucket, + { + inputs, + ops: operations, + changed: true, + targetOp + }, + bucketContext, + context + ); + return result.stats; + } - if (resolvedDefinitionId == null) { - return null; + private async finalizeCompactedBucket({ + context, + compactedOpId, + compactionResult, + puts + }: { + context: CompactionContext; + compactedOpId: InternalOpId; + compactionResult: CompactionResult; + puts: number; + }) { + await context.lease.throwIfLost(); + const startedStats = bucketStats(context.state); + const delta = { + count: compactionResult.bucketStats.count - startedStats.count, + bytes: compactionResult.bucketStats.bytes - startedStats.bytes, + chunks: compactionResult.bucketStats.chunks - startedStats.chunks + }; + const coveredClaimedHead = compactedOpId >= context.lastOp; + const concurrentWriteCheck = { $gt: ['$last_op', context.lastOp] }; + const remainingFullWorkCheck = coveredClaimedHead ? concurrentWriteCheck : true; + const nextAfterPartialFullCompact = this.rescheduleAtOrAfter( + { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.minCompactChunkIntervalMs } }, + context.rescheduleNotBefore + ); + const nextCheckForUncompactedWork = this.rescheduleAtOrAfter( + { + $min: [ + new Date(firstUncompactedWrite(context.state).getTime() + this.maxCompactFullIntervalMs), + { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.minCompactChunkIntervalMs } } + ] + }, + context.rescheduleNotBefore + ); + const update: mongo.Document = { + compacted_state: { + op_id: compactedOpId, + checksum: BigInt(compactionResult.compactedState.checksum), + count: compactionResult.compactedState.count, + bytes: compactionResult.compactedState.bytes, + chunks: compactionResult.compactedState.chunks, + at: '$$NOW' + }, + bucket_stats: { + count: { $add: ['$bucket_stats.count', delta.count] }, + bytes: { $add: ['$bucket_stats.bytes', delta.bytes] }, + chunks: { $add: ['$bucket_stats.chunks', delta.chunks] } + }, + first_uncompacted_write: + context.kind == CompactionKind.Full + ? { $cond: [remainingFullWorkCheck, '$$NOW', '$$REMOVE'] } + : '$first_uncompacted_write', + next_compact_check: + context.kind == CompactionKind.Full + ? { $cond: [remainingFullWorkCheck, nextAfterPartialFullCompact, '$$REMOVE'] } + : nextCheckForUncompactedWork + }; + if (context.kind == CompactionKind.Full) { + update.last_full_compact = { + op_id: compactedOpId, + count: compactionResult.compactedState.count, + puts, + at: '$$NOW' + }; } - return new BucketDataContextV3(this.db, { + await context.lease.finalize(update); + } + + private async finalizeSkippedBucket(context: CompactionContext) { + // A maxOpId cap can exclude the first remaining document entirely. Avoid + // immediately claiming the same no-progress bucket again in this run. + await this.rescheduleClaimedBucket( + context.lease, + { + ...context.decision, + nextCompactCheck: { + $max: [ + context.decision.nextCompactCheck, + { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.minCompactChunkIntervalMs } } + ] + } + }, + context.rescheduleNotBefore + ); + } + + /** + * Read bucket stats directly from bucket_data documents. + */ + private async readBucketStats( + bucket: string, + definitionId: BucketDefinitionId, + maxOp: InternalOpId, + lowerBound?: BucketDataKey + ): Promise { + const context = new BucketDataContextV3(this.db, { bucket, - definitionId: resolvedDefinitionId, + definitionId, replicationStreamId: this.group_id }); + const [stats] = await this.db + .bucketData(this.group_id, definitionId) + .aggregate<{ count: number; bytes: number | bigint; chunks: number; checksum: bigint }>([ + { + $match: { + _id: + lowerBound == null + ? { $gte: context.minId, $lte: context.docId(maxOp) } + : { $gt: lowerBound, $lte: context.docId(maxOp) } + } + }, + { + $group: { + _id: null, + count: { $sum: '$count' }, + bytes: { $sum: '$size' }, + chunks: { $sum: 1 }, + checksum: { $sum: '$checksum' } + } + } + ]) + .toArray(); + return { + count: Number(stats?.count ?? 0), + bytes: BigInt(stats?.bytes ?? 0), + chunks: Number(stats?.chunks ?? 0), + checksum: + typeof stats?.checksum == 'bigint' + ? Number(BigInt.asIntN(32, stats.checksum)) + : addChecksums(0, Number(stats?.checksum ?? 0)) + }; } - protected override async compactSingleBucket(bucket: string, definitionId: BucketDefinitionId | null = null) { - const bucketContext = await this.getBucketDataContext(bucket, definitionId); - if (bucketContext == null) { - return; - } - - const resolvedDefinitionId = bucketContext.key.definitionId; + private async compactSingleBucketFully(context: CompactionContext) { + const bucket = context.state._id.b; + const resolvedDefinitionId = context.state._id.d; + const bucketContext = new BucketDataContextV3(this.db, { + bucket, + definitionId: resolvedDefinitionId, + replicationStreamId: this.group_id + }); const collection = this.db.bucketData(this.group_id, resolvedDefinitionId); - const context = { replicationStreamId: this.group_id, definitionId: resolvedDefinitionId }; - + const dataContext = { replicationStreamId: this.group_id, definitionId: resolvedDefinitionId }; const lowerBound = bucketContext.minId; - let upperBound = bucketContext.docId(this.maxOpId + 1n); + let upperBound = bucketContext.docId(context.targetOp + 1n); - let totalChecksum = 0; let totalOpCount = 0; - let totalOpBytes = 0; let lastNotPut: bigint | null = null; let opsSincePut = 0; let compactedOpId: bigint | null = null; let clearBoundary: { opId: bigint; documentId: BucketDataKey } | null = null; + let compactedStats = emptyBucketStats(); const seen = new Map(); let trackingSize = 0; + let putCount = 0; let pendingGroup: PendingCompactionGroup | null = null; // --- Read batch from MongoDB --- while (true) { this.signal?.throwIfAborted(); + await context.lease.throwIfLost(); const pipeline: mongo.Document[] = [ { @@ -280,7 +750,7 @@ export class MongoCompactorV3 extends MongoCompactor { // merging is useful, and writes each final object at most once. for (const doc of batchDocs) { compactedOpId ??= doc._id.o; - const originalOps = Array.from(loadBucketDataDocument(context, doc)); + const originalOps = Array.from(loadBucketDataDocument(dataContext, doc)); let changed = false; const compactedOps: BucketDataDoc[] = []; @@ -313,6 +783,7 @@ export class MongoCompactorV3 extends MongoCompactor { } compactedOps.push(op); if (op.op == 'PUT') { + putCount++; lastNotPut = null; opsSincePut = 0; } else { @@ -334,10 +805,6 @@ export class MongoCompactorV3 extends MongoCompactor { } compactedOps.reverse(); - for (const op of compactedOps) { - totalChecksum = addChecksums(totalChecksum, Number(op.checksum)); - totalOpBytes += op.data?.length ?? 0; - } totalOpCount += compactedOps.length; const candidate: PendingCompactionGroup = { @@ -361,13 +828,14 @@ export class MongoCompactorV3 extends MongoCompactor { }; } else { const flushedGroup = pendingGroup; - const documentId = await this.flushCompactionGroup(bucket, flushedGroup, bucketContext, context); + const result = await this.flushCompactionGroup(bucket, flushedGroup, bucketContext, dataContext); + compactedStats = combineAdjacentStats(compactedStats, result.stats); if ( lastNotPut != null && flushedGroup.ops[0].o <= lastNotPut && flushedGroup.ops[flushedGroup.ops.length - 1].o >= lastNotPut ) { - clearBoundary = { opId: lastNotPut, documentId }; + clearBoundary = { opId: lastNotPut, documentId: result.documentId }; } pendingGroup = candidate; } @@ -385,16 +853,18 @@ export class MongoCompactorV3 extends MongoCompactor { } if (pendingGroup != null) { - const documentId = await this.flushCompactionGroup(bucket, pendingGroup, bucketContext, context); + const result = await this.flushCompactionGroup(bucket, pendingGroup, bucketContext, dataContext); + compactedStats = combineAdjacentStats(compactedStats, result.stats); if ( lastNotPut != null && pendingGroup.ops[0].o <= lastNotPut && pendingGroup.ops[pendingGroup.ops.length - 1].o >= lastNotPut ) { - clearBoundary = { opId: lastNotPut, documentId }; + clearBoundary = { opId: lastNotPut, documentId: result.documentId }; } } if (compactedOpId == null) { + await this.finalizeSkippedBucket(context); return; } @@ -404,36 +874,33 @@ export class MongoCompactorV3 extends MongoCompactor { throw new ReplicationAssertionError(`Missing CLEAR boundary document for bucket ${bucket}`); } - totalOpCount += await this.clearBucketLeading( + const clearResult = await this.clearBucketLeading( lastNotPut, clearBoundary.documentId, bucketContext, collection, - context + dataContext ); + totalOpCount += clearResult.opCountDiff; + compactedStats = applyStatsReplacement(compactedStats, clearResult.before, clearResult.after); } + const tailStats = + compactedOpId == context.lastOp + ? undefined + : await this.readBucketStats(bucket, resolvedDefinitionId, context.lastOp, bucketContext.docId(compactedOpId)); + const result: CompactionResult = { + compactedState: compactedStats, + bucketStats: tailStats == null ? compactedStats : combineAdjacentStats(compactedStats, tailStats) + }; + // --- Finalize: update bucket checksums and state --- - this.updateBucketChecksums( - { - bucket, - definitionId: resolvedDefinitionId, - seen: new Map(), - trackingSize: 0, - lastNotPut: lastNotPut, - opsSincePut: opsSincePut, - checksum: totalChecksum, - opCount: totalOpCount, - opBytes: totalOpBytes - }, - compactedOpId - ); - if (this.bucketStateUpdates.length > 0) { - await this.writeBucketStateUpdates(); - this.bucketStateUpdates = []; - } + await this.finalizeCompactedBucket({ context, compactedOpId, compactionResult: result, puts: putCount }); - logger.info(`Compacted bucket ${bucket}: ${totalOpCount} surviving ops`); + this.compactedBucketCount++; + this.logger.info( + `Compacted bucket ${bucket}: ${totalOpCount} ops, ${result.bucketStats.chunks} chunks, ${formatBytes(result.bucketStats.bytes)}` + ); } /** @@ -447,9 +914,12 @@ export class MongoCompactorV3 extends MongoCompactor { group: PendingCompactionGroup, bucketContext: BucketDataContextV3, context: { replicationStreamId: number; definitionId: string } - ): Promise { + ): Promise { if (group.inputs.length == 1 && !group.changed) { - return group.inputs[0]._id; + return { + documentId: group.inputs[0]._id, + stats: statsForDocument(group.inputs[0]) + }; } const inputs = group.inputs; @@ -507,7 +977,10 @@ export class MongoCompactorV3 extends MongoCompactor { } finally { await session.endSession(); } - return documents[0]._id; + return { + documentId: documents[0]._id, + stats: statsForDocuments(documents) + }; } /** @@ -516,7 +989,8 @@ export class MongoCompactorV3 extends MongoCompactor { * before the known boundary document, then splits that boundary document * if it contains ops on both sides of lastNotPut. * - * Returns the op count diff after replacing cleared ops with CLEAR ops. + * Returns the op count and stored-stat changes after replacing cleared ops + * with CLEAR ops. */ private async clearBucketLeading( lastNotPut: bigint, @@ -524,8 +998,10 @@ export class MongoCompactorV3 extends MongoCompactor { bucketContext: BucketDataContextV3, collection: mongo.Collection, context: { replicationStreamId: number; definitionId: string } - ): Promise { + ): Promise { let opCountDiff = 0; + let before = emptyBucketStats(); + let after = emptyBucketStats(); const session = this.db.client.startSession(); try { let done = false; @@ -542,11 +1018,13 @@ export class MongoCompactorV3 extends MongoCompactor { ); done = batch.done; opCountDiff += batch.opCountDiff; + before = combineAdjacentStats(before, batch.before); + after = combineAdjacentStats(after, batch.after); } // The final step is to process the "boundary" document: It may contain some CLEAR/MOVE/REMOVE operations, // potentially followed by PUT operations. This is only a single document, so no need for batching. - opCountDiff += await this.clearBoundaryDocument( + const boundaryResult = await this.clearBoundaryDocument( session, lastNotPut, boundaryDocId, @@ -554,11 +1032,14 @@ export class MongoCompactorV3 extends MongoCompactor { collection, context ); + opCountDiff += boundaryResult.opCountDiff; + before = combineAdjacentStats(before, boundaryResult.before); + after = combineAdjacentStats(after, boundaryResult.after); } finally { await session.endSession(); } - return opCountDiff; + return { opCountDiff, before, after }; } private async clearLeadingFullDocuments( @@ -568,17 +1049,21 @@ export class MongoCompactorV3 extends MongoCompactor { bucketContext: BucketDataContextV3, collection: mongo.Collection, context: { replicationStreamId: number; definitionId: string } - ): Promise<{ done: boolean; opCountDiff: number }> { + ): Promise<{ done: boolean; opCountDiff: number } & CompactionStatsReplacement> { const bucket = bucketContext.key.bucket; this.signal?.throwIfAborted(); - const prepared = await this.prepareCompactionUploads(bucket, context, [lastNotPut]); + let prepared: PreparedObjectStorageUpload[] | undefined; let done = false; let opCountDiff = 0; + let before = emptyBucketStats(); + let after = emptyBucketStats(); await session.withTransaction( async () => { done = false; opCountDiff = 0; + before = emptyBucketStats(); + after = emptyBucketStats(); const oldStoragePaths: string[] = []; const query = collection.find( { @@ -595,6 +1080,7 @@ export class MongoCompactorV3 extends MongoCompactor { min_op: 1, checksum: 1, count: 1, + size: 1, target_op: 1, has_clear_op: 1, storage_ref: 1 @@ -609,6 +1095,7 @@ export class MongoCompactorV3 extends MongoCompactor { let lastDocId: BucketDataKey | null = null; let clearOpCount = 0; let gotNonClearOp = false; + const inputStats = emptyBucketStats(); for await (const doc of query.stream()) { if (doc.min_op > lastNotPut) { @@ -618,6 +1105,11 @@ export class MongoCompactorV3 extends MongoCompactor { } lastDocId = doc._id; + const documentStats = statsForDocument(doc); + inputStats.count += documentStats.count; + inputStats.bytes += documentStats.bytes; + inputStats.chunks += documentStats.chunks; + inputStats.checksum = addChecksums(inputStats.checksum, documentStats.checksum); if (doc.storage_ref) { oldStoragePaths.push(doc.storage_ref.path); } @@ -646,6 +1138,7 @@ export class MongoCompactorV3 extends MongoCompactor { return; } + prepared ??= await this.prepareCompactionUploads(bucket, context, [lastNotPut]); this.logger.info(`Flushing CLEAR for ${clearedOpCount} ops at ${lastDocId?.o}`); await collection.deleteMany( { @@ -671,6 +1164,8 @@ export class MongoCompactorV3 extends MongoCompactor { await this.finishObjectStorageReplacement(oldStoragePaths, persisted.storagePaths, persisted.uploads, session); opCountDiff = -clearedOpCount + 1; + before = inputStats; + after = statsForDocuments(persisted.documents); }, { writeConcern: { w: 'majority' }, @@ -678,7 +1173,7 @@ export class MongoCompactorV3 extends MongoCompactor { } ); - return { done, opCountDiff }; + return { done, opCountDiff, before, after }; } private async clearBoundaryDocument( @@ -688,15 +1183,19 @@ export class MongoCompactorV3 extends MongoCompactor { bucketContext: BucketDataContextV3, collection: mongo.Collection, context: { replicationStreamId: number; definitionId: string } - ): Promise { + ): Promise { const bucket = bucketContext.key.bucket; this.signal?.throwIfAborted(); const prepared = await this.prepareCompactionUploads(bucket, context, [lastNotPut, boundaryDocId.o]); let opCountDiff = 0; + let before = emptyBucketStats(); + let after = emptyBucketStats(); await session.withTransaction( async () => { opCountDiff = 0; + before = emptyBucketStats(); + after = emptyBucketStats(); const oldStoragePaths: string[] = []; const query = collection.find( { @@ -716,6 +1215,7 @@ export class MongoCompactorV3 extends MongoCompactor { min_op: 1, checksum: 1, count: 1, + size: 1, target_op: 1, ops: 1, storage_ref: 1 @@ -729,6 +1229,7 @@ export class MongoCompactorV3 extends MongoCompactor { let clearedOpCount = 0; let maxTargetOp: bigint | null = null; const boundarySurvivors: BucketDataDoc[] = []; + const inputStats = emptyBucketStats(); for await (const doc of query.stream()) { docsRead++; @@ -736,6 +1237,12 @@ export class MongoCompactorV3 extends MongoCompactor { throw new ReplicationAssertionError(`Unexpected extra document before CLEAR boundary in bucket ${bucket}`); } + const documentStats = statsForDocument(doc); + inputStats.count += documentStats.count; + inputStats.bytes += documentStats.bytes; + inputStats.chunks += documentStats.chunks; + inputStats.checksum = addChecksums(inputStats.checksum, documentStats.checksum); + const isBoundaryDoc = doc._id.o == boundaryDocId.o; if (doc.storage_ref) { oldStoragePaths.push(doc.storage_ref.path); @@ -802,6 +1309,8 @@ export class MongoCompactorV3 extends MongoCompactor { await this.finishObjectStorageReplacement(oldStoragePaths, persisted.storagePaths, persisted.uploads, session); opCountDiff = -clearedOpCount + 1; + before = inputStats; + after = statsForDocuments(persisted.documents); }, { writeConcern: { w: 'majority' }, @@ -809,7 +1318,7 @@ export class MongoCompactorV3 extends MongoCompactor { } ); - return opCountDiff; + return { opCountDiff, before, after }; } /** diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index f67dff166..e74d5ef45 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -3,6 +3,8 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { ServiceAssertionError } from '@powersync/lib-services-framework'; import { CheckpointChanges, + CompactInitialReplicationOptions, + CompactInitialReplicationResults, GetCheckpointChangesOptions, InternalOpId, internalToExternalOpId, @@ -192,6 +194,22 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { return new MongoCompactorV3(this, this.db, options); } + override async compactInitialReplication( + options: CompactInitialReplicationOptions + ): Promise { + this.logger.info(`Compacting chunks after initial replication...`); + const start = Date.now(); + const maxOpId = options.maxOpId ?? (await this.fetchPersistedOpHead()) ?? undefined; + const compactedBuckets = await this.createMongoCompactor({ + ...options, + maxOpId, + compactChunksOnly: true, + logger: this.logger + }).compact(); + this.logger.info(`Compacted chunks after initial replication in ${(Date.now() - start) / 1000}s`); + return { buckets: compactedBuckets }; + } + protected createMongoParameterCompactor( checkpoint: InternalOpId, options: storage.CompactOptions diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts index 563b65033..bc36afd40 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -15,6 +15,7 @@ import { import { SourceRecordLookupState } from '../common/SourceRecordStore.js'; import { serializeBucketData } from './bucket-format.js'; import { chunkBucketData } from './chunking.js'; +import { DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS } from './compaction-constants.js'; import { BucketDataDocumentV3, BucketStateDocumentV3, @@ -232,8 +233,10 @@ export class PersistedBatchV3 extends PersistedBatch { const createInserts: (() => Promise>)[] = []; for (const [bucket, ops] of operationsByBucket) { + this.resetBucketPersistedBytes(definitionId, bucket); for (const chunk of chunkBucketData(ops)) { const serialized = serializeBucketData(bucket, chunk); + this.incrementBucketPersistedChunk(definitionId, bucket, serialized.size); if (lifecycle == null || serialized.size <= this.inlineThresholdBytes) { createInserts.push(async () => ({ insertOne: { @@ -379,15 +382,42 @@ export class PersistedBatchV3 extends PersistedBatch { b: state.bucket } }, - update: { - $set: { - last_op: state.lastOp - }, - $inc: { - 'estimate_since_compact.count': state.incrementCount, - 'estimate_since_compact.bytes': state.incrementBytes + // A pipeline update makes initialisation and scheduling one atomic + // writer operation. In particular, a later write cannot move an + // already-due compact check into the future. + update: [ + { + $set: { + last_op: state.lastOp, + bucket_stats: { + count: { $add: [{ $ifNull: ['$bucket_stats.count', 0] }, state.incrementCount] }, + bytes: { $add: [{ $ifNull: ['$bucket_stats.bytes', 0n] }, BigInt(state.incrementBytes)] }, + chunks: { $add: [{ $ifNull: ['$bucket_stats.chunks', 0] }, state.incrementChunks] } + }, + first_uncompacted_write: { $ifNull: ['$first_uncompacted_write', '$$NOW'] }, + next_compact_check: { + $let: { + vars: { + requested: { + $dateAdd: { + startDate: '$$NOW', + unit: 'millisecond', + amount: DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS + } + } + }, + in: { + $cond: [ + { $lt: [{ $ifNull: ['$next_compact_check', '$$requested'] }, '$$requested'] }, + '$next_compact_check', + '$$requested' + ] + } + } + } + } } - }, + ], upsert: true } } satisfies mongo.AnyBulkWriteOperation; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts index 895a0729b..544422f1d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts @@ -117,11 +117,14 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo { }, { name: 'bucket_updates', unique: true } ); + // V3 workers claim only buckets that have reached their scheduled check + // time, keeping scheduled scans proportional to pending work. await bucketState.createIndex( + { next_compact_check: 1 }, { - 'estimate_since_compact.count': -1 - }, - { name: 'dirty_count' } + name: 'next_compact_check', + partialFilterExpression: { next_compact_check: { $exists: true } } + } ); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/compact-utils.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/compact-utils.ts new file mode 100644 index 000000000..51d754ee9 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/compact-utils.ts @@ -0,0 +1,320 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { addChecksums, InternalOpId } from '@powersync/service-core'; +import { BucketDataDoc } from '../common/BucketDataDoc.js'; +import { AVAILABLE_LEASE_EXPR, CompactionLease } from './CompactionLease.js'; +import { BucketDataDocumentV3, BucketStateDocumentV3 } from './models.js'; + +/** + * Perform chunk compaction if at least this many chunks have been added since + * the latest compaction. + * + * A lower value increases compaction frequency and can increase chunk rewrites + * and object-storage operations. A higher value leaves more small chunks for + * longer, which can hurt sync performance. + */ +const MERGE_CHUNKS_THRESHOLD = 8; + +export const FULL_COMPACT_RESCHEDULE_MARGIN_MS = 60 * 1000; + +export interface PendingCompactionGroup { + /** + * Input documents are ordered from oldest to newest, matching `ops`. + * Keeping the inputs intact lets unchanged singletons retain their object. + */ + inputs: BucketDataDocumentV3[]; + ops: BucketDataDoc[]; + changed: boolean; + targetOp: InternalOpId | null; +} + +export interface CompactIntervalConfig { + readonly minCompactChunkIntervalMs: number; + readonly minCompactFullIntervalMs: number; + readonly maxCompactFullIntervalMs: number; +} +export interface CompactTargetConfig { + readonly maxOpIdCap: InternalOpId | undefined; +} + +export enum CompactionKind { + Full = 'full', + Chunks = 'chunks' +} + +export interface CompactionDecision { + kind: CompactionKind | null; + nextCompactCheck: mongo.Document; +} + +export interface ScheduledCompactionOptions { + /** Process checks scheduled this far after the captured job start. */ + dueAheadMs?: number; + /** Used by initial replication, which must not run a full compact. */ + forceKind?: CompactionKind; +} + +export class CompactionContext { + constructor( + readonly lease: CompactionLease, + readonly kind: CompactionKind, + readonly decision: CompactionDecision, + readonly rescheduleNotBefore: Date | undefined, + readonly targetOp: InternalOpId + ) {} + + get state() { + return this.lease.state; + } + + get startedAt() { + return this.lease.startedAt; + } + + get lastOp() { + return this.lease.lastOp; + } +} + +export interface BucketStats { + count: number; + bytes: bigint; + chunks: number; +} + +/** Bucket stats read from bucket-data documents, including their checksum. */ +export interface BucketStatsWithChecksum extends BucketStats { + checksum: number; +} + +export interface CompactionResult { + /** Metadata cached at compacted_state.op_id. */ + compactedState: BucketStatsWithChecksum; + /** Complete bucket metadata through the op head captured at claim time. */ + bucketStats: BucketStats; +} + +export function bucketStats(state: BucketStateDocumentV3): BucketStats { + return { + count: state.bucket_stats.count, + bytes: state.bucket_stats.bytes, + chunks: state.bucket_stats.chunks + }; +} + +export function emptyBucketStats(): BucketStatsWithChecksum { + return { count: 0, bytes: 0n, chunks: 0, checksum: 0 }; +} + +export function statsForDocument( + document: Pick +): BucketStatsWithChecksum { + return { + count: document.count, + bytes: BigInt(document.size), + chunks: 1, + checksum: addChecksums(0, Number(document.checksum)) + }; +} + +export function statsForDocuments( + documents: Iterable> +): BucketStatsWithChecksum { + let result = emptyBucketStats(); + for (const document of documents) { + result = combineAdjacentStats(result, statsForDocument(document)); + } + return result; +} + +/** A scheduled bucket always has writes awaiting a full compact. */ +export function firstUncompactedWrite(state: BucketStateDocumentV3): Date { + if (state.first_uncompacted_write == null) { + throw new ReplicationAssertionError(`Scheduled V3 bucket ${state._id.b} has no first uncompacted write`); + } + return state.first_uncompacted_write; +} + +export function combineChunkStats( + previous: NonNullable, + compactedTail: BucketStatsWithChecksum, + overlappingCompactedChunk: BucketStatsWithChecksum +): BucketStatsWithChecksum { + return { + count: previous.count - overlappingCompactedChunk.count + compactedTail.count, + bytes: previous.bytes - overlappingCompactedChunk.bytes + compactedTail.bytes, + chunks: previous.chunks - 1 + compactedTail.chunks, + checksum: addChecksums( + addChecksums(Number(previous.checksum), -overlappingCompactedChunk.checksum), + compactedTail.checksum + ) + }; +} + +export function combineAdjacentStats( + first: BucketStatsWithChecksum, + second: BucketStatsWithChecksum +): BucketStatsWithChecksum { + return { + count: first.count + second.count, + bytes: first.bytes + second.bytes, + chunks: first.chunks + second.chunks, + checksum: addChecksums(first.checksum, second.checksum) + }; +} + +export function applyCompactionDelta( + total: BucketStats, + before: BucketStatsWithChecksum, + after: BucketStatsWithChecksum +): BucketStats { + return { + count: total.count - before.count + after.count, + bytes: total.bytes - before.bytes + after.bytes, + chunks: total.chunks - before.chunks + after.chunks + }; +} + +/** Replace one stored range in an accumulated statistics snapshot. */ +export function applyStatsReplacement( + total: BucketStatsWithChecksum, + before: BucketStatsWithChecksum, + after: BucketStatsWithChecksum +): BucketStatsWithChecksum { + return { + ...applyCompactionDelta(total, before, after), + checksum: addChecksums(addChecksums(total.checksum, -before.checksum), after.checksum) + }; +} + +/** + * This checks that the bucket state hasn't changed by a concurrent write since we checked. + * + * If it has changed, we'll re-check in the next batch. + */ +export function unclaimedSnapshotFilter(state: BucketStateDocumentV3): mongo.Filter { + return { + _id: state._id, + last_op: state.last_op, + next_compact_check: state.next_compact_check, + first_uncompacted_write: state.first_uncompacted_write ?? { $exists: false }, + bucket_stats: state.bucket_stats, + compacted_state: state.compacted_state ?? { $exists: false }, + last_full_compact: state.last_full_compact ?? { $exists: false }, + ...AVAILABLE_LEASE_EXPR + }; +} + +/** + * Calculate the earliest full compaction time from the first uncompacted + * write, bounded by the maximum retention interval. + */ +export function fullCompactionCheckAt(state: BucketStateDocumentV3, config: CompactIntervalConfig): Date { + const firstWrite = firstUncompactedWrite(state); + const stats = bucketStats(state); + const lastFull = state.last_full_compact; + + // The number of operations since the last full compact. + // We may make this more specific in the future, to track new updates and deletes only, ignoring + // full new inserts, but that requires more granular tracking when replicating. + const uncompactedCount = lastFull == null ? stats.count : Math.max(0, stats.count - lastFull.count); + const compactedRows = lastFull?.puts ?? 0; + + // If no full compact has ever been performed: ratio = 1, compact after minCompactFullIntervalMs. + // If every row has been updated or deleted exactly once since the last full compact: ratio = 0.5, compact after minCompactFullIntervalMs * 2. + // If 10% of rows has been updated since the last full compact, compact after minCompactFullIntervalMs * 11. + // If every row has been updated multiple times, the ratio tends closer to 1 again. + const ratio = uncompactedCount == 0 ? 0 : uncompactedCount / (compactedRows + uncompactedCount); + const fullIntervalMs = ratio == 0 ? config.maxCompactFullIntervalMs : config.minCompactFullIntervalMs / ratio; + return new Date(firstWrite.getTime() + Math.min(fullIntervalMs, config.maxCompactFullIntervalMs)); +} + +export function chooseCompactionKind( + state: BucketStateDocumentV3, + now: Date, + config: CompactIntervalConfig +): CompactionDecision { + // For chunk compaction, we consider the number of chunks added. + // Right now, we trigger a compact if the interval has passed and at least a threshold of chunks were added. + // A future policy could also use bytes per chunk or records per chunk to + // decide when to compact. + const fullCheckAt = fullCompactionCheckAt(state, config); + // Schedule a little late so a worker using a slightly earlier clock does + // not wake before the exact full-compaction condition is true. + const fullCheckWithMargin = new Date(fullCheckAt.getTime() + FULL_COMPACT_RESCHEDULE_MARGIN_MS); + const compacted = state.compacted_state; + const chunksSinceCompact = Math.max(0, state.bucket_stats.chunks - (compacted?.chunks ?? 0)); + const shouldCompactChunks = chunksSinceCompact >= MERGE_CHUNKS_THRESHOLD; + const canCheckChunks = + compacted == null || now.getTime() - compacted.at.getTime() >= config.minCompactChunkIntervalMs; + // Too few new chunks cannot make chunk compaction eligible. Do not poll this + // bucket at the chunk-compaction interval; only wake it for its full-compact check. + const nextCompactCheck: mongo.Document = !shouldCompactChunks + ? fullCheckWithMargin + : { + $min: [ + fullCheckWithMargin, + { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: config.minCompactChunkIntervalMs } } + ] + }; + let kind: CompactionKind | null = null; + if (now >= fullCheckAt) { + kind = CompactionKind.Full; + } else if (canCheckChunks && shouldCompactChunks) { + kind = CompactionKind.Chunks; + } + return { + kind, + nextCompactCheck + }; +} + +export function forcedCompactionKind( + state: BucketStateDocumentV3, + forceKind: CompactionKind | undefined, + config: CompactTargetConfig +): CompactionKind | null { + if (forceKind == null) { + return null; + } + const maxOpId = config.maxOpIdCap == null || state.last_op < config.maxOpIdCap ? state.last_op : config.maxOpIdCap; + return state.compacted_state == null || state.compacted_state.op_id < maxOpId ? forceKind : null; +} + +/** + * Read one bounded prefix from a compaction cursor. + * + * The document that would cross the byte limit is deliberately not returned: + * pagination resumes past the last returned `_id`, so that document remains + * eligible for the next query. The first document is always accepted to + * ensure progress when a single document exceeds the configured byte limit. + * + * `hasMore` is conservative when the document limit is reached. An extra empty + * query is preferable to exhausting the cursor just to determine whether the + * limited MongoDB query contained another document. + */ +export async function readCompactionBatch( + cursor: mongo.AggregationCursor, + options: { byteLimit: number; documentLimit: number } +): Promise<{ documents: BucketDataDocumentV3[]; hasMore: boolean }> { + const documents: BucketDataDocumentV3[] = []; + let cumulativeBytes = 0; + + try { + for await (const document of cursor) { + if (documents.length > 0 && cumulativeBytes + document.size > options.byteLimit) { + return { documents, hasMore: true }; + } + + documents.push(document); + cumulativeBytes += document.size; + + if (documents.length >= options.documentLimit) { + return { documents, hasMore: true }; + } + } + return { documents, hasMore: false }; + } finally { + await cursor.close(); + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/compaction-constants.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/compaction-constants.ts new file mode 100644 index 000000000..0ab77acf9 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/compaction-constants.ts @@ -0,0 +1,5 @@ +/** + * Writers use this delay before scheduling their first chunk compaction. + * Initial replication includes the same delay when it picks up that work. + */ +export const DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS = 5 * 60 * 1000; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts index b38011326..3e2273c3e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts @@ -16,7 +16,6 @@ import * as bson from 'bson'; import { BucketDataKey, BucketParameterDocumentBase, - BucketStateDocumentBase, CurrentBucket, OpType, ReplicaId, @@ -164,10 +163,57 @@ export interface SourceTableDocumentV3 { source_metadata?: JsonValue; } -export interface BucketStateDocumentV3 extends BucketStateDocumentBase { - _id: BucketStateDocumentBase['_id'] & { +export interface BucketStateDocumentV3 { + _id: { + b: string; d: BucketDefinitionId; }; + + /** Must always identify an actual operation in this logical stream. */ + last_op: bigint; + + /** The next time a compact worker should inspect this bucket. */ + next_compact_check: Date | undefined; + /** + * Scheduling epoch for work not covered by a full compact. Writers set the + * first actual write time; a partial full compact advances it to completion. + */ + first_uncompacted_write: Date | undefined; + + /** + * A checksum cache and the statistics captured by the latest compact (full + * or chunk). Keeping these separate from bucket_stats lets writers only + * update one set of counters. + */ + compacted_state?: { + op_id: InternalOpId; + checksum: bigint; + count: number; + bytes: bigint; + at: Date; + chunks: number; + }; + + /** Statistics for the prefix covered by the most recent full compact. */ + last_full_compact?: { + op_id: InternalOpId; + count: number; + at: Date; + puts: number; + }; + + /** Current aggregate bucket statistics, maintained by writers and compactors. */ + bucket_stats: { + count: number; + bytes: bigint; + chunks: number; + }; + + /** A short-lived ownership marker used to distribute bucket compaction. */ + compact_lease?: { + expires_at: Date; + id: unknown; + }; } export interface BucketOperation { diff --git a/modules/module-mongodb-storage/test/src/compact-utils.test.ts b/modules/module-mongodb-storage/test/src/compact-utils.test.ts new file mode 100644 index 000000000..61bb2f6c8 --- /dev/null +++ b/modules/module-mongodb-storage/test/src/compact-utils.test.ts @@ -0,0 +1,226 @@ +import { + applyCompactionDelta, + applyStatsReplacement, + bucketStats, + chooseCompactionKind, + combineAdjacentStats, + combineChunkStats, + CompactIntervalConfig, + CompactionKind, + emptyBucketStats, + forcedCompactionKind, + fullCompactionCheckAt, + statsForDocument, + statsForDocuments +} from '@module/storage/implementation/v3/compact-utils.js'; +import { BucketStateDocumentV3 } from '@module/storage/implementation/v3/models.js'; +import { describe, expect, test } from 'vitest'; + +const INTERVAL_CONFIG: CompactIntervalConfig = { + minCompactChunkIntervalMs: 1_000, + minCompactFullIntervalMs: 10_000, + maxCompactFullIntervalMs: 100_000 +}; + +function bucketState(overrides: Partial = {}): BucketStateDocumentV3 { + return { + _id: { d: 'definition', b: 'bucket[]' }, + last_op: 10n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { count: 10, bytes: 100n, chunks: 1 }, + ...overrides + }; +} + +function compactedState(overrides: Partial> = {}) { + return { + op_id: 5n, + checksum: 50n, + count: 5, + bytes: 50n, + chunks: 1, + at: new Date(0), + ...overrides + }; +} + +describe('V3 compact utilities', () => { + test.each([ + { + name: 'uses the minimum interval before the first full compact', + state: bucketState(), + expected: new Date(10_000) + }, + { + name: 'scales the interval by the uncompacted row ratio', + state: bucketState({ + bucket_stats: { count: 100, bytes: 100n, chunks: 1 }, + last_full_compact: { op_id: 90n, count: 90, puts: 90, at: new Date(0) } + }), + expected: new Date(100_000) + }, + { + name: 'uses the maximum interval when there are no uncompacted rows', + state: bucketState({ + bucket_stats: { count: 100, bytes: 100n, chunks: 1 }, + last_full_compact: { op_id: 10n, count: 100, puts: 100, at: new Date(0) } + }), + expected: new Date(100_000) + } + ])('$name', ({ state, expected }) => { + expect(fullCompactionCheckAt(state, INTERVAL_CONFIG)).toEqual(expected); + }); + + test('rejects a scheduled bucket without an uncompacted write', () => { + expect(() => fullCompactionCheckAt(bucketState({ first_uncompacted_write: undefined }), INTERVAL_CONFIG)).toThrow( + 'Scheduled V3 bucket bucket[] has no first uncompacted write' + ); + }); + + test('chooses a full compact when its check is due', () => { + const decision = chooseCompactionKind( + bucketState({ bucket_stats: { count: 10, bytes: 100n, chunks: 10 } }), + new Date(10_000), + INTERVAL_CONFIG + ); + + expect(decision.kind).toBe(CompactionKind.Full); + expect(decision.nextCompactCheck).toEqual({ + $min: [new Date(70_000), { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: 1_000 } }] + }); + }); + + test('chooses a chunk compact after enough chunks and the minimum interval', () => { + const decision = chooseCompactionKind( + bucketState({ + bucket_stats: { count: 10, bytes: 100n, chunks: 9 }, + compacted_state: compactedState({ chunks: 1 }) + }), + new Date(5_000), + INTERVAL_CONFIG + ); + + expect(decision.kind).toBe(CompactionKind.Chunks); + }); + + test('waits when the chunk interval has not elapsed', () => { + const decision = chooseCompactionKind( + bucketState({ + bucket_stats: { count: 10, bytes: 100n, chunks: 9 }, + compacted_state: compactedState({ chunks: 1, at: new Date(4_500) }) + }), + new Date(5_000), + INTERVAL_CONFIG + ); + + expect(decision.kind).toBeNull(); + }); + + test('schedules only the full check when too few chunks were added', () => { + const decision = chooseCompactionKind( + bucketState({ bucket_stats: { count: 7, bytes: 70n, chunks: 7 } }), + new Date(5_000), + INTERVAL_CONFIG + ); + + expect(decision).toEqual({ kind: null, nextCompactCheck: new Date(70_000) }); + }); + + test.each([ + { + name: 'does not force an unspecified kind', + forceKind: undefined, + compactedOpId: undefined, + maxOpIdCap: undefined, + expected: null + }, + { + name: 'forces work without previous compacted state', + forceKind: CompactionKind.Chunks, + compactedOpId: undefined, + maxOpIdCap: undefined, + expected: CompactionKind.Chunks + }, + { + name: 'skips work already compacted through the cap', + forceKind: CompactionKind.Chunks, + compactedOpId: 5n, + maxOpIdCap: 5n, + expected: null + }, + { + name: 'skips work already compacted through the bucket head', + forceKind: CompactionKind.Chunks, + compactedOpId: 10n, + maxOpIdCap: undefined, + expected: null + }, + { + name: 'forces work beyond the compacted state', + forceKind: CompactionKind.Chunks, + compactedOpId: 5n, + maxOpIdCap: 6n, + expected: CompactionKind.Chunks + } + ])('$name', ({ forceKind, compactedOpId, maxOpIdCap, expected }) => { + const state = bucketState({ + compacted_state: compactedOpId == null ? undefined : compactedState({ op_id: compactedOpId }) + }); + expect(forcedCompactionKind(state, forceKind, { maxOpIdCap })).toBe(expected); + }); + + test('derives and combines bucket statistics', () => { + const state = bucketState({ bucket_stats: { count: 12, bytes: 120n, chunks: 3 } }); + expect(bucketStats(state)).toEqual({ count: 12, bytes: 120n, chunks: 3 }); + expect(emptyBucketStats()).toEqual({ count: 0, bytes: 0n, chunks: 0, checksum: 0 }); + expect(statsForDocument({ count: 2, size: 20, checksum: 10n })).toEqual({ + count: 2, + bytes: 20n, + chunks: 1, + checksum: 10 + }); + expect( + statsForDocuments([ + { count: 2, size: 20, checksum: 10n }, + { count: 3, size: 30, checksum: 20n } + ]) + ).toEqual({ count: 5, bytes: 50n, chunks: 2, checksum: 30 }); + expect( + combineAdjacentStats( + { count: 2, bytes: 20n, chunks: 1, checksum: 10 }, + { count: 3, bytes: 30n, chunks: 2, checksum: 20 } + ) + ).toEqual({ count: 5, bytes: 50n, chunks: 3, checksum: 30 }); + }); + + test('replaces an overlapping chunk in cached statistics', () => { + expect( + combineChunkStats( + compactedState({ count: 10, bytes: 100n, chunks: 4, checksum: 100n }), + { count: 6, bytes: 60n, chunks: 2, checksum: 50 }, + { count: 3, bytes: 30n, chunks: 1, checksum: 30 } + ) + ).toEqual({ count: 13, bytes: 130n, chunks: 5, checksum: 120 }); + }); + + test('applies compacted statistics as a delta to total bucket statistics', () => { + expect( + applyCompactionDelta( + { count: 20, bytes: 200n, chunks: 8 }, + { count: 5, bytes: 50n, chunks: 3, checksum: 10 }, + { count: 3, bytes: 30n, chunks: 1, checksum: 10 } + ) + ).toEqual({ count: 18, bytes: 180n, chunks: 6 }); + }); + + test('applies a stored range replacement to statistics including its checksum', () => { + expect( + applyStatsReplacement( + { count: 20, bytes: 200n, chunks: 8, checksum: 100 }, + { count: 5, bytes: 50n, chunks: 3, checksum: 40 }, + { count: 3, bytes: 30n, chunks: 1, checksum: 20 } + ) + ).toEqual({ count: 18, bytes: 180n, chunks: 6, checksum: 80 }); + }); +}); diff --git a/modules/module-mongodb-storage/test/src/storage.test.ts b/modules/module-mongodb-storage/test/src/storage.test.ts index 5606af296..26adfaea2 100644 --- a/modules/module-mongodb-storage/test/src/storage.test.ts +++ b/modules/module-mongodb-storage/test/src/storage.test.ts @@ -1,7 +1,7 @@ import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js'; import { mongo } from '@powersync/lib-service-mongodb'; import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { register, test_utils } from '@powersync/service-core-tests'; +import { compactActive, register, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { env } from './env.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; @@ -115,7 +115,7 @@ bucket_definitions: await managedWriter.markAllSnapshotDone('1/1'); await managedWriter.keepalive('8/0'); - await bucketStorage.compact({ + await compactActive(factory, { compactBuckets: [], deleteCheckpointRequestsBefore: new Date('2024-02-01T00:00:00.000Z') }); diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index a91edc32f..258acf9b1 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -2,8 +2,12 @@ import { BucketDataDoc } from '@module/storage/implementation/common/BucketDataD import { MongoSyncBucketStorage } from '@module/storage/implementation/createMongoSyncBucketStorage.js'; import { loadBucketDataDocument, serializeBucketData } from '@module/storage/implementation/v3/bucket-format.js'; import { chunkBucketData, DEFAULT_MAX_DOC_SIZE_BYTES } from '@module/storage/implementation/v3/chunking.js'; +import { DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS } from '@module/storage/implementation/v3/compaction-constants.js'; +import { CompactionLease } from '@module/storage/implementation/v3/CompactionLease.js'; import { BucketDataDocumentV3 } from '@module/storage/implementation/v3/models.js'; +import { ObjectStorageError } from '@module/storage/implementation/v3/object-storage/ObjectStorage.js'; import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; +import { logger as defaultLogger } from '@powersync/lib-services-framework'; import { addChecksums, CheckpointChecksumInvalidatedError, @@ -11,10 +15,10 @@ import { SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, compactActive, register, test_utils } from '@powersync/service-core-tests'; import * as bson from 'bson'; -import { describe, expect, test } from 'vitest'; -import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; +import { describe, expect, test, vi } from 'vitest'; +import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; describe('Mongo Sync Bucket Storage Compact', () => { register.registerCompactTests(INITIALIZED_MONGO_STORAGE_FACTORY); @@ -58,15 +62,18 @@ describe('Mongo Sync Bucket Storage Compact', () => { return bucketStorage.getCheckpoint(); }; - const setup = async () => { + const setup = async (storageVersion?: number) => { await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); const syncRules = await factory.updateSyncRules( - updateSyncRulesFromYaml(` + updateSyncRulesFromYaml( + ` bucket_definitions: by_user: parameters: select request.user_id() as user_id data: [select * from test where owner_id = bucket.user_id] - `) + `, + { storageVersion } + ) ); const bucketStorage = factory.getInstance(syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; @@ -75,20 +82,17 @@ bucket_definitions: return { bucketStorage, checkpoint, factory, syncRules: syncRulesContent }; }; - test('full compact', async () => { + test('V1 full compact with blank bucket_state', async () => { const { bucketStorage, checkpoint, factory, syncRules } = await setup(); const storageDb = bucketStorage.db; - // Simulate bucket_state from old version not being available if (storageDb.storageConfig.incrementalReprocessing) { - // This should actually never happen on V3, but we test this anyway. - // Can remove this if it causes issues in the future. - await (storageDb as VersionedPowerSyncMongoV3).bucketState(bucketStorage.replicationStreamId).deleteMany({}); - } else { - await factory.db.bucket_state.deleteMany({}); + return; } + // Simulate a V1 deployment which pre-dates bucket-state population. + await factory.db.bucket_state.deleteMany({}); - await bucketStorage.compact({ + await compactActive(factory, { clearBatchLimit: 200, moveBatchLimit: 10, moveBatchQueryLimit: 10, @@ -113,18 +117,21 @@ bucket_definitions: }); }); - test('populatePersistentChecksumCache', async () => { + test.each(TEST_STORAGE_VERSIONS)('compactInitialReplication (storage v%s)', async (storageVersion) => { // Populate old replication stream - const { factory } = await setup(); + const { factory } = await setup(storageVersion); // Now populate another replication stream (bucket definition name changed) const syncRules = await factory.updateSyncRules( - updateSyncRulesFromYaml(` + updateSyncRulesFromYaml( + ` bucket_definitions: by_user2: parameters: select request.user_id() as user_id data: [select * from test where owner_id = bucket.user_id] - `) + `, + { storageVersion } + ) ); const bucketStorage = factory.getInstance(syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; @@ -132,21 +139,23 @@ bucket_definitions: await populate(bucketStorage, 2); const { checkpoint } = await bucketStorage.getCheckpoint(); - // Default is to small small numbers - should be a no-op - const result0 = await bucketStorage.populatePersistentChecksumCache({ + // V3's initial lite pass processes every bucket with no prior compact state. + // Earlier storage versions use the default minimum-change threshold. + const result0 = await bucketStorage.compactInitialReplication({ maxOpId: checkpoint }); - expect(result0.buckets).toEqual(0); + expect(result0.buckets).toEqual(storageVersion >= storage.STORAGE_VERSION_3 ? 2 : 0); - // This should cache the checksums for the two buckets - const result1 = await bucketStorage.populatePersistentChecksumCache({ + // For V1/V2, lower the threshold to populate the checksum cache. V3 has + // already updated its compacted state, so another initial pass is a no-op. + const result1 = await bucketStorage.compactInitialReplication({ maxOpId: checkpoint, minBucketChanges: 1 }); - expect(result1.buckets).toEqual(2); + expect(result1.buckets).toEqual(storageVersion >= storage.STORAGE_VERSION_3 ? 0 : 2); - // This should be a no-op, as the checksums are already cached - const result2 = await bucketStorage.populatePersistentChecksumCache({ + // Repeating it stays a no-op. + const result2 = await bucketStorage.compactInitialReplication({ maxOpId: checkpoint, minBucketChanges: 1 }); @@ -168,88 +177,82 @@ bucket_definitions: }); }); - test('dirty bucket discovery handles bigint bucket_state bytes', async () => { - await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); - const syncRules = await factory.updateSyncRules( - updateSyncRulesFromYaml(` -bucket_definitions: - global: - data: [select * from test] - `) - ); - const bucketStorage = factory.getInstance(syncRules); - const storageDb = bucketStorage.db; + test('v3 initial chunk compaction includes writer-scheduled work with a custom chunk interval', async () => { + const { bucketStorage, checkpoint } = await setup(storage.STORAGE_VERSION_3); + const result = await (bucketStorage as MongoSyncBucketStorage) + .createMongoCompactor({ + maxOpId: checkpoint, + compactChunksOnly: true, + minCompactChunkIntervalMs: 1 + }) + .compact(); + + expect(result).toBe(2); + }); - // This simulates bucket_state created using bigint bytes. - // This typically happens when buckets get very large (> 2GiB). We don't want to create that much - // data in the tests, so we directly insert the bucket_state here. - if (storageDb.storageConfig.incrementalReprocessing) { - const bucketStateCollection = (storageDb as VersionedPowerSyncMongoV3).bucketState( - bucketStorage.replicationStreamId - ); - await bucketStateCollection.insertOne({ - _id: { - d: '1', - b: 'global[]' - }, - last_op: 5n, - compacted_state: { - op_id: 3n, - count: 3, - checksum: 0n, - bytes: 7n - }, - estimate_since_compact: { - count: 2, - bytes: 5n - } - }); - } else { - await factory.db.bucket_state.insertOne({ - _id: { - g: bucketStorage.replicationStreamId, - b: 'global[]' - }, - last_op: 5n, - compacted_state: { - op_id: 3n, - count: 3, - checksum: 0n, - bytes: 7n - }, - estimate_since_compact: { - count: 2, - bytes: 5n - } - }); - } + test('v3 repeated initial chunk compaction reschedules overdue full work beyond the current run', async () => { + const { bucketStorage, checkpoint } = await setup(storage.STORAGE_VERSION_3); + const firstResult = await bucketStorage.compactInitialReplication({ maxOpId: checkpoint }); + expect(firstResult.buckets).toBe(2); - // This test uses a couple of "internal" APIs of the compactor. - const compactor = bucketStorage.createMongoCompactor({ maxOpId: 5n }); + const bucketStateCollection = (bucketStorage.db as VersionedPowerSyncMongoV3).bucketState( + bucketStorage.replicationStreamId + ); + await bucketStateCollection.updateMany( + {}, + { + $set: { + first_uncompacted_write: new Date(0), + next_compact_check: new Date(0) + } + } + ); - const dirtyBuckets = compactor.dirtyBucketBatches({ - minBucketChanges: 1, - minChangeRatio: 0.39 + const [{ now }] = await (bucketStorage.db as VersionedPowerSyncMongoV3).db + .aggregate<{ now: Date }>([{ $documents: [{}] }, { $project: { _id: 0, now: '$$NOW' } }]) + .toArray(); + const result = await bucketStorage.compactInitialReplication({ + maxOpId: checkpoint, + signal: AbortSignal.timeout(2_000) }); - const firstBatch = await dirtyBuckets.next(); - expect(firstBatch.done).toBe(false); - expect(firstBatch.value).toHaveLength(1); - expect(firstBatch.value[0].bucket).toBe('global[]'); - expect(firstBatch.value[0].estimatedCount).toBe(5); - expect(typeof firstBatch.value[0].estimatedCount).toBe('number'); - expect(firstBatch.value[0].dirtyRatio).toBeCloseTo(5 / 12); + expect(result.buckets).toBe(0); + const states = await bucketStateCollection.find({}).toArray(); + expect(states).toHaveLength(2); + for (const state of states) { + expect(state.next_compact_check!.getTime()).toBeGreaterThan( + now.getTime() + DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS + ); + } + }); + + test('v3 replication writes initialize scheduled compaction state', async () => { + const { bucketStorage } = await setup(); + const storageDb = bucketStorage.db; - const checksumBuckets = await compactor.dirtyBucketBatchForChecksums({ - minBucketChanges: 1 - }); - expect(checksumBuckets).toEqual([ - { - bucket: 'global[]', - definitionId: storageDb.storageConfig.incrementalReprocessing ? '1' : null, - estimatedCount: 5 - } - ]); + if (!storageDb.storageConfig.incrementalReprocessing) { + return; + } + const bucketStateCollection = (storageDb as VersionedPowerSyncMongoV3).bucketState( + bucketStorage.replicationStreamId + ); + const states = await bucketStateCollection.find({}).toArray(); + expect(states).toHaveLength(2); + for (const state of states) { + expect(state.last_op).toBeGreaterThan(0n); + expect(state.bucket_stats.count).toBe(1); + expect(state.bucket_stats.chunks).toBe(1); + expect(state.first_uncompacted_write).toBeInstanceOf(Date); + expect(state.next_compact_check).toBeInstanceOf(Date); + expect(state.compacted_state).toBeUndefined(); + expect(state).not.toHaveProperty('estimate_since_compact'); + + const docs = await (storageDb as VersionedPowerSyncMongoV3) + .bucketData(bucketStorage.replicationStreamId, state._id.d) + .find({ '_id.b': state._id.b }) + .toArray(); + expect(state.bucket_stats.bytes).toBe(BigInt(docs.reduce((total, document) => total + document.size, 0))); + } }); }); }); @@ -362,7 +365,9 @@ bucket_definitions: await bucketStateCollection.insertOne({ _id: { d: definitionId, b: BUCKET }, last_op: lastOp, - estimate_since_compact: { count: 10, bytes: 100 } + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { count: 10, bytes: 100n, chunks: 1 } }); } @@ -371,12 +376,133 @@ bucket_definitions: clearBatchLimit: 200, moveBatchLimit: 10, moveBatchQueryLimit: 10, - minBucketChanges: 1, - minChangeRatio: 0, maxOpId }); } + test('a successful lease renewal clears a transient renewal error', async () => { + const { bucketStateCollection, ctx } = await setupV3Storage(); + await insertBucketState(bucketStateCollection, ctx.definitionId, 1n); + const lease = await CompactionLease.claim( + bucketStateCollection, + { _id: { d: ctx.definitionId, b: BUCKET } }, + undefined, + 10 * 60 * 1000 + ); + expect(lease).not.toBeNull(); + + try { + const transientError = new Error('temporary MongoDB error'); + const renew = (lease as any).renew.bind(lease); + const updateOne = vi.spyOn(bucketStateCollection, 'updateOne').mockRejectedValueOnce(transientError); + await renew().catch((error: unknown) => { + (lease as any).renewalError = error; + }); + await expect(lease!.throwIfLost()).rejects.toBe(transientError); + + updateOne.mockRestore(); + await renew(); + await expect(lease!.throwIfLost()).resolves.toBeUndefined(); + } finally { + await lease?.[Symbol.asyncDispose](); + } + }); + + test('a failed scheduled bucket does not block other scheduled buckets', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3Storage(); + const badBucket = 'bad[]'; + const goodBucket = 'good[]'; + const goodDocument = serializeBucketData(goodBucket, [ + makeOp(2, 'good', 'good', { ...ctx, bucket: goodBucket }, sourceTableId) + ]); + await insertDocs(collection, [goodDocument]); + await bucketStateCollection.insertMany([ + { + _id: { d: ctx.definitionId, b: badBucket }, + last_op: 2n, + next_compact_check: new Date(0), + // A malformed scheduled bucket with an expired lease must be + // rescheduled without preventing valid buckets from compacting. + first_uncompacted_write: new Date(0), + bucket_stats: { count: 1, bytes: 1n, chunks: 1 }, + compact_lease: { id: new bson.ObjectId(), expires_at: new Date(0) } + }, + { + _id: { d: ctx.definitionId, b: goodBucket }, + last_op: 2n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { count: 1, bytes: BigInt(goodDocument.size), chunks: 1 } + } + ]); + await bucketStateCollection.updateOne( + { _id: { d: ctx.definitionId, b: badBucket } }, + { $unset: { first_uncompacted_write: '' } } + ); + + await (bucketStorage as MongoSyncBucketStorage) + .createMongoCompactor({ maxOpId: 2n, compactChunksOnly: true }) + .compact(); + + const [badState, goodState] = await Promise.all([ + bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: badBucket } }), + bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: goodBucket } }) + ]); + expect(badState?.next_compact_check).toBeInstanceOf(Date); + expect(badState!.next_compact_check!.getTime()).toBeGreaterThan(0); + expect(goodState?.compacted_state?.op_id).toBe(2n); + }); + + test('aborting scheduled compaction does not reschedule the remaining batch', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3Storage(); + const buckets = ['first[]', 'second[]', 'third[]']; + const documents = buckets.map((bucket) => + serializeBucketData(bucket, [makeOp(2, bucket, bucket, { ...ctx, bucket }, sourceTableId)]) + ); + await insertDocs(collection, documents); + await bucketStateCollection.insertMany( + documents.map((document, index) => ({ + _id: { d: ctx.definitionId, b: buckets[index] }, + last_op: 2n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { count: document.count, bytes: BigInt(document.size), chunks: 1 } + })) + ); + + const abortController = new AbortController(); + const testLogger = defaultLogger.child({}); + vi.spyOn(testLogger, 'info').mockImplementation((message: unknown) => { + if (typeof message == 'string' && message.startsWith('Compacted bucket chunks ')) { + abortController.abort(); + } + return testLogger; + }); + const errorLog = vi.spyOn(testLogger, 'error'); + + await expect( + bucketStorage + .createMongoCompactor({ + maxOpId: 2n, + compactChunksOnly: true, + signal: abortController.signal, + logger: testLogger + }) + .compact() + ).rejects.toThrow(); + + const states = await bucketStateCollection.find({ '_id.b': { $in: buckets } }).toArray(); + const completed = states.filter((state) => state.compacted_state != null); + const remaining = states.filter((state) => state.compacted_state == null); + expect(completed).toHaveLength(1); + expect(remaining).toHaveLength(2); + for (const state of remaining) { + expect(state.next_compact_check).toEqual(new Date(0)); + expect(state.compact_lease).toBeUndefined(); + } + expect(errorLog).not.toHaveBeenCalled(); + }); + test('1. ops[] ordering - preserves caller ordering (no implicit sort)', () => { const ops = [ makeBucketDataDoc({ o: 5n, data: '{"id":"c"}' }), @@ -835,7 +961,9 @@ bucket_definitions: await bucketStateCollection.insertOne({ _id: { d: definitionId, b: BUCKET }, last_op: 30n, - estimate_since_compact: { count: 3, bytes: 100 } + next_compact_check: undefined, + first_uncompacted_write: undefined, + bucket_stats: { count: 3, bytes: 100n, chunks: 1 } }); const request = checksumRequest(); @@ -874,7 +1002,9 @@ bucket_definitions: await bucketStateCollection.insertOne({ _id: { d: definitionId, b: BUCKET }, last_op: 0n, - estimate_since_compact: { count: 0, bytes: 0 } + next_compact_check: undefined, + first_uncompacted_write: undefined, + bucket_stats: { count: 0, bytes: 0n, chunks: 0 } }); const request = checksumRequest(); @@ -894,7 +1024,9 @@ bucket_definitions: await bucketStateCollection.insertOne({ _id: { d: definitionId, b: BUCKET }, last_op: 0n, - estimate_since_compact: { count: 0, bytes: 0 } + next_compact_check: undefined, + first_uncompacted_write: undefined, + bucket_stats: { count: 0, bytes: 0n, chunks: 0 } }); const request = checksumRequest(); @@ -913,7 +1045,9 @@ bucket_definitions: await bucketStateCollection.insertOne({ _id: { d: definitionId, b: BUCKET }, last_op: 10n, - estimate_since_compact: { count: 1, bytes: 100 } + next_compact_check: undefined, + first_uncompacted_write: undefined, + bucket_stats: { count: 1, bytes: 100n, chunks: 1 } }); const request = checksumRequest(); @@ -926,7 +1060,7 @@ bucket_definitions: await collection.insertOne(serializeBucketData(BUCKET, [clear, afterClear])); await bucketStateCollection.updateOne( { _id: { d: definitionId, b: BUCKET } }, - { $set: { last_op: 30n, 'estimate_since_compact.count': 2 } } + { $set: { last_op: 30n, 'bucket_stats.count': 2 } } ); const after = await bucketStorage.getChecksums(test_utils.testCheckpoint(30n), [request]); @@ -1004,7 +1138,9 @@ bucket_definitions: await bucketStateCollection.insertOne({ _id: { d: definitionId, b: BUCKET }, last_op: lastOp, - estimate_since_compact: { count: 10, bytes: 100 } + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { count: 10, bytes: 100n, chunks: 1 } }); } @@ -1030,6 +1166,156 @@ bucket_definitions: return collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray(); } + test('scheduled compaction claims only due buckets and clears completed full work', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); + const docs = [ + serializeBucketData(BUCKET, [makeOp(1, 'A', 'old', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(2, 'A', 'new', ctx, sourceTableId)]) + ]; + await insertDocs(collection, docs); + await bucketStateCollection.insertOne({ + _id: { d: ctx.definitionId, b: BUCKET }, + last_op: 2n, + next_compact_check: new Date(Date.now() + 60_000), + first_uncompacted_write: new Date(Date.now() - 60_000), + bucket_stats: { + count: 2, + bytes: BigInt(docs[0].size + docs[1].size), + chunks: 2 + } + }); + + await bucketStorage.compact({ maxOpId: 2n, maxCompactFullIntervalMs: 0 }); + expect( + (await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }))?.compacted_state + ).toBeUndefined(); + + await bucketStateCollection.updateOne( + { _id: { d: ctx.definitionId, b: BUCKET } }, + { $set: { next_compact_check: new Date(Date.now() - 1) } } + ); + await bucketStorage.compact({ maxOpId: 2n, maxCompactFullIntervalMs: 0 }); + + const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(state?.last_full_compact?.op_id).toBe(2n); + expect(state?.first_uncompacted_write).toBeUndefined(); + expect(state?.next_compact_check).toBeUndefined(); + expect(state?.compact_lease).toBeUndefined(); + }); + + test('capped full compaction records its prefix and starts a fresh scheduling window', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); + const prefix = serializeBucketData(BUCKET, [makeOp(1, 'A', 'a', ctx, sourceTableId)]); + const untouchedTail = serializeBucketData(BUCKET, [makeOp(2, 'B', 'b', ctx, sourceTableId)]); + await insertDocs(collection, [prefix, untouchedTail]); + await bucketStateCollection.insertOne({ + _id: { d: ctx.definitionId, b: BUCKET }, + last_op: 2n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { + count: prefix.count + untouchedTail.count, + bytes: BigInt(prefix.size + untouchedTail.size), + chunks: 2 + } + }); + + await bucketStorage.compact({ maxOpId: 1n }); + + const partialState = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(partialState?.last_full_compact).toMatchObject({ + op_id: 1n, + count: prefix.count, + puts: 1 + }); + expect(partialState?.first_uncompacted_write).toBeInstanceOf(Date); + expect(partialState!.first_uncompacted_write!.getTime()).toBeGreaterThan(0); + expect(partialState!.next_compact_check!.getTime()).toBeGreaterThanOrEqual( + partialState!.first_uncompacted_write!.getTime() + DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS + ); + + const lastFullCompactAt = partialState!.last_full_compact!.at; + const freshFirstUncompactedWrite = partialState!.first_uncompacted_write!; + await bucketStateCollection.updateOne( + { _id: { d: ctx.definitionId, b: BUCKET } }, + { $set: { next_compact_check: new Date(0) } } + ); + + await bucketStorage.compact({ maxOpId: 2n }); + + const deferredState = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(deferredState?.last_full_compact?.at).toEqual(lastFullCompactAt); + expect(deferredState?.first_uncompacted_write).toEqual(freshFirstUncompactedWrite); + expect(deferredState!.next_compact_check!.getTime()).toBeGreaterThan(Date.now()); + + // Even after the scheduling window has elapsed, a run with the old cap + // must not scan and record the same full-compaction prefix again. + await bucketStateCollection.updateOne( + { _id: { d: ctx.definitionId, b: BUCKET } }, + { $set: { first_uncompacted_write: new Date(0), next_compact_check: new Date(0) } } + ); + + await bucketStorage.compact({ maxOpId: 1n }); + + const rescheduledState = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(rescheduledState?.last_full_compact?.at).toEqual(lastFullCompactAt); + expect(rescheduledState?.first_uncompacted_write).toEqual(new Date(0)); + expect(rescheduledState!.next_compact_check!.getTime()).toBeGreaterThan(0); + }); + + test('explicit compaction skips a bucket with no outstanding full-compaction work', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); + const document = serializeBucketData(BUCKET, [makeOp(1, 'A', 'value', ctx, sourceTableId)]); + await insertDocs(collection, [document]); + await bucketStateCollection.insertOne({ + _id: { d: ctx.definitionId, b: BUCKET }, + last_op: 1n, + next_compact_check: undefined, + first_uncompacted_write: undefined, + compacted_state: { + op_id: 1n, + checksum: document.checksum, + count: document.count, + bytes: BigInt(document.size), + chunks: 1, + at: new Date() + }, + last_full_compact: { + op_id: 1n, + count: document.count, + puts: 1, + at: new Date() + }, + bucket_stats: { count: document.count, bytes: BigInt(document.size), chunks: 1 } + }); + + await expect(bucketStorage.compact({ compactBuckets: [BUCKET], maxOpId: 1n })).resolves.toBeUndefined(); + + const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(state?.compact_lease).toBeUndefined(); + expect(state?.next_compact_check).toBeNull(); + expect(state?.last_full_compact?.op_id).toBe(1n); + }); + + test('concurrent scheduled compactors lease a bucket to one worker', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); + const document = serializeBucketData(BUCKET, [makeOp(1, 'A', 'value', ctx, sourceTableId)]); + await insertDocs(collection, [document]); + await bucketStateCollection.insertOne({ + _id: { d: ctx.definitionId, b: BUCKET }, + last_op: 1n, + next_compact_check: new Date(Date.now() - 1), + first_uncompacted_write: new Date(0), + bucket_stats: { count: 1, bytes: BigInt(document.size), chunks: 1 } + }); + + await Promise.all([bucketStorage.compact({ maxOpId: 1n }), bucketStorage.compact({ maxOpId: 1n })]); + + const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(state?.last_full_compact?.op_id).toBe(1n); + expect(state?.compact_lease).toBeUndefined(); + }); + test('1. superseded ops become MOVE tombstones', async () => { const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); // Doc1: [A@10, B@20, A@30] — A@10 superseded by A@30, becomes MOVE tombstone @@ -1271,7 +1557,9 @@ bucket_definitions: await bucketStateCollection.insertOne({ _id: { d: definitionId, b: BUCKET }, last_op: lastOp, - estimate_since_compact: { count: 10, bytes: 100 } + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { count: 10, bytes: 100n, chunks: 1 } }); } @@ -1483,7 +1771,9 @@ bucket_definitions: await bucketStateCollection.insertOne({ _id: { d: definitionId, b: BUCKET }, last_op: lastOp, - estimate_since_compact: { count: 10, bytes: 100 } + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { count: 10, bytes: 100n, chunks: 1 } }); } @@ -1501,6 +1791,277 @@ bucket_definitions: ); } + async function setupCompactedTail() { + const setup = await setupV3(); + const { collection, bucketStateCollection, ctx, sourceTableId } = setup; + const documents = [ + serializeBucketData(BUCKET, [makeOp(1, 'A', 'a', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(2, 'B', 'b', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(3, 'C', 'c', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(4, 'D', 'd', ctx, sourceTableId)]) + ]; + await insertDocs(collection, documents); + await bucketStateCollection.insertOne({ + _id: { d: ctx.definitionId, b: BUCKET }, + last_op: 4n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + compacted_state: { + op_id: 2n, + checksum: documents[0].checksum + documents[1].checksum, + count: documents[0].count + documents[1].count, + bytes: BigInt(documents[0].size + documents[1].size), + chunks: 2, + at: new Date(0) + }, + bucket_stats: { + count: documents.reduce((total, document) => total + document.count, 0), + bytes: BigInt(documents.reduce((total, document) => total + document.size, 0)), + chunks: documents.length + } + }); + return setup; + } + + async function expectRecoveredCompactedTail(collection: any, bucketStateCollection: any, definitionId: string) { + const documents = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray(); + expect(documents.flatMap((document: BucketDataDocumentV3) => document.ops!.map((op) => op.o))).toEqual([ + 1n, + 2n, + 3n, + 4n + ]); + + const bytes = BigInt(documents.reduce((total: number, document: BucketDataDocumentV3) => total + document.size, 0)); + const state = await bucketStateCollection.findOne({ _id: { d: definitionId, b: BUCKET } }); + expect(state?.compacted_state).toMatchObject({ + op_id: 4n, + checksum: 70n, + count: 4, + bytes, + chunks: documents.length + }); + expect(state?.bucket_stats).toEqual({ count: 4, bytes, chunks: documents.length }); + } + + test('initial compaction merges small chunks and refreshes bucket metadata', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); + await insertDocs(collection, [ + serializeBucketData(BUCKET, [makeOp(1, 'A', 'a', ctx, sourceTableId), makeOp(2, 'B', 'b', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(3, 'C', 'c', ctx, sourceTableId), makeOp(4, 'D', 'd', ctx, sourceTableId)]) + ]); + await insertBucketState(bucketStateCollection, ctx.definitionId, 4n); + + const result = await bucketStorage.compactInitialReplication({ maxOpId: 4n }); + + expect(result).toEqual({ buckets: 1 }); + const documents = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray(); + expect(documents).toHaveLength(1); + expect(documents[0].ops!.map((op) => op.o)).toEqual([1n, 2n, 3n, 4n]); + + const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(state?.compacted_state).toMatchObject({ + op_id: 4n, + count: 4, + checksum: 70n + }); + }); + + test('capped chunk compaction repairs stale bucket stats after a committed first merge', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); + const operations = [ + makeOp(1, 'A', 'a', ctx, sourceTableId), + makeOp(2, 'B', 'b', ctx, sourceTableId), + makeOp(3, 'C', 'c', ctx, sourceTableId), + makeOp(4, 'D', 'd', ctx, sourceTableId) + ]; + const originalDocuments = [ + serializeBucketData(BUCKET, operations.slice(0, 2)), + serializeBucketData(BUCKET, operations.slice(2)) + ]; + const committedMerge = serializeBucketData(BUCKET, operations); + const untouchedTail = serializeBucketData(BUCKET, [makeOp(5, 'E', 'e', ctx, sourceTableId)]); + await insertDocs(collection, [committedMerge, untouchedTail]); + await bucketStateCollection.insertOne({ + _id: { d: ctx.definitionId, b: BUCKET }, + last_op: 5n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { + count: originalDocuments.reduce((total, document) => total + document.count, untouchedTail.count), + bytes: BigInt(originalDocuments.reduce((total, document) => total + document.size, untouchedTail.size)), + chunks: originalDocuments.length + 1 + } + }); + + const result = await bucketStorage.compactInitialReplication({ maxOpId: 4n }); + + expect(result).toEqual({ buckets: 1 }); + const expectedStats = { + count: committedMerge.count + untouchedTail.count, + bytes: BigInt(committedMerge.size + untouchedTail.size), + chunks: 2 + }; + const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(state?.bucket_stats).toEqual(expectedStats); + expect(state?.compacted_state).toMatchObject({ + op_id: 4n, + checksum: committedMerge.checksum, + count: committedMerge.count, + bytes: BigInt(committedMerge.size), + chunks: 1 + }); + }); + + test('later full compaction claims fresh state after a committed replacement', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); + const documents = [ + serializeBucketData(BUCKET, [makeOp(1, 'A', 'a', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(2, 'B', 'b', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(3, 'C', 'c', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(4, 'D', 'd', ctx, sourceTableId)]) + ]; + await insertDocs(collection, documents); + await bucketStateCollection.insertOne({ + _id: { d: ctx.definitionId, b: BUCKET }, + last_op: 4n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + bucket_stats: { + count: documents.reduce((total, document) => total + document.count, 0), + bytes: BigInt(documents.reduce((total, document) => total + document.size, 0)), + chunks: documents.length + } + }); + + const concurrentDocument = serializeBucketData(BUCKET, [makeOp(5, 'E', 'e', ctx, sourceTableId)]); + const compactor = bucketStorage.createMongoCompactor({ maxOpId: 5n, compactBuckets: [BUCKET] }); + const originalFlush = (compactor as any).flushCompactionGroup.bind(compactor); + let injectedFailure = false; + vi.spyOn(compactor as any, 'flushCompactionGroup').mockImplementation(async (...args: any[]) => { + const result = await originalFlush(...args); + if (!injectedFailure) { + injectedFailure = true; + await insertDocs(collection, [concurrentDocument]); + await bucketStateCollection.updateOne( + { _id: { d: ctx.definitionId, b: BUCKET } }, + { + $set: { last_op: 5n }, + $inc: { + 'bucket_stats.count': concurrentDocument.count, + 'bucket_stats.bytes': BigInt(concurrentDocument.size), + 'bucket_stats.chunks': 1 + } + } + ); + throw new ObjectStorageError('failure after committed full-compaction replacement', { + cause: new Error('socket reset'), + retryable: true + }); + } + return result; + }); + + await expect(compactor.compact()).rejects.toThrow('failure after committed full-compaction replacement'); + + expect(injectedFailure).toBe(true); + const interruptedState = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(interruptedState?.last_op).toBe(5n); + expect(interruptedState?.compacted_state).toBeUndefined(); + expect(interruptedState?.compact_lease).toBeUndefined(); + + await expect(bucketStorage.createMongoCompactor({ maxOpId: 5n, compactBuckets: [BUCKET] }).compact()).resolves.toBe( + 1 + ); + + const currentDocuments = await collection.find({ '_id.b': BUCKET }).toArray(); + expect(currentDocuments).toHaveLength(1); + const expectedStats = { + count: currentDocuments.reduce((total, document) => total + document.count, 0), + bytes: BigInt(currentDocuments.reduce((total, document) => total + document.size, 0)), + chunks: currentDocuments.length + }; + const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(state?.bucket_stats).toEqual(expectedStats); + expect(state?.compacted_state).toMatchObject({ op_id: 5n, ...expectedStats }); + expect(state?.last_full_compact?.op_id).toBe(5n); + expect(state?.first_uncompacted_write).toBeUndefined(); + expect(state?.next_compact_check).toBeUndefined(); + }); + + test('chunk compaction treats a missing cached op as a resume hint', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); + const documents = [ + serializeBucketData(BUCKET, [makeOp(1, 'A', 'a', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(3, 'C', 'c', ctx, sourceTableId)]), + serializeBucketData(BUCKET, [makeOp(4, 'D', 'd', ctx, sourceTableId)]) + ]; + await insertDocs(collection, documents); + await bucketStateCollection.insertOne({ + _id: { d: ctx.definitionId, b: BUCKET }, + last_op: 4n, + next_compact_check: new Date(0), + first_uncompacted_write: new Date(0), + compacted_state: { + op_id: 2n, + checksum: 21n, + count: 2, + bytes: 100n, + chunks: 2, + at: new Date(0) + }, + bucket_stats: { count: 4, bytes: 200n, chunks: 4 } + }); + + await expect(bucketStorage.createMongoCompactor({ maxOpId: 4n, compactChunksOnly: true }).compact()).resolves.toBe( + 1 + ); + + const currentDocuments = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray(); + expect(currentDocuments.flatMap((document) => document.ops!.map((op) => op.o))).toEqual([1n, 3n, 4n]); + const bytes = BigInt(currentDocuments.reduce((total, document) => total + document.size, 0)); + const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(state?.compacted_state).toMatchObject({ + op_id: 4n, + checksum: 56n, + count: 3, + bytes, + chunks: currentDocuments.length + }); + expect(state?.bucket_stats).toEqual({ count: 3, bytes, chunks: currentDocuments.length }); + }); + + test('later chunk compaction rebuilds state after a committed partial merge', async () => { + const { bucketStorage, collection, bucketStateCollection, ctx } = await setupCompactedTail(); + const failedCompactor = bucketStorage.createMongoCompactor({ + maxOpId: 4n, + compactChunksOnly: true + }); + const originalFlush = (failedCompactor as any).flushCompactionGroup.bind(failedCompactor); + const flushSpy = vi + .spyOn(failedCompactor as any, 'flushCompactionGroup') + .mockImplementation(async (...args: any[]) => { + const result = await originalFlush(...args); + throw new Error('failure after committed chunk merge'); + }); + + await expect(failedCompactor.compact()).resolves.toBe(0); + flushSpy.mockRestore(); + + const interruptedState = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(interruptedState?.compacted_state?.op_id).toBe(2n); + expect(await collection.findOne({ _id: { b: BUCKET, o: 2n } })).toBeNull(); + await bucketStateCollection.updateOne( + { _id: { d: ctx.definitionId, b: BUCKET } }, + { $set: { next_compact_check: new Date(0) } } + ); + + const result = await bucketStorage.createMongoCompactor({ maxOpId: 4n, compactChunksOnly: true }).compact(); + + expect(result).toBe(1); + await expectRecoveredCompactedTail(collection, bucketStateCollection, ctx.definitionId); + }); + test('1. multi-batch compaction preserves checksum and creates MOVE tombstones', async () => { const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3(); @@ -1616,6 +2177,13 @@ bucket_definitions: expect(op600).toBeDefined(); expect(op600!.op).toBe('PUT'); expect(op600!.row_id).toBe('F'); + + const state = await bucketStateCollection.findOne({ _id: { d: ctx.definitionId, b: BUCKET } }); + expect(state?.bucket_stats).toEqual({ + count: docsAfter.reduce((total, document) => total + document.count, 0), + bytes: BigInt(docsAfter.reduce((total, document) => total + document.size, 0)), + chunks: docsAfter.length + }); }); test('3. seen map overflow - some old ops pass through without tombstoning', async () => { diff --git a/modules/module-mongodb-storage/test/src/storage_s3_checksums.test.ts b/modules/module-mongodb-storage/test/src/storage_s3_checksums.test.ts index d34d687f2..aaaa8ee8d 100644 --- a/modules/module-mongodb-storage/test/src/storage_s3_checksums.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_s3_checksums.test.ts @@ -4,7 +4,7 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, compactActive, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js'; import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; @@ -60,8 +60,7 @@ describe('V3 checksums with S3 object storage', () => { { $set: { last_op: 3n, - compacted_state: { op_id: 3n, count: 0, checksum: 0n, bytes: null }, - estimate_since_compact: { count: 3, bytes: 100 } + compacted_state: { op_id: 3n, count: 0, checksum: 0n, bytes: 0n, chunks: 0, at: new Date() } } }, { upsert: true } @@ -154,7 +153,7 @@ describe('V3 checksums with S3 object storage', () => { const checkpoint = await bucketStorage.getCheckpoint(); // Compact — produces CLEAR doc from collapsed MOVEs - await bucketStorage.compact({ + await compactActive(factory, { maxOpId: checkpoint.checkpoint, compactBuckets: [request.bucket], clearBatchLimit: 200, diff --git a/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts b/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts index 7c4a20934..03d66c92a 100644 --- a/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts @@ -1,5 +1,5 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, compactActive, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js'; import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; @@ -29,7 +29,7 @@ function memoryS3Factory(options: { inlineThresholdBytes?: number } = {}) { } describe('S3 compaction storage lifecycle', () => { - test('retries transient object storage failures', async () => { + test('a later compaction recovers from a transient object storage failure', async () => { const { memoryStorage, factory: factoryGen } = memoryS3Factory(); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); @@ -63,12 +63,14 @@ describe('S3 compaction storage lifecycle', () => { const checkpoint = await bucketStorage.getCheckpoint(); const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n); - await bucketStorage.compact({ + const compactOptions = { maxOpId: checkpoint.checkpoint, compactBuckets: [request.bucket], minBucketChanges: 1, minChangeRatio: 0 - }); + }; + await expect(compactActive(factory, compactOptions)).rejects.toThrow('temporary object storage failure'); + await compactActive(factory, compactOptions); expect(injectedFailure).toBe(true); const batch = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request])); @@ -107,7 +109,7 @@ describe('S3 compaction storage lifecycle', () => { const checkpoint = await bucketStorage.getCheckpoint(); const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n); - await bucketStorage.compact({ + await compactActive(factory, { maxOpId: checkpoint.checkpoint, compactBuckets: [request.bucket], minBucketChanges: 1, @@ -167,7 +169,7 @@ describe('S3 compaction storage lifecycle', () => { const checkpoint = await bucketStorage.getCheckpoint(); const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n); - await bucketStorage.compact({ + await compactActive(factory, { maxOpId: checkpoint.checkpoint, compactBuckets: [request.bucket], clearBatchLimit: 200, @@ -230,7 +232,7 @@ describe('S3 compaction storage lifecycle', () => { const lowerPaths = new Set(docsBefore.slice(0, 6).map((doc) => doc.storage_ref!.path)); const upperPaths = new Set(docsBefore.slice(6).map((doc) => doc.storage_ref!.path)); - await bucketStorage.compact({ + await compactActive(factory, { maxOpId, compactBuckets: [request.bucket], clearBatchLimit: 200, @@ -337,14 +339,14 @@ describe('S3 compaction storage lifecycle', () => { const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n); const bucket = request.bucket; - // Read bucket_state before compaction to confirm it exists and has - // estimate_since_compact populated by the writer. + // Read bucket_state before compaction to confirm the writer recorded its + // aggregate statistics and scheduled a compact check. const bucketStateBefore = await bucketStateCollection.findOne({ _id: { d: definitionId, b: bucket } }); expect(bucketStateBefore).toBeDefined(); - expect(bucketStateBefore!.estimate_since_compact).toBeDefined(); - expect(bucketStateBefore!.estimate_since_compact!.count).toBeGreaterThan(0); + expect(bucketStateBefore!.bucket_stats.count).toBeGreaterThan(0); + expect(bucketStateBefore!.next_compact_check).toBeDefined(); // Record the input operations and object path. const batchBefore = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request])); @@ -357,7 +359,7 @@ describe('S3 compaction storage lifecycle', () => { expect(expectedCompactedOpId).not.toBeNull(); // Compact the bucket. - await bucketStorage.compact({ + await compactActive(factory, { maxOpId: checkpoint.checkpoint, compactBuckets: [bucket], clearBatchLimit: 200, diff --git a/modules/module-mongodb/src/replication/MongoSnapshotter.ts b/modules/module-mongodb/src/replication/MongoSnapshotter.ts index ffe4fc248..2c898442b 100644 --- a/modules/module-mongodb/src/replication/MongoSnapshotter.ts +++ b/modules/module-mongodb/src/replication/MongoSnapshotter.ts @@ -345,10 +345,10 @@ export class MongoSnapshotter { return; } - // Populate the cache _after_ initial replication, but _before_ we switch to this replication stream. + // Compact storage _after_ initial replication, but _before_ we switch to this replication stream. // Keeping snapshot_done false until this completes makes this resumable after interruption. // No checkpoint exists yet - storage defaults to its highest persisted op id. - await this.storage.populatePersistentChecksumCache({ + await this.storage.compactInitialReplication({ signal: this.abortSignal }); diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index 7274a46fc..5e2396df2 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -305,8 +305,8 @@ export class BinLogStream { } if (lastOp != null) { - // Populate the cache _after_ initial replication, but _before_ we switch to this replication stream. - await this.storage.populatePersistentChecksumCache({ + // Compact storage _after_ initial replication, but _before_ we switch to this replication stream. + await this.storage.compactInitialReplication({ // No checkpoint yet, but we do have the opId. maxOpId: lastOp, signal: this.abortSignal diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index fc078dba6..3dafed680 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -4,6 +4,8 @@ import { BucketChecksum, CHECKPOINT_INVALIDATE_ALL, CheckpointChanges, + CompactInitialReplicationOptions, + CompactInitialReplicationResults, GetCheckpointChangesOptions, InternalOpId, internalToExternalOpId, @@ -11,11 +13,10 @@ import { maxLsn, ParameterSetLimitExceededError, PartialChecksum, - PopulateChecksumCacheOptions, - PopulateChecksumCacheResults, ReplicationCheckpoint, storage, StorageVersionConfig, + SyncRuleState, utils, WatchWriteCheckpointOptions } from '@powersync/service-core'; @@ -146,6 +147,14 @@ export class PostgresSyncRulesStorage } async compact(options?: storage.CompactOptions): Promise { + if (options?.incrementalOnly) { + // Not supported yet + this.logger.info('Incremental compacting is not supported on Postgres storage yet.'); + return; + } else if (this.replicationStream.state != SyncRuleState.ACTIVE) { + this.logger.info(`Skipping compacting of replication stream in ${this.replicationStream.state} state.`); + return; + } let maxOpId = options?.maxOpId; if (maxOpId == null) { const checkpoint = await this.getCheckpoint(); @@ -160,7 +169,9 @@ export class PostgresSyncRulesStorage }).compact(); } - async populatePersistentChecksumCache(_options: PopulateChecksumCacheOptions): Promise { + async compactInitialReplication( + _options: CompactInitialReplicationOptions + ): Promise { // no-op - checksum cache is not implemented for Postgres yet return { buckets: 0 }; } diff --git a/modules/module-postgres-storage/test/src/storage.test.ts b/modules/module-postgres-storage/test/src/storage.test.ts index 0620a2b4a..725967838 100644 --- a/modules/module-postgres-storage/test/src/storage.test.ts +++ b/modules/module-postgres-storage/test/src/storage.test.ts @@ -1,5 +1,5 @@ import { framework, storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, compactActive, register, test_utils } from '@powersync/service-core-tests'; import * as t from 'ts-codec'; import { describe, expect, test } from 'vitest'; import { CLEAR_BATCH_LIMIT } from '../../src/storage/PostgresSyncRulesStorage.js'; @@ -105,7 +105,12 @@ bucket_definitions: WHERE user_id = 'user2' `.execute(); - await bucketStorage.compact({ + { + await using activationWriter = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await activationWriter.markAllSnapshotDone('1/1'); + await activationWriter.keepalive('1/1'); + } + await compactActive(factory, { compactBuckets: [], deleteCheckpointRequestsBefore: new Date('2024-02-01T00:00:00.000Z') }); diff --git a/modules/module-postgres-storage/test/src/storage_compacting.test.ts b/modules/module-postgres-storage/test/src/storage_compacting.test.ts index 43d4132ae..b6ba586c0 100644 --- a/modules/module-postgres-storage/test/src/storage_compacting.test.ts +++ b/modules/module-postgres-storage/test/src/storage_compacting.test.ts @@ -1,5 +1,5 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, compactActive, register, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { PostgresCompactor } from '../../src/storage/PostgresCompactor.js'; import { POSTGRES_STORAGE_FACTORY } from './util.js'; @@ -44,7 +44,7 @@ bucket_definitions: // Compact with an explicit bucket name — exercises the this.buckets // iteration path, NOT the compactAllBuckets discovery path. - await bucketStorage.compact({ + await compactActive(factory, { compactBuckets: [bucketRequest(syncRulesContent, 'global[]').bucket], minBucketChanges: 1 }); diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index ce13dd9b1..d4e0716b3 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -10,6 +10,7 @@ import { } from '@powersync/lib-services-framework'; import { BucketStorageBatch, + formatBytes, getUuidReplicaIdentityBson, MetricsEngine, RelationCache, @@ -47,7 +48,7 @@ import { SimpleSnapshotQuery, SnapshotQuery } from './SnapshotQuery.js'; -import { computeWalBudgetReport, formatBytes, formatWalBudgetLine } from './wal-budget-utils.js'; +import { computeWalBudgetReport, formatWalBudgetLine } from './wal-budget-utils.js'; export interface WalStreamOptions { logger?: Logger; @@ -633,8 +634,8 @@ WHERE oid = $1::regclass`, const lastOp = flushResults?.flushed_op; if (lastOp != null) { - // Populate the cache _after_ initial replication, but _before_ we switch to this replication stream. - await this.storage.populatePersistentChecksumCache({ + // Compact storage _after_ initial replication, but _before_ we switch to this replication stream. + await this.storage.compactInitialReplication({ // No checkpoint yet, but we do have the opId. maxOpId: lastOp, signal: this.abort_signal diff --git a/modules/module-postgres/src/replication/wal-budget-utils.ts b/modules/module-postgres/src/replication/wal-budget-utils.ts index 3637824f7..49ac2939d 100644 --- a/modules/module-postgres/src/replication/wal-budget-utils.ts +++ b/modules/module-postgres/src/replication/wal-budget-utils.ts @@ -1,13 +1,4 @@ -export function formatBytes(bytes: number): string { - if (bytes >= 1024 * 1024 * 1024) { - return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`; - } else if (bytes >= 1024 * 1024) { - return `${(bytes / (1024 * 1024)).toFixed(1)}MB`; - } else if (bytes >= 1024) { - return `${(bytes / 1024).toFixed(1)}KB`; - } - return `${bytes}B`; -} +import { formatBytes } from '@powersync/service-core'; export function formatDuration(hours: number): string { if (hours >= 24) { diff --git a/modules/module-postgres/test/src/slow_tests.test.ts b/modules/module-postgres/test/src/slow_tests.test.ts index d8e37300a..832508549 100644 --- a/modules/module-postgres/test/src/slow_tests.test.ts +++ b/modules/module-postgres/test/src/slow_tests.test.ts @@ -183,10 +183,15 @@ bucket_definitions: break; } - const checkpoint = await storage.getCheckpoint(); - const opsBefore = await helpers.getBucketData('global[]', checkpoint); - await storage.compact({ maxOpId: checkpoint.checkpoint }); - const opsAfter = await helpers.getBucketData('global[]', checkpoint); + const active = await f.getActiveSyncConfig(); + if (active == null) { + continue; + } + const activeHelpers = new StorageDataHelpers(active.storage, syncRulesContent); + const checkpoint = await active.storage.getCheckpoint(); + const opsBefore = await activeHelpers.getBucketData('global[]', checkpoint); + await active.storage.compact({ maxOpId: checkpoint.checkpoint }); + const opsAfter = await activeHelpers.getBucketData('global[]', checkpoint); test_utils.validateCompactedBucket(opsBefore, opsAfter); } diff --git a/modules/module-postgres/test/src/wal_budget.test.ts b/modules/module-postgres/test/src/wal_budget.test.ts index b4c52235a..83ff64067 100644 --- a/modules/module-postgres/test/src/wal_budget.test.ts +++ b/modules/module-postgres/test/src/wal_budget.test.ts @@ -1,32 +1,6 @@ -import { - computeWalBudgetReport, - formatBytes, - formatDuration, - formatWalBudgetLine -} from '@module/replication/wal-budget-utils.js'; +import { computeWalBudgetReport, formatDuration, formatWalBudgetLine } from '@module/replication/wal-budget-utils.js'; import { describe, expect, test } from 'vitest'; -describe('formatBytes', () => { - test('formats bytes', () => { - expect(formatBytes(500)).toBe('500B'); - }); - - test('formats kilobytes', () => { - expect(formatBytes(1024)).toBe('1.0KB'); - expect(formatBytes(1536)).toBe('1.5KB'); - }); - - test('formats megabytes', () => { - expect(formatBytes(1024 * 1024)).toBe('1.0MB'); - expect(formatBytes(1.5 * 1024 * 1024)).toBe('1.5MB'); - }); - - test('formats gigabytes', () => { - expect(formatBytes(1024 * 1024 * 1024)).toBe('1.0GB'); - expect(formatBytes(10.5 * 1024 * 1024 * 1024)).toBe('10.5GB'); - }); -}); - describe('formatDuration', () => { test('formats minutes', () => { expect(formatDuration(0.5)).toBe('30 minutes'); diff --git a/packages/service-core-tests/src/tests/register-compacting-tests.ts b/packages/service-core-tests/src/tests/register-compacting-tests.ts index a30a6cca9..916952855 100644 --- a/packages/service-core-tests/src/tests/register-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-compacting-tests.ts @@ -2,7 +2,7 @@ import { addChecksums, storage, updateSyncRulesFromYaml } from '@powersync/servi import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; import { bucketRequest } from '../test-utils/test-utils-index.js'; -import { bucketRequestMap, bucketRequests } from './util.js'; +import { bucketRequestMap, bucketRequests, compactActive } from './util.js'; export function registerCompactTests(config: storage.TestStorageConfig) { const generateStorageFactory = config.factory; @@ -81,7 +81,7 @@ bucket_definitions: ]); expect(batchBefore.targetOp).toEqual(null); - await bucketStorage.compact({ + await compactActive(factory, { clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1, @@ -202,7 +202,7 @@ bucket_definitions: } ]); - await bucketStorage.compact({ + await compactActive(factory, { clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1, @@ -301,7 +301,7 @@ bucket_definitions: await writer2.flush(); const checkpoint2 = writer2.last_flushed_op!; - await bucketStorage.compact({ + await compactActive(factory, { clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1, @@ -416,7 +416,7 @@ bucket_definitions: const checkpoint = writer.last_flushed_op!; - await bucketStorage.compact({ + await compactActive(factory, { clearBatchLimit: 100, moveBatchLimit: 100, moveBatchQueryLimit: 100, // Larger limit for a larger window of operations @@ -507,7 +507,7 @@ bucket_definitions: await writer.commit('1/1'); await writer.flush(); - await bucketStorage.compact({ + await compactActive(factory, { clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1, @@ -595,7 +595,7 @@ bucket_definitions: await writer2.commit('2/1'); await writer2.flush(); - await bucketStorage.compact({ + await compactActive(factory, { clearBatchLimit: 20, moveBatchLimit: 10, moveBatchQueryLimit: 10, @@ -658,7 +658,7 @@ bucket_definitions: expect(checkpointBeforeCompact.checkpoint).toEqual(checkpoint1); // With default options, Postgres compaction should use the active checkpoint. - await bucketStorage.compact({ + await compactActive(factory, { moveBatchLimit: 1, moveBatchQueryLimit: 1, minBucketChanges: 1, diff --git a/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts index 93fc8a0d8..f7ba779b7 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts @@ -1,6 +1,7 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; +import { compactActive } from './util.js'; /** * @example @@ -399,11 +400,12 @@ bucket_definitions: checkpoint_requested_at: new Date('2024-01-01T00:00:00.000Z') }); await writer.flush(); + await writer.keepalive('1/1'); await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'persistent' })).resolves.toEqual(5n); await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'temporary' })).resolves.toEqual(6n); - await bucketStorage.compact({ + await compactActive(factory, { compactBuckets: [], deleteCheckpointRequestsBefore: new Date(Date.now() + 1_000) }); diff --git a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts index 2783260c9..ff67ada62 100644 --- a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts @@ -2,6 +2,7 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { ScopedParameterLookup } from '@powersync/service-sync-rules'; import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; +import { compactActive } from './util.js'; export function registerParameterCompactTests(config: storage.TestStorageConfig) { const generateStorageFactory = config.factory; @@ -74,7 +75,7 @@ bucket_definitions: expect(parameters2).toEqual([]); const statsBefore = await bucketStorage.factory.getStorageMetrics(); - await bucketStorage.compact({ compactParameterData: true }); + await compactActive(factory, { compactParameterData: true }); // Check consistency const parameters1b = await checkpoint1.getParameterSets([lookup], 1000); @@ -154,7 +155,7 @@ bucket_definitions: expect(parameters1).toEqual([]); const statsBefore = await bucketStorage.factory.getStorageMetrics(); - await bucketStorage.compact({ compactParameterData: true, compactParameterCacheLimit: cacheLimit }); + await compactActive(factory, { compactParameterData: true, compactParameterCacheLimit: cacheLimit }); // Check consistency const parameters1b = await checkpoint1.getParameterSets([lookup], 1000); diff --git a/packages/service-core-tests/src/tests/register-sync-tests.ts b/packages/service-core-tests/src/tests/register-sync-tests.ts index 2bb4c6c75..f3387f1b3 100644 --- a/packages/service-core-tests/src/tests/register-sync-tests.ts +++ b/packages/service-core-tests/src/tests/register-sync-tests.ts @@ -15,6 +15,7 @@ import { fileURLToPath } from 'url'; import { expect, test, vi } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; import { bucketRequest, METRICS_HELPER } from '../test-utils/test-utils-index.js'; +import { compactActive } from './util.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -1308,6 +1309,7 @@ bucket_definitions: const bucketStorage = await f.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + const bucket = bucketRequest(syncRules.syncConfigContent[0], 'mybucket[]').bucket; await writer.markAllSnapshotDone('0/1'); await writer.save({ @@ -1386,10 +1388,7 @@ bucket_definitions: await writer.commit('0/2'); - await bucketStorage.compact({ - minBucketChanges: 1, - minChangeRatio: 0 - }); + await compactActive(f, { compactBuckets: [bucket] }); const lines2 = await getCheckpointLines(iter, { consume: true }); @@ -1470,6 +1469,7 @@ bucket_definitions: const bucketStorage = await f.getInstance(syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + const highPriorityBucket = bucketRequest(syncRules.syncConfigContent[0], 'high_priority[]').bucket; await writer.markAllSnapshotDone('0/1'); await writer.save({ @@ -1552,9 +1552,10 @@ bucket_definitions: }); await writer.commit('0/2'); - await bucketStorage.compact({ - minBucketChanges: 1, - minChangeRatio: 0 + await compactActive(f, { + // Explicitly compact the high-priority bucket: V3 schedules background + // compaction, while this test needs compaction at this exact point. + compactBuckets: [highPriorityBucket] }); const lines = await getCheckpointLines(iter, { consume: true }); diff --git a/packages/service-core-tests/src/tests/util.ts b/packages/service-core-tests/src/tests/util.ts index 9e5c99e83..f70d19df3 100644 --- a/packages/service-core-tests/src/tests/util.ts +++ b/packages/service-core-tests/src/tests/util.ts @@ -8,6 +8,19 @@ import { } from '@powersync/service-sync-rules'; import { bucketRequest } from '../test-utils/general-utils.js'; +/** + * Resolve storage again after test writes activate the sync config. The storage + * instance used by the writer retains its original PROCESSING stream snapshot, + * while compact() expects an instance constructed with the current state. + */ +export async function compactActive(factory: storage.BucketStorageFactory, options: storage.CompactOptions) { + const active = await factory.getActiveSyncConfig(); + if (active == null) { + throw new Error('Expected an active sync config before compacting'); + } + await active.storage.compact(options); +} + export function bucketRequestMap( syncRules: storage.PersistedSyncConfigContent, buckets: Iterable diff --git a/packages/service-core/src/entry/commands/compact-action.ts b/packages/service-core/src/entry/commands/compact-action.ts index 38b040b59..78e82f781 100644 --- a/packages/service-core/src/entry/commands/compact-action.ts +++ b/packages/service-core/src/entry/commands/compact-action.ts @@ -28,7 +28,11 @@ export function registerCompactAction(program: Command) { .command(COMMAND_NAME) .option(`-b, --buckets [buckets]`, 'Full bucket names, comma-separated (e.g., "global[],mybucket[\\"user1\\"]")') .option('-p, --parameter-indexes', 'Compacting parameter indexes. Defaults to set unless --buckets is provided.') - .option('--no-parameter-indexes', 'Disabling compacting parameter indexes.'); + .option('--no-parameter-indexes', 'Disabling compacting parameter indexes.') + .option( + '--incremental-only', + '[EXPERIMENTAL] Perform incremental compacting only. Implies --no-parameter-indexes.' + ); wrapConfigCommand(compactCommand); @@ -48,7 +52,12 @@ export function registerCompactAction(program: Command) { } } + const incremental: boolean = options.incrementalOnly ?? false; + let compactParameters: boolean | null = options.parameterIndexes; + if (incremental) { + compactParameters = false; + } if (buckets == null) { logger.info('Compacting storage for all buckets...'); @@ -87,29 +96,30 @@ export function registerCompactAction(program: Command) { Date.now() - config.api_parameters.checkpoint_request_retention_minutes * MINUTE_MS ); - const active = (await bucketStorage.getActiveSyncConfig())?.storage; - if (active == null) { - logger.info('No active instance to compact'); - return; - } - if (buckets != null) { - logger.info('Performing compaction...'); - await active.compact({ - memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, - compactBuckets: buckets, - compactParameterData: compactParameters ?? false, - deleteCheckpointRequestsBefore, - signal: abortController.signal - }); - } else { - await active.compact({ - memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, - compactParameterData: compactParameters ?? true, - deleteCheckpointRequestsBefore, - signal: abortController.signal - }); + const streams = await bucketStorage.getReplicatingReplicationStreams(); + for (let stream of streams) { + const storage = bucketStorage.getInstance(stream); + logger.info(`[${stream.replicationStreamName}] Performing compaction...`); + if (buckets != null) { + await storage.compact({ + memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, + compactBuckets: buckets, + compactParameterData: compactParameters ?? false, + incrementalOnly: incremental, + deleteCheckpointRequestsBefore, + signal: abortController.signal + }); + } else { + await storage.compact({ + memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, + compactParameterData: compactParameters ?? true, + incrementalOnly: incremental, + deleteCheckpointRequestsBefore, + signal: abortController.signal + }); + } + logger.info(`[${stream.replicationStreamName}] Successfully compacted storage.`); } - logger.info('Successfully compacted storage.'); } catch (e) { logger.error(`Failed to compact:`, e); // Indirectly triggers lifeCycleEngine.stop diff --git a/packages/service-core/src/storage/SyncRulesBucketStorage.ts b/packages/service-core/src/storage/SyncRulesBucketStorage.ts index 8ecf5f4bf..e3087d173 100644 --- a/packages/service-core/src/storage/SyncRulesBucketStorage.ts +++ b/packages/service-core/src/storage/SyncRulesBucketStorage.ts @@ -100,9 +100,9 @@ export interface SyncRulesBucketStorage compact(options?: CompactOptions): Promise; /** - * Lightweight "compact" process to populate the checksum cache, if any. + * Compact storage after initial replication, before the first checkpoint exists. */ - populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise; + compactInitialReplication(options: CompactInitialReplicationOptions): Promise; // ## Read operations @@ -321,6 +321,15 @@ export interface CompactOptions { compactParameterData?: boolean; + /** + * Only perform compaction that can be done incrementally. + * + * This includes full bucket compaction on MongoDB V3 storage. + * + * On MongoDB v1 and Postgres storage, this makes compacting a no-op. + */ + incrementalOnly?: boolean; + /** * Delete client-requested write checkpoints created before this time. * @@ -354,6 +363,18 @@ export interface CompactOptions { */ minChangeRatio?: number; + /** Minimum delay before a V3 bucket is checked for chunk compaction. Default: five minutes. */ + minCompactChunkIntervalMs?: number; + + /** Minimum elapsed write pressure before the v3 sliding-scale full compact. Default: two hours. */ + minCompactFullIntervalMs?: number; + + /** Maximum age of writes not covered by a v3 full compact. */ + maxCompactFullIntervalMs?: number; + + /** How long a v3 worker owns a claimed bucket before another worker may recover it. */ + compactLeaseDurationMs?: number; + /** * Internal/testing use: Cache size for compacting parameters. */ @@ -364,12 +385,12 @@ export interface CompactOptions { logger?: Logger; } -export interface PopulateChecksumCacheOptions { +export interface CompactInitialReplicationOptions { /** - * Compute checksums up to this op id. + * Compact data up to this op id. * * Defaults to the highest persisted op id for the replication stream, which covers - * the common case of populating the cache right after initial replication, before + * the common case of compacting right after initial replication, before * the first checkpoint exists. */ maxOpId?: util.InternalOpId; @@ -377,9 +398,9 @@ export interface PopulateChecksumCacheOptions { signal?: AbortSignal; } -export interface PopulateChecksumCacheResults { +export interface CompactInitialReplicationResults { /** - * Number of buckets we have calculated checksums for. + * Number of buckets processed. */ buckets: number; } diff --git a/packages/service-core/src/util/utils.ts b/packages/service-core/src/util/utils.ts index 19c1afe1b..c0578dde2 100644 --- a/packages/service-core/src/util/utils.ts +++ b/packages/service-core/src/util/utils.ts @@ -309,3 +309,17 @@ export function estimateRowSize(record: sync_rules.ToastableSqliteRow | undefine } return size; } + +export function formatBytes(bytes: number | bigint): string { + if (typeof bytes == 'bigint') { + bytes = Number(bytes); // We round either way + } + if (bytes >= 1024 * 1024 * 1024) { + return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`; + } else if (bytes >= 1024 * 1024) { + return `${(bytes / (1024 * 1024)).toFixed(1)}MB`; + } else if (bytes >= 1024) { + return `${(bytes / 1024).toFixed(1)}KB`; + } + return `${bytes}B`; +} diff --git a/packages/service-core/test/src/util/utils.test.ts b/packages/service-core/test/src/util/utils.test.ts new file mode 100644 index 000000000..b4b47d4c4 --- /dev/null +++ b/packages/service-core/test/src/util/utils.test.ts @@ -0,0 +1,23 @@ +import { formatBytes } from '@/index.js'; +import { describe, expect, test } from 'vitest'; + +describe('formatBytes', () => { + test('formats bytes', () => { + expect(formatBytes(500)).toBe('500B'); + }); + + test('formats kilobytes', () => { + expect(formatBytes(1024)).toBe('1.0KB'); + expect(formatBytes(1536)).toBe('1.5KB'); + }); + + test('formats megabytes', () => { + expect(formatBytes(1024 * 1024)).toBe('1.0MB'); + expect(formatBytes(1.5 * 1024 * 1024)).toBe('1.5MB'); + }); + + test('formats gigabytes', () => { + expect(formatBytes(1024 * 1024 * 1024)).toBe('1.0GB'); + expect(formatBytes(10.5 * 1024 * 1024 * 1024)).toBe('10.5GB'); + }); +});