From 3d420abae0019fefd5ce01a253eb7410dbfdd96f Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Mon, 17 Aug 2026 14:15:30 -0700 Subject: [PATCH 1/9] feat(wallet): accelerate storage synchronization --- docs/packages/wallet/wallet-toolbox-client.md | 11 +- docs/packages/wallet/wallet-toolbox-mobile.md | 11 +- docs/packages/wallet/wallet-toolbox.md | 13 +- docs/reference/package-api-migrations.md | 18 +-- docs/reference/stack-facts.md | 6 +- governance/package-release-notes.json | 24 +-- governance/repository-health/baselines.json | 6 +- packages/wallet/wallet-toolbox/CHANGELOG.md | 11 ++ packages/wallet/wallet-toolbox/README.md | 6 +- .../wallet/wallet-toolbox/client/package.json | 2 +- .../client/platform-budget.json | 8 +- .../wallet/wallet-toolbox/mobile/package.json | 2 +- packages/wallet/wallet-toolbox/package.json | 2 +- .../src/sdk/WalletStorage.interfaces.ts | 27 ++++ .../wallet-toolbox/src/storage/StorageKnex.ts | 56 +++++++ .../src/storage/StorageReader.ts | 9 ++ .../src/storage/methods/getSyncChunk.test.ts | 137 ++++++++++++++++++ .../src/storage/methods/getSyncChunk.ts | 69 +++++++-- .../remoting/__test/StorageServerRpc.test.ts | 4 +- .../__tests__/entityValidationHelpers.test.ts | 25 ++++ .../src/storage/schema/KnexMigrations.ts | 16 ++ .../test/storage/KnexMigrations.test.ts | 51 ++++++- .../test/wallet/sync/Wallet.sync.test.ts | 23 ++- 23 files changed, 478 insertions(+), 59 deletions(-) create mode 100644 packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.test.ts diff --git a/docs/packages/wallet/wallet-toolbox-client.md b/docs/packages/wallet/wallet-toolbox-client.md index d77a65565..1dacb553a 100644 --- a/docs/packages/wallet/wallet-toolbox-client.md +++ b/docs/packages/wallet/wallet-toolbox-client.md @@ -3,9 +3,9 @@ id: pkg-wallet-toolbox-client title: '@bsv/wallet-toolbox-client' kind: package domain: wallet -version: '2.10.2' -last_updated: '2026-08-14' -last_verified: '2026-08-14' +version: '2.11.0' +last_updated: '2026-08-17' +last_verified: '2026-08-17' review_cadence_days: 30 npm: 'https://www.npmjs.com/package/@bsv/wallet-toolbox-client' repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client' @@ -18,6 +18,11 @@ tags: [wallet, browser, indexeddb, storage, brc-100] `@bsv/wallet-toolbox-client` is the browser-safe Wallet Toolbox distribution. It includes the BRC-100 wallet, signer, services, IndexedDB storage, and remote storage client without Node-only Knex, SQLite, MySQL, or filesystem adapters. + +Wallet replication uses adaptive, size-aware source reads. Browser clients can +set `includeTotals` on sync-chunk requests to obtain optional source record +totals for exact progress displays; older providers remain compatible and may +omit the totals. Related browser `noSend` chains retain local action batching, while unrelated actions cannot join or commit the active workspace. Supported remote providers can resume a soft-expired workspace using its exact persisted inputs. diff --git a/docs/packages/wallet/wallet-toolbox-mobile.md b/docs/packages/wallet/wallet-toolbox-mobile.md index 861bb427a..09a0b6757 100644 --- a/docs/packages/wallet/wallet-toolbox-mobile.md +++ b/docs/packages/wallet/wallet-toolbox-mobile.md @@ -3,9 +3,9 @@ id: pkg-wallet-toolbox-mobile title: '@bsv/wallet-toolbox-mobile' kind: package domain: wallet -version: '2.10.2' -last_updated: '2026-08-14' -last_verified: '2026-08-14' +version: '2.11.0' +last_updated: '2026-08-17' +last_verified: '2026-08-17' review_cadence_days: 30 npm: 'https://www.npmjs.com/package/@bsv/wallet-toolbox-mobile' repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile' @@ -18,6 +18,11 @@ tags: [wallet, react-native, mobile, storage, brc-100] `@bsv/wallet-toolbox-mobile` is the React Native and mobile-safe Wallet Toolbox distribution. It includes wallet, signer, services, monitoring, and remote storage surfaces without Knex, SQLite/MySQL, IndexedDB, or Node-only IO. + +Wallet replication uses adaptive, size-aware source reads. Mobile clients can +set `includeTotals` on sync-chunk requests to obtain optional source record +totals for exact progress displays; older providers remain compatible and may +omit the totals. Related mobile `noSend` chains retain local action batching, while unrelated actions cannot join or commit the active workspace. Supported remote providers can resume a soft-expired workspace using its exact persisted inputs. diff --git a/docs/packages/wallet/wallet-toolbox.md b/docs/packages/wallet/wallet-toolbox.md index c26021863..8d13429a2 100644 --- a/docs/packages/wallet/wallet-toolbox.md +++ b/docs/packages/wallet/wallet-toolbox.md @@ -4,9 +4,9 @@ title: '@bsv/wallet-toolbox' kind: package domain: wallet npm: '@bsv/wallet-toolbox' -version: '2.10.2' -last_updated: '2026-08-14' -last_verified: '2026-08-14' +version: '2.11.0' +last_updated: '2026-08-17' +last_verified: '2026-08-17' review_cadence_days: 30 status: stable tags: ['wallet', 'brc100'] @@ -56,6 +56,13 @@ Opt-in remote-storage timing spans retain trace and parent-span correlation in the telemetry sink without adding headers to authenticated requests. BRC-103, BRC-104, AuthFetch, and the storage RPC wire contract remain unchanged. +Wallet replication fills bounded pages with adaptive, size-aware reads. A +client can set `includeTotals` on `RequestSyncChunkArgs` to request optional +source record totals for exact progress and ETA displays. Providers that do not +implement efficient totals omit them, preserving compatibility without adding +count-query overhead. Knex storage creates the user-scoped proof lookup indexes +through its normal migration path. + UMP account lookup accepts one verified matching token as an existing account. When no token verifies, one clean empty overlay response establishes a new account even if other hosts fail or return malformed records. Multiple distinct diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index f3b093b6c..61726b71f 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -52,9 +52,9 @@ and clean-consumer tests remain the executable type authority. | `@bsv/verifast` | `0.3.0` | `0.3.4` | patch | [API and usage](../packages/sdk/verifast.md) | No consumer migration is required; exports, verification behavior, worker protocols, package paths, and runtime defaults are unchanged. | | `@bsv/wallet-helper` | `0.1.1` | `0.1.6` | patch | [API and usage](../packages/helpers/wallet-helper.md) | No consumer migration is required; fluent builder APIs and transaction semantics are unchanged. | | `@bsv/wallet-relay` | `0.2.2` | `0.3.5` | minor | [API and usage](../packages/wallet/wallet-relay.md) | No wallet RPC migration is required; upgrade to @bsv/sdk 2.4.1 or later. Existing relay sessions and number arrays remain valid, and host applications continue to provide their matching Express runtime and type graph. | -| `@bsv/wallet-toolbox` | `2.10.0` | `2.10.2` | patch | [API and usage](../packages/wallet/wallet-toolbox.md) | No consumer migration is required. Canonical AtomicBEEF and existing number-array behavior are unchanged; upgrade to @bsv/sdk 2.4.1 or later for cross-version JSON and wallet-error compatibility. | -| `@bsv/wallet-toolbox-client` | `2.10.0` | `2.10.2` | patch | [API and usage](../packages/wallet/wallet-toolbox-client.md) | No consumer migration is required. Browser exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.4.1 or later. | -| `@bsv/wallet-toolbox-mobile` | `2.10.0` | `2.10.2` | patch | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | No consumer migration is required. React Native exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.4.1 or later. | +| `@bsv/wallet-toolbox` | `2.10.2` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox.md) | No consumer migration is required. Existing sync requests and providers remain wire-compatible. Clients may optionally set includeTotals; capable providers return totals, older providers ignore the hint, and SQL storage creates the new indexes through its normal migration path. | +| `@bsv/wallet-toolbox-client` | `2.10.2` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-client.md) | No consumer migration is required. Existing browser sync calls remain valid; applications may optionally request and consume source totals, while older remote providers continue to work without them. | +| `@bsv/wallet-toolbox-mobile` | `2.10.2` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | No consumer migration is required. Existing mobile sync calls remain valid; applications may optionally request and consume source totals, while older remote providers continue to work without them. | | `create-bsv-app` | `1.0.2` | `1.1.0` | minor | [API and usage](../packages/helpers/create-bsv-app.md) | Existing mainnet and testnet scaffolds are unchanged. New TTN projects pass --network ttn or select TerraTestNet in the configurator. | `none` means the source manifest matches the recorded npm baseline. Any other @@ -477,8 +477,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox.md](../packages/wallet/wallet-toolbox.md) - Source: [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) -- Release note: Retains BRC-95 internalization compatibility, preserves typed AtomicBEEF, competing BEEF, and wallet review errors across portable and historical wallet JSON representations, and leaves opaque WAB response JSON unchanged. -- Migration: No consumer migration is required. Canonical AtomicBEEF and existing number-array behavior are unchanged; upgrade to @bsv/sdk 2.4.1 or later for cross-version JSON and wallet-error compatibility. +- Release note: Accelerates wallet-storage replication with adaptive size-aware source reads and user-scoped SQL proof indexes, and adds opt-in source record totals for exact progress reporting. +- Migration: No consumer migration is required. Existing sync requests and providers remain wire-compatible. Clients may optionally set includeTotals; capable providers return totals, older providers ignore the hint, and SQL storage creates the new indexes through its normal migration path. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | ---------------------------------------------------- | -------------------------- | @@ -491,8 +491,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-client.md](../packages/wallet/wallet-toolbox-client.md) - Source: [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) -- Release note: Carries the browser Wallet Toolbox internalization and BRC-100 JSON byte-boundary compatibility fixes, including portable wallet review errors and historical response recovery. -- Migration: No consumer migration is required. Browser exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.4.1 or later. +- Release note: Carries adaptive size-aware wallet sync reads and the backward-compatible includeTotals progress contract for browser and WebView wallets. +- Migration: No consumer migration is required. Existing browser sync calls remain valid; applications may optionally request and consume source totals, while older remote providers continue to work without them. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | @@ -503,8 +503,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-mobile.md](../packages/wallet/wallet-toolbox-mobile.md) - Source: [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) -- Release note: Carries the mobile Wallet Toolbox internalization and BRC-100 JSON byte-boundary compatibility fixes, including portable wallet review errors and historical response recovery. -- Migration: No consumer migration is required. React Native exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.4.1 or later. +- Release note: Carries adaptive size-aware wallet sync reads and the backward-compatible includeTotals progress contract for React Native wallets. +- Migration: No consumer migration is required. Existing mobile sync calls remain valid; applications may optionally request and consume source totals, while older remote providers continue to work without them. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | diff --git a/docs/reference/stack-facts.md b/docs/reference/stack-facts.md index aa3585130..c62718516 100644 --- a/docs/reference/stack-facts.md +++ b/docs/reference/stack-facts.md @@ -65,9 +65,9 @@ authorized release action. | wallet | `@bsv/btms` | `1.2.1` | node-library | node-cjs, node-esm | node | `>=22` | [packages/wallet/btms](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms) | | wallet | `@bsv/btms-permission-module` | `1.1.3` | node-library | node-esm | node | `>=22` | [packages/wallet/btms-permission-module](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms-permission-module) | | wallet | `@bsv/wallet-relay` | `0.3.5` | cli-library | browser-bundler, browser-esm, cli, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/ts-wallet-relay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/ts-wallet-relay) | -| wallet | `@bsv/wallet-toolbox` | `2.10.2` | node-library | node-cjs | node | `>=22` | [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) | -| wallet | `@bsv/wallet-toolbox-client` | `2.10.2` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) | -| wallet | `@bsv/wallet-toolbox-mobile` | `2.10.2` | react-native-library | react-native-metro | react-native | `>=22` | [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) | +| wallet | `@bsv/wallet-toolbox` | `2.11.0` | node-library | node-cjs | node | `>=22` | [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) | +| wallet | `@bsv/wallet-toolbox-client` | `2.11.0` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) | +| wallet | `@bsv/wallet-toolbox-mobile` | `2.11.0` | react-native-library | react-native-metro | react-native | `>=22` | [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) | ## Standalone infrastructure manifests diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index 393ed8dea..254b33136 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -194,24 +194,24 @@ }, { "name": "@bsv/wallet-toolbox", - "publishedVersion": "2.10.0", - "releaseType": "patch", - "summary": "Retains BRC-95 internalization compatibility, preserves typed AtomicBEEF, competing BEEF, and wallet review errors across portable and historical wallet JSON representations, and leaves opaque WAB response JSON unchanged.", - "migration": "No consumer migration is required. Canonical AtomicBEEF and existing number-array behavior are unchanged; upgrade to @bsv/sdk 2.4.1 or later for cross-version JSON and wallet-error compatibility." + "publishedVersion": "2.10.2", + "releaseType": "minor", + "summary": "Accelerates wallet-storage replication with adaptive size-aware source reads and user-scoped SQL proof indexes, and adds opt-in source record totals for exact progress reporting.", + "migration": "No consumer migration is required. Existing sync requests and providers remain wire-compatible. Clients may optionally set includeTotals; capable providers return totals, older providers ignore the hint, and SQL storage creates the new indexes through its normal migration path." }, { "name": "@bsv/wallet-toolbox-client", - "publishedVersion": "2.10.0", - "releaseType": "patch", - "summary": "Carries the browser Wallet Toolbox internalization and BRC-100 JSON byte-boundary compatibility fixes, including portable wallet review errors and historical response recovery.", - "migration": "No consumer migration is required. Browser exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.4.1 or later." + "publishedVersion": "2.10.2", + "releaseType": "minor", + "summary": "Carries adaptive size-aware wallet sync reads and the backward-compatible includeTotals progress contract for browser and WebView wallets.", + "migration": "No consumer migration is required. Existing browser sync calls remain valid; applications may optionally request and consume source totals, while older remote providers continue to work without them." }, { "name": "@bsv/wallet-toolbox-mobile", - "publishedVersion": "2.10.0", - "releaseType": "patch", - "summary": "Carries the mobile Wallet Toolbox internalization and BRC-100 JSON byte-boundary compatibility fixes, including portable wallet review errors and historical response recovery.", - "migration": "No consumer migration is required. React Native exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.4.1 or later." + "publishedVersion": "2.10.2", + "releaseType": "minor", + "summary": "Carries adaptive size-aware wallet sync reads and the backward-compatible includeTotals progress contract for React Native wallets.", + "migration": "No consumer migration is required. Existing mobile sync calls remain valid; applications may optionally request and consume source totals, while older remote providers continue to work without them." }, { "name": "create-bsv-app", diff --git a/governance/repository-health/baselines.json b/governance/repository-health/baselines.json index dedb65461..c0e7143b0 100644 --- a/governance/repository-health/baselines.json +++ b/governance/repository-health/baselines.json @@ -325,8 +325,8 @@ "@bsv/btms": "1.2.1", "@bsv/btms-permission-module": "1.1.3", "@bsv/wallet-relay": "0.3.5", - "@bsv/wallet-toolbox-client": "2.10.2", - "@bsv/wallet-toolbox-mobile": "2.10.2", - "@bsv/wallet-toolbox": "2.10.2" + "@bsv/wallet-toolbox-client": "2.11.0", + "@bsv/wallet-toolbox-mobile": "2.11.0", + "@bsv/wallet-toolbox": "2.11.0" } } diff --git a/packages/wallet/wallet-toolbox/CHANGELOG.md b/packages/wallet/wallet-toolbox/CHANGELOG.md index c1416b71c..93937717b 100644 --- a/packages/wallet/wallet-toolbox/CHANGELOG.md +++ b/packages/wallet/wallet-toolbox/CHANGELOG.md @@ -26,6 +26,17 @@ attention to changes that materially alter behavior or extend functionality. transaction startup/commit cycles, failed pages roll back without advancing the checkpoint, and abort cleanup preserves the original storage error. +- Fill wallet-storage sync pages with adaptive, size-aware source queries and + add composite SQL indexes for user-scoped proof lookups. Sync clients may + request optional source record totals for exact progress and ETA displays; + older clients and providers remain wire-compatible and do not incur count + queries unless totals are requested. The authoritative Linux Vite fixture is + 1,607,393 raw bytes and the local gzip fixture is 378,833 bytes; those + ceilings advance by 400 and 100 bytes to 1,607,400 and 378,900. The measured + authoritative Linux esbuild fixture is 1,252,871 raw and 345,202 gzip bytes; + those ceilings advance by 400 and 300 bytes to 1,252,900 and 345,300. Other + browser compressed and mobile ceilings remain unchanged. + - Make verified phone changes interruption-safe by staging the replacement key in WAB, publishing the UMP rotation, and then finalizing WAB. Authentication can recover an interrupted transition from the current or pending key and diff --git a/packages/wallet/wallet-toolbox/README.md b/packages/wallet/wallet-toolbox/README.md index 62ce42971..ca1515fd3 100644 --- a/packages/wallet/wallet-toolbox/README.md +++ b/packages/wallet/wallet-toolbox/README.md @@ -48,7 +48,11 @@ The toolbox publishes three npm packages from this repo: Wallet storage replication applies each received page and its durable sync checkpoint in one provider transaction. IndexedDB and Knex therefore avoid per-record transaction startup, and a failed page rolls back without advancing -the checkpoint. The sync wire format and persisted schemas are unchanged. +the checkpoint. Sources fill each bounded page with adaptive, size-aware reads, +and Knex storage adds user-scoped proof lookup indexes. Clients may set +`includeTotals` on a sync-chunk request to receive optional source record totals +for exact progress reporting. Older providers ignore the hint, and totals are +not counted unless requested. ### UMP account continuity and phone changes diff --git a/packages/wallet/wallet-toolbox/client/package.json b/packages/wallet/wallet-toolbox/client/package.json index e4463788f..1ca49d415 100644 --- a/packages/wallet/wallet-toolbox/client/package.json +++ b/packages/wallet/wallet-toolbox/client/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox-client", - "version": "2.10.2", + "version": "2.11.0", "type": "module", "sideEffects": false, "engines": { diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 5dd65a8f6..6d27cf99a 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -2,13 +2,13 @@ "profile": "browser", "maximumBytes": { "vite": { - "raw": 1607000, - "gzip": 378800, + "raw": 1607400, + "gzip": 378900, "brotli": 297000 }, "esbuild": { - "raw": 1252500, - "gzip": 345000, + "raw": 1252900, + "gzip": 345300, "brotli": 277300 } } diff --git a/packages/wallet/wallet-toolbox/mobile/package.json b/packages/wallet/wallet-toolbox/mobile/package.json index 08d60cc2e..eeb4fab1a 100644 --- a/packages/wallet/wallet-toolbox/mobile/package.json +++ b/packages/wallet/wallet-toolbox/mobile/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox-mobile", - "version": "2.10.2", + "version": "2.11.0", "type": "module", "sideEffects": false, "engines": { diff --git a/packages/wallet/wallet-toolbox/package.json b/packages/wallet/wallet-toolbox/package.json index 42e564efe..79ec07ffc 100644 --- a/packages/wallet/wallet-toolbox/package.json +++ b/packages/wallet/wallet-toolbox/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox", - "version": "2.10.2", + "version": "2.11.0", "sideEffects": false, "type": "commonjs", "engines": { diff --git a/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts b/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts index 1b085e578..2d4c0665c 100644 --- a/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts +++ b/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts @@ -564,6 +564,12 @@ export interface RequestSyncChunkArgs { * The maximum number of items (records) to be returned. */ maxItems: number + /** + * Include source-side record totals for this `since` window when the + * provider can calculate them efficiently. Older providers ignore this + * optional hint and remain wire-compatible. + */ + includeTotals?: boolean /** * For each entity in dependency order, the offset at which to start returning items * from `since`. @@ -585,6 +591,24 @@ export interface RequestSyncChunkArgs { offsets: Array<{ name: string, offset: number }> } +export interface SyncChunkTotals { + totalRecords: number + records: { + provenTxs: number + outputBaskets: number + outputTags: number + txLabels: number + transactions: number + outputs: number + txLabelMaps: number + outputTagMaps: number + certificates: number + certificateFields: number + commissions: number + provenTxReqs: number + } +} + /** * Result received from remote `WalletStorage` in response to a `RequestSyncChunkArgs` request. * @@ -597,6 +621,9 @@ export interface SyncChunk { toStorageIdentityKey: string userIdentityKey: string + /** Optional progress totals requested with `includeTotals`. */ + totals?: SyncChunkTotals + user?: TableUser provenTxs?: TableProvenTx[] provenTxReqs?: TableProvenTxReq[] diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts b/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts index cab11752d..d5dbdd75a 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts @@ -54,6 +54,8 @@ import { PurgeParams, PurgeResults, TrxToken, + RequestSyncChunkArgs, + SyncChunkTotals, WalletStorageProvider } from '../sdk/WalletStorage.interfaces' import { WERR_INTERNAL, WERR_INVALID_PARAMETER, WERR_NOT_IMPLEMENTED, WERR_UNAUTHORIZED } from '../sdk/WERR_errors' @@ -325,6 +327,60 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide return this.validateEntities(rs, undefined, ['isDeleted']) } + private async countSyncQuery (query: Knex.QueryBuilder): Promise { + const row = await query.count<{ count: string | number }>({ count: '*' }).first() + return Number(row?.count ?? 0) + } + + override async getSyncChunkTotals (args: RequestSyncChunkArgs, userId: number): Promise { + const since = args.since + const [ + provenTxs, + outputBaskets, + outputTags, + txLabels, + transactions, + outputs, + txLabelMaps, + outputTagMaps, + certificates, + certificateFields, + commissions, + provenTxReqs + ] = await Promise.all([ + this.countSyncQuery(this.getProvenTxsForUserQuery({ userId, since })), + this.countOutputBaskets({ partial: { userId }, since }), + this.countOutputTags({ partial: { userId }, since }), + this.countTxLabels({ partial: { userId }, since }), + this.countTransactions({ partial: { userId }, since, noRawTx: true }), + this.countOutputs({ partial: { userId }, since, noScript: true }), + this.countSyncQuery(this.getTxLabelMapsForUserQuery({ userId, since })), + this.countSyncQuery(this.getOutputTagMapsForUserQuery({ userId, since })), + this.countCertificates({ partial: { userId }, since }), + this.countCertificateFields({ partial: { userId }, since }), + this.countCommissions({ partial: { userId }, since }), + this.countSyncQuery(this.getProvenTxReqsForUserQuery({ userId, since })) + ]) + const records = { + provenTxs, + outputBaskets, + outputTags, + txLabels, + transactions, + outputs, + txLabelMaps, + outputTagMaps, + certificates, + certificateFields, + commissions, + provenTxReqs + } + return { + totalRecords: Object.values(records).reduce((total, count) => total + count, 0), + records + } + } + override async listActions (auth: AuthId, vargs: Validation.ValidListActionsArgs): Promise { if (auth.userId == null) throw new WERR_UNAUTHORIZED() return await listActions(this, auth, vargs) diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageReader.ts b/packages/wallet/wallet-toolbox/src/storage/StorageReader.ts index 6e5553342..f464e889e 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageReader.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageReader.ts @@ -105,6 +105,15 @@ export abstract class StorageReader implements sdk.WalletStorageSyncReader { abstract getTxLabelMapsForUser (args: sdk.FindForUserSincePagedArgs): Promise abstract getOutputTagMapsForUser (args: sdk.FindForUserSincePagedArgs): Promise + /** + * Optional efficient source-side totals for synchronization progress. + * Providers without a native count implementation omit the metadata rather + * than loading every matching record merely to count it. + */ + async getSyncChunkTotals (_args: sdk.RequestSyncChunkArgs, _userId: number): Promise { + return undefined + } + async findUserByIdentityKey (key: string): Promise { return verifyOneOrNone(await this.findUsers({ partial: { identityKey: key } })) } diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.test.ts b/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.test.ts new file mode 100644 index 000000000..bebc79a03 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.test.ts @@ -0,0 +1,137 @@ +import { RequestSyncChunkArgs } from '../../sdk/WalletStorage.interfaces' +import { StorageReader } from '../StorageReader' +import { getSyncChunk } from './getSyncChunk' + +const entityNames = [ + 'provenTx', + 'outputBasket', + 'outputTag', + 'txLabel', + 'transaction', + 'output', + 'txLabelMap', + 'outputTagMap', + 'certificate', + 'certificateField', + 'commission', + 'provenTxReq' +] + +function makeArgs(maxItems = 250, maxRoughSize = 2 * 1024 * 1024): RequestSyncChunkArgs { + return { + identityKey: '02'.repeat(33), + maxItems, + maxRoughSize, + offsets: entityNames.map(name => ({ name, offset: 0 })), + fromStorageIdentityKey: '11'.repeat(32), + toStorageIdentityKey: '22'.repeat(32) + } +} + +function makeStorage(provenTxCount: number, payloadBytes = 0) { + const now = new Date('2026-08-17T00:00:00.000Z') + const provenTxs = Array.from({ length: provenTxCount }, (_, index) => ({ + provenTxId: index + 1, + txid: index.toString(16).padStart(64, '0'), + created_at: now, + updated_at: now, + rawTx: Array.from({ length: payloadBytes }, () => 1) + })) + const getProvenTxsForUser = jest.fn(async ({ paged }: { paged?: { limit: number; offset?: number } }) => { + const offset = paged?.offset ?? 0 + return provenTxs.slice(offset, offset + (paged?.limit ?? provenTxs.length)) + }) + const empty = jest.fn(async () => []) + const storage = { + findUserByIdentityKey: jest.fn(async () => ({ + userId: 1, + identityKey: '02'.repeat(33), + created_at: now, + updated_at: now + })), + getProvenTxsForUser, + getSyncChunkTotals: jest.fn(async () => ({ + totalRecords: provenTxCount, + records: { + provenTxs: provenTxCount, + outputBaskets: 0, + outputTags: 0, + txLabels: 0, + transactions: 0, + outputs: 0, + txLabelMaps: 0, + outputTagMaps: 0, + certificates: 0, + certificateFields: 0, + commissions: 0, + provenTxReqs: 0 + } + })), + findOutputBaskets: empty, + findOutputTags: empty, + findTxLabels: empty, + findTransactions: empty, + findOutputs: empty, + getTxLabelMapsForUser: empty, + getOutputTagMapsForUser: empty, + findCertificates: empty, + findCertificateFields: empty, + findCommissions: empty, + getProvenTxReqsForUser: empty + } as unknown as StorageReader + return { storage, getProvenTxsForUser, getSyncChunkTotals: storage.getSyncChunkTotals as jest.Mock } +} + +describe('getSyncChunk query batching', () => { + test('fills a 250-record proven transaction page in a bounded number of queries', async () => { + const { storage, getProvenTxsForUser } = makeStorage(250) + + const chunk = await getSyncChunk(storage, makeArgs()) + + expect(chunk.provenTxs).toHaveLength(250) + expect(getProvenTxsForUser).toHaveBeenCalledTimes(3) + expect(getProvenTxsForUser.mock.calls.map(call => call[0].paged?.limit)).toEqual([10, 80, 160]) + }) + + test('uses observed record size to bound read-ahead for large records', async () => { + const { storage, getProvenTxsForUser } = makeStorage(250, 20_000) + + const chunk = await getSyncChunk(storage, makeArgs(250, 750_000)) + + expect(chunk.provenTxs!.length).toBeGreaterThan(10) + expect(chunk.provenTxs!.length).toBeLessThan(20) + expect(getProvenTxsForUser.mock.calls[1][0].paged?.limit).toBeLessThan(10) + }) + + test('includes efficient source totals only when requested', async () => { + const { storage, getSyncChunkTotals } = makeStorage(250) + const args = makeArgs() + args.includeTotals = true + + const chunk = await getSyncChunk(storage, args) + + expect(chunk.totals).toMatchObject({ totalRecords: 250, records: { provenTxs: 250 } }) + expect(getSyncChunkTotals).toHaveBeenCalledTimes(1) + }) + + test('does not add count-query overhead for legacy requests', async () => { + const { storage, getSyncChunkTotals } = makeStorage(25) + + const chunk = await getSyncChunk(storage, makeArgs()) + + expect(chunk.totals).toBeUndefined() + expect(getSyncChunkTotals).not.toHaveBeenCalled() + }) + + test('continues synchronization when optional totals cannot be counted', async () => { + const { storage, getSyncChunkTotals } = makeStorage(25) + getSyncChunkTotals.mockRejectedValueOnce(new Error('count unavailable')) + const args = makeArgs() + args.includeTotals = true + + const chunk = await getSyncChunk(storage, args) + + expect(chunk.provenTxs).toHaveLength(25) + expect(chunk.totals).toBeUndefined() + }) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.ts b/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.ts index e6a6a491e..cd507482d 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.ts @@ -15,6 +15,45 @@ import { FindForUserSincePagedArgs, RequestSyncChunkArgs, SyncChunk } from '../. import { verifyTruthy } from '../../utility/utilityHelpers' import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../sdk/WERR_errors' +const MIN_SYNC_QUERY_ITEMS = 10 +const MAX_SYNC_QUERY_ITEMS = 250 +const SYNC_QUERY_GROWTH_FACTOR = 8 + +function getNextSyncQueryLimit( + currentLimit: number, + itemsRemaining: number, + roughSizeRemaining: number, + batchRoughSize: number, + batchItemCount: number +): number { + const averageItemSize = Math.max(1, Math.ceil(batchRoughSize / batchItemCount)) + const estimatedItemsRemaining = Math.max(1, Math.floor(roughSizeRemaining / averageItemSize)) + return Math.min( + itemsRemaining, + MAX_SYNC_QUERY_ITEMS, + currentLimit * SYNC_QUERY_GROWTH_FACTOR, + estimatedItemsRemaining + ) +} + +function appendSyncItems(chunker: ChunkerArgs, items: any[], offset: number, itemCount: number, roughSize: number) { + let batchRoughSize = 0 + let done = false + for (const item of items) { + offset++ + chunker.addItem(item) + itemCount-- + const itemRoughSize = JSON.stringify(item).length + batchRoughSize += itemRoughSize + roughSize -= itemRoughSize + if (itemCount <= 0 || roughSize < 0) { + done = true + break + } + } + return { offset, itemCount, roughSize, batchRoughSize, done } +} + /** * Gets the next sync chunk of updated data from un-remoted storage (could be using a remote DB connection). * @param storage @@ -35,6 +74,10 @@ export async function getSyncChunk(storage: StorageReader, args: RequestSyncChun const user = verifyTruthy(await storage.findUserByIdentityKey(args.identityKey)) if (args.since == null || user.updated_at > new Date(args.since)) r.user = user + const totalsPromise = + args.includeTotals === true + ? storage.getSyncChunkTotals(args, user.userId).catch(() => undefined) + : Promise.resolve(undefined) const chunkers: ChunkerArgs[] = [ { @@ -237,8 +280,8 @@ export async function getSyncChunk(storage: StorageReader, args: RequestSyncChun throw new WERR_INVALID_PARAMETER('offsets', `in dependency order. '${a.name}' expected, found ${oname}.`) } let preAddCalled = false + let limit = Math.min(itemCount, Math.max(MIN_SYNC_QUERY_ITEMS, Math.ceil(args.maxItems / a.maxDivider))) while (!done) { - const limit = Math.min(itemCount, Math.max(10, args.maxItems / a.maxDivider)) if (limit <= 0) break const items = await a.findItems(storage, { userId: user.userId, @@ -251,16 +294,18 @@ export async function getSyncChunk(storage: StorageReader, args: RequestSyncChun preAddCalled = true } if (items.length === 0) break - for (const item of items) { - offset++ - a.addItem(item) - itemCount-- - roughSize -= JSON.stringify(item).length - if (itemCount <= 0 || roughSize < 0) { - done = true - break - } - } + const appended = appendSyncItems(a, items, offset, itemCount, roughSize) + offset = appended.offset + itemCount = appended.itemCount + roughSize = appended.roughSize + done = appended.done + if (done || items.length < limit) break + + // The small first query protects providers from loading hundreds of + // unexpectedly large binary records. Once their actual encoded size is + // known, grow the next query to fill the remaining page in a bounded + // number of database round trips. + limit = getNextSyncQueryLimit(limit, itemCount, roughSize, appended.batchRoughSize, items.length) } } @@ -270,6 +315,8 @@ export async function getSyncChunk(storage: StorageReader, args: RequestSyncChun } } + r.totals = await totalsPromise + return r } diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts index 7100d43c4..cc28dc6d7 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts @@ -375,9 +375,9 @@ describe('StorageServer JSON-RPC boundary', () => { invoke(server, 'enforceRpcRequestBudgets', 'listActions', [{}, { limit: Number.MAX_SAFE_INTEGER + 1 }]) ).rejects.toThrow('positive safe integers') - const syncParams: any[] = [{ maxRoughSize: 'unbounded' }] + const syncParams: any[] = [{ maxRoughSize: 'unbounded', includeTotals: true }] await invoke(server, 'enforceRpcRequestBudgets', 'getSyncChunk', syncParams) - expect(syncParams[0]).toEqual({ maxItems: 5, maxRoughSize: 128 }) + expect(syncParams[0]).toEqual({ maxItems: 5, maxRoughSize: 128, includeTotals: true }) const oversizedSyncParams: any[] = [{ maxItems: 4, maxRoughSize: 129 }] await invoke(server, 'enforceRpcRequestBudgets', 'getSyncChunk', oversizedSyncParams) diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__tests__/entityValidationHelpers.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__tests__/entityValidationHelpers.test.ts index f427727e1..971be630f 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/__tests__/entityValidationHelpers.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__tests__/entityValidationHelpers.test.ts @@ -238,6 +238,31 @@ describe('entityValidationHelpers', () => { expect(result.provenTxs).toBeUndefined() }) + test('preserves optional progress totals for remoted chunks', () => { + const chunk: SyncChunk = { + ...baseChunk(), + totals: { + totalRecords: 12, + records: { + provenTxs: 1, + outputBaskets: 1, + outputTags: 1, + txLabels: 1, + transactions: 1, + outputs: 1, + txLabelMaps: 1, + outputTagMaps: 1, + certificates: 1, + certificateFields: 1, + commissions: 1, + provenTxReqs: 1 + } + } + } + + expect(validateSyncChunkEntities(chunk).totals).toEqual(chunk.totals) + }) + test('validates the user entity when present', () => { const chunk: SyncChunk = { ...baseChunk(), diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts b/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts index 5a4ef9e79..896bcff97 100644 --- a/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts +++ b/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts @@ -16,6 +16,7 @@ export const MONITOR_CREATED_AT_INDEX_MIGRATION = '2026-07-14-002 add monitor cr export const CREATE_ACTION_FUNDING_INDEX_MIGRATION = '2026-08-02-001 add createAction funding selection index' export const PAYMENT_REPLAY_MIGRATION = '2026-08-04-001 add payment replay claims' export const MANAGED_CHANGE_POLICY_MIGRATION = '2026-08-10-001 upgrade managed change liquidity defaults' +export const WALLET_SYNC_SOURCE_INDEX_MIGRATION = '2026-08-17-001 add wallet sync source indexes' interface Migration { up: (knex: Knex) => Promise @@ -177,6 +178,21 @@ export class KnexMigrations implements MigrationSource { } } + migrations[WALLET_SYNC_SOURCE_INDEX_MIGRATION] = { + async up(knex) { + await knex.schema.alterTable('transactions', table => { + table.index(['userId', 'provenTxId'], 'idx_transactions_user_proven_tx') + table.index(['userId', 'txid'], 'idx_transactions_user_txid') + }) + }, + async down(knex) { + await knex.schema.alterTable('transactions', table => { + table.dropIndex(['userId', 'provenTxId'], 'idx_transactions_user_proven_tx') + table.dropIndex(['userId', 'txid'], 'idx_transactions_user_txid') + }) + } + } + migrations['2026-07-15-001 add action batch reservations and blobs'] = { async up(knex) { const dbtype = await determineDBType(knex) diff --git a/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts b/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts index aa7aa7632..b376bee72 100644 --- a/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts +++ b/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts @@ -6,6 +6,7 @@ import { MANAGED_CHANGE_POLICY_MIGRATION, MONITOR_CREATED_AT_INDEX_MIGRATION, StorageKnex, + WALLET_SYNC_SOURCE_INDEX_MIGRATION, wait } from '../../src/index.all' import { Knex } from 'knex' @@ -206,7 +207,55 @@ describe('KnexMigrations tests', () => { } }) - test('5a upgrades only exact untouched managed-change defaults', async () => { + test('5a creates and uses the wallet sync source indexes', async () => { + const localSQLiteFile = await _tu.newTmpFile('migratesyncindexes.sqlite', false, false, false) + const knex = _tu.createLocalSQLite(localSQLiteFile) + + try { + await knex.schema.createTable('proven_txs', table => { + table.increments('provenTxId') + }) + await knex.schema.createTable('transactions', table => { + table.increments('transactionId') + table.integer('userId').notNullable() + table.integer('provenTxId').nullable() + table.string('txid', 64).nullable() + }) + const source = new KnexMigrations('test', 'wallet sync index test', '1'.repeat(64), 1000) + const migration = await source.getMigration(WALLET_SYNC_SOURCE_INDEX_MIGRATION) + await migration.up(knex) + + const indexes = await knex('sqlite_master') + .where({ type: 'index' }) + .whereIn('name', ['idx_transactions_user_proven_tx', 'idx_transactions_user_txid']) + .pluck('name') + expect(indexes.sort()).toEqual(['idx_transactions_user_proven_tx', 'idx_transactions_user_txid']) + + const provenPlan = await knex.raw( + 'EXPLAIN QUERY PLAN SELECT * FROM proven_txs WHERE EXISTS (' + + 'SELECT * FROM transactions WHERE proven_txs.provenTxId = transactions.provenTxId AND transactions.userId = ?)', + [1] + ) as Array<{ detail: string }> + expect(provenPlan.some(step => step.detail.includes('idx_transactions_user_proven_tx'))).toBe(true) + + const requestPlan = await knex.raw( + 'EXPLAIN QUERY PLAN SELECT * FROM transactions WHERE userId = ? AND txid = ?', + [1, '00'.repeat(32)] + ) as Array<{ detail: string }> + expect(requestPlan.some(step => step.detail.includes('idx_transactions_user_txid'))).toBe(true) + + await migration.down?.(knex) + const indexesAfterDown = await knex('sqlite_master') + .where({ type: 'index' }) + .whereIn('name', ['idx_transactions_user_proven_tx', 'idx_transactions_user_txid']) + .pluck('name') + expect(indexesAfterDown).toEqual([]) + } finally { + await knex.destroy() + } + }) + + test('5b upgrades only exact untouched managed-change defaults', async () => { const localSQLiteFile = await _tu.newTmpFile('migratemanagedchange.sqlite', false, false, false) const knex = _tu.createLocalSQLite(localSQLiteFile) diff --git a/packages/wallet/wallet-toolbox/test/wallet/sync/Wallet.sync.test.ts b/packages/wallet/wallet-toolbox/test/wallet/sync/Wallet.sync.test.ts index 4c0c8699c..3d796da96 100644 --- a/packages/wallet/wallet-toolbox/test/wallet/sync/Wallet.sync.test.ts +++ b/packages/wallet/wallet-toolbox/test/wallet/sync/Wallet.sync.test.ts @@ -1,4 +1,12 @@ -import { verifyOne, verifyOneOrNone, verifyTruthy, wait, Wallet, WalletStorageManager } from '../../../src/index.client' +import { + EntitySyncState, + verifyOne, + verifyOneOrNone, + verifyTruthy, + wait, + Wallet, + WalletStorageManager +} from '../../../src/index.client' import { StorageKnex } from '../../../src/storage/StorageKnex' import { _tu, logger, TestWalletNoSetup } from '../../utils/TestUtilsWalletStorage' @@ -32,6 +40,19 @@ describe('Wallet sync tests', () => { const manager = new WalletStorageManager(identityKey, storage, [tmpStore]) const auth = await manager.getAuth() + const destinationSyncState = await EntitySyncState.fromStorage(tmpStore, identityKey, _srcSettings) + const totalsArgs = destinationSyncState.makeRequestSyncChunkArgs( + identityKey, + tmpStore.getSettings().storageIdentityKey, + 2 * 1024 * 1024, + 250 + ) + totalsArgs.includeTotals = true + const firstChunk = await storage.getSyncChunk(totalsArgs) + expect(firstChunk.totals?.totalRecords).toBeGreaterThan(1000) + expect(firstChunk.totals?.totalRecords).toBe( + Object.values(firstChunk.totals?.records ?? {}).reduce((total, count) => total + count, 0) + ) { const r = await manager.syncToWriter(auth, tmpStore) expect(r.inserts).toBeGreaterThan(1000) From bc57cac4ca9671fe68278a4ce3230de9e5a1b92c Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Tue, 18 Aug 2026 15:55:13 -0700 Subject: [PATCH 2/9] fix(wallet): disambiguate storage sync checkpoints --- docs/packages/wallet/wallet-toolbox-client.md | 4 +- docs/packages/wallet/wallet-toolbox-mobile.md | 4 +- docs/packages/wallet/wallet-toolbox.md | 7 +- docs/reference/package-api-migrations.md | 12 +-- packages/wallet/wallet-toolbox/README.md | 5 +- .../src/sdk/WalletStorage.interfaces.ts | 6 ++ .../src/storage/StorageProvider.ts | 6 +- .../src/storage/StorageReaderWriter.ts | 27 +++++- .../src/storage/__test/StorageIdb.test.ts | 90 +++++++++++++++++++ .../schema/entities/EntitySyncState.ts | 1 + .../test/wallet/sync/Wallet.sync.test.ts | 26 ++++++ 11 files changed, 174 insertions(+), 14 deletions(-) diff --git a/docs/packages/wallet/wallet-toolbox-client.md b/docs/packages/wallet/wallet-toolbox-client.md index 1dacb553a..2e48ff44e 100644 --- a/docs/packages/wallet/wallet-toolbox-client.md +++ b/docs/packages/wallet/wallet-toolbox-client.md @@ -22,7 +22,9 @@ storage client without Node-only Knex, SQLite, MySQL, or filesystem adapters. Wallet replication uses adaptive, size-aware source reads. Browser clients can set `includeTotals` on sync-chunk requests to obtain optional source record totals for exact progress displays; older providers remain compatible and may -omit the totals. +omit the totals. New clients also carry the registered writer-local sync-state +identifier to disambiguate legacy duplicate checkpoints on upgraded providers; +older protocol peers remain compatible. Related browser `noSend` chains retain local action batching, while unrelated actions cannot join or commit the active workspace. Supported remote providers can resume a soft-expired workspace using its exact persisted inputs. diff --git a/docs/packages/wallet/wallet-toolbox-mobile.md b/docs/packages/wallet/wallet-toolbox-mobile.md index 09a0b6757..70e27413c 100644 --- a/docs/packages/wallet/wallet-toolbox-mobile.md +++ b/docs/packages/wallet/wallet-toolbox-mobile.md @@ -22,7 +22,9 @@ remote storage surfaces without Knex, SQLite/MySQL, IndexedDB, or Node-only IO. Wallet replication uses adaptive, size-aware source reads. Mobile clients can set `includeTotals` on sync-chunk requests to obtain optional source record totals for exact progress displays; older providers remain compatible and may -omit the totals. +omit the totals. New clients also carry the registered writer-local sync-state +identifier to disambiguate legacy duplicate checkpoints on upgraded providers; +older protocol peers remain compatible. Related mobile `noSend` chains retain local action batching, while unrelated actions cannot join or commit the active workspace. Supported remote providers can resume a soft-expired workspace using its exact persisted inputs. diff --git a/docs/packages/wallet/wallet-toolbox.md b/docs/packages/wallet/wallet-toolbox.md index 8d13429a2..afad09c29 100644 --- a/docs/packages/wallet/wallet-toolbox.md +++ b/docs/packages/wallet/wallet-toolbox.md @@ -60,8 +60,11 @@ Wallet replication fills bounded pages with adaptive, size-aware reads. A client can set `includeTotals` on `RequestSyncChunkArgs` to request optional source record totals for exact progress and ETA displays. Providers that do not implement efficient totals omit them, preserving compatibility without adding -count-query overhead. Knex storage creates the user-scoped proof lookup indexes -through its normal migration path. +count-query overhead. Clients also carry the writer-local sync-state identifier +selected at provider registration so upgraded providers can safely address a +legacy duplicate checkpoint. Older clients and providers remain supported. +Knex storage creates the user-scoped proof lookup indexes through its normal +migration path. UMP account lookup accepts one verified matching token as an existing account. When no token verifies, one clean empty overlay response establishes a new diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index 61726b71f..ac5db67d6 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -477,8 +477,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox.md](../packages/wallet/wallet-toolbox.md) - Source: [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) -- Release note: Accelerates wallet-storage replication with adaptive size-aware source reads and user-scoped SQL proof indexes, and adds opt-in source record totals for exact progress reporting. -- Migration: No consumer migration is required. Existing sync requests and providers remain wire-compatible. Clients may optionally set includeTotals; capable providers return totals, older providers ignore the hint, and SQL storage creates the new indexes through its normal migration path. +- Release note: Accelerates wallet-storage replication with adaptive size-aware source reads and user-scoped SQL proof indexes, adds opt-in source record totals for exact progress reporting, and disambiguates legacy duplicate sync checkpoints. +- Migration: No consumer migration is required. Existing sync requests and providers remain wire-compatible. Clients may optionally set includeTotals; capable providers return totals, older providers ignore the hint, and upgraded clients automatically carry a writer-local sync-state identifier that upgraded providers use when legacy checkpoints collide. SQL storage creates the new indexes through its normal migration path. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | ---------------------------------------------------- | -------------------------- | @@ -491,8 +491,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-client.md](../packages/wallet/wallet-toolbox-client.md) - Source: [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) -- Release note: Carries adaptive size-aware wallet sync reads and the backward-compatible includeTotals progress contract for browser and WebView wallets. -- Migration: No consumer migration is required. Existing browser sync calls remain valid; applications may optionally request and consume source totals, while older remote providers continue to work without them. +- Release note: Carries adaptive size-aware wallet sync reads, the backward-compatible includeTotals progress contract, and duplicate-checkpoint disambiguation for browser and WebView wallets. +- Migration: No consumer migration is required. Existing browser sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, and older remote providers continue to work without either addition. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | @@ -503,8 +503,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-mobile.md](../packages/wallet/wallet-toolbox-mobile.md) - Source: [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) -- Release note: Carries adaptive size-aware wallet sync reads and the backward-compatible includeTotals progress contract for React Native wallets. -- Migration: No consumer migration is required. Existing mobile sync calls remain valid; applications may optionally request and consume source totals, while older remote providers continue to work without them. +- Release note: Carries adaptive size-aware wallet sync reads, the backward-compatible includeTotals progress contract, and duplicate-checkpoint disambiguation for React Native wallets. +- Migration: No consumer migration is required. Existing mobile sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, and older remote providers continue to work without either addition. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | diff --git a/packages/wallet/wallet-toolbox/README.md b/packages/wallet/wallet-toolbox/README.md index ca1515fd3..92546590f 100644 --- a/packages/wallet/wallet-toolbox/README.md +++ b/packages/wallet/wallet-toolbox/README.md @@ -52,7 +52,10 @@ the checkpoint. Sources fill each bounded page with adaptive, size-aware reads, and Knex storage adds user-scoped proof lookup indexes. Clients may set `includeTotals` on a sync-chunk request to receive optional source record totals for exact progress reporting. Older providers ignore the hint, and totals are -not counted unless requested. +not counted unless requested. New clients also send the writer-local sync-state +identifier selected during provider registration. New providers use it to +disambiguate legacy duplicate checkpoints, while either side remains compatible +with older protocol peers. ### UMP account continuity and phone changes diff --git a/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts b/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts index 2d4c0665c..d8207fec3 100644 --- a/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts +++ b/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts @@ -535,6 +535,12 @@ export type SyncStatus = 'success' | 'error' | 'identified' | 'updated' | 'unkno export type SyncProtocolVersion = '0.1.0' export interface RequestSyncChunkArgs { + /** + * The writer-local sync state selected when the source provider was + * registered. New clients include this to disambiguate legacy databases + * that contain multiple rows for a reused storage identity key. + */ + syncStateId?: number /** * The storageIdentityKey of the storage supplying the update SyncChunk data. */ diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts b/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts index 7be9cd45c..cac0f5259 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts @@ -1174,7 +1174,11 @@ export abstract class StorageProvider extends StorageReaderWriter implements Wal const ss = new EntitySyncState( verifyOne( await this.findSyncStates({ - partial: { + partial: args.syncStateId == null ? { + storageIdentityKey: args.fromStorageIdentityKey, + userId: user.userId + } : { + syncStateId: args.syncStateId, storageIdentityKey: args.fromStorageIdentityKey, userId: user.userId }, diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts b/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts index f780351a1..eeaa38a86 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts @@ -32,6 +32,7 @@ import { DEFAULT_MANAGED_CHANGE_MINIMUM_SATOSHIS, DEFAULT_MANAGED_CHANGE_TARGET_UTXOS } from './methods/managedChangePolicy' +import { WERR_INVALID_OPERATION } from '../sdk/WERR_errors' export abstract class StorageReaderWriter extends StorageReader { abstract dropAllData (): Promise @@ -361,14 +362,31 @@ export abstract class StorageReaderWriter extends StorageReader { storageIdentityKey: string, storageName: string ): Promise<{ syncState: TableSyncState, isNew: boolean }> { - const partial = { userId: auth.userId as number, storageIdentityKey, storageName } + const partial = { userId: auth.userId as number, storageIdentityKey } for (let retry = 0; ; retry++) { try { const now = new Date() - let syncState = verifyOneOrNone(await this.findSyncStates({ partial })) + const matches = await this.findSyncStates({ partial }) + let syncState: TableSyncState | undefined + if (matches.length === 1) { + syncState = matches[0] + } else if (matches.length > 1) { + // Older releases included storageName in the lookup and could create + // duplicate rows when a provider was renamed or two apps reused a + // provider identity. Preserve exact-name access so upgraded clients + // can identify and repair those rows without guessing a checkpoint. + const exactMatches = matches.filter(s => s.storageName === storageName) + if (exactMatches.length !== 1) { + throw new WERR_INVALID_OPERATION( + 'Storage identity has conflicting sync states. Use a unique identity for each storage provider.' + ) + } + syncState = exactMatches[0] + } if (syncState == null) { syncState = { ...partial, + storageName, created_at: now, updated_at: now, syncStateId: 0, @@ -380,6 +398,11 @@ export abstract class StorageReaderWriter extends StorageReader { await this.insertSyncState(syncState) return { syncState, isNew: true } } + if (syncState.storageName !== storageName) { + syncState.storageName = storageName + syncState.updated_at = now + await this.updateSyncState(syncState.syncStateId, { storageName, updated_at: now }) + } return { syncState, isNew: false } } catch (error_: unknown) { if (retry > 0) throw error_ diff --git a/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts b/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts index b48a7adff..f3475d3d7 100644 --- a/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts @@ -251,6 +251,96 @@ describe('StorageIdb tests', () => { } }) + test('uses the registered sync state id when legacy provider identities collide', async () => { + const storage = await makeStorage() + try { + const identityKey = '06'.repeat(33) + const userId = await insertUser(storage, identityKey) + const now = new Date() + const sourceIdentityKey = 'shared-local-storage-identity' + const legacy: TableSyncState = { + syncStateId: 0, + userId, + storageIdentityKey: sourceIdentityKey, + storageName: 'metanet-wallet', + status: 'unknown', + init: false, + refNum: 'legacy-colliding-sync-state', + syncMap: JSON.stringify(createSyncMap()), + created_at: now, + updated_at: now + } + const selected: TableSyncState = { + ...legacy, + syncStateId: 0, + storageName: 'BSV Desktop Wallet', + refNum: 'selected-colliding-sync-state' + } + await storage.insertSyncState(legacy) + await storage.insertSyncState(selected) + + const registered = await storage.findOrInsertSyncStateAuth( + { identityKey, userId }, + sourceIdentityKey, + selected.storageName + ) + expect(registered.isNew).toBe(false) + expect(registered.syncState.syncStateId).toBe(selected.syncStateId) + await expect(storage.findOrInsertSyncStateAuth( + { identityKey, userId }, + sourceIdentityKey, + 'Unrecognized local provider' + )).rejects.toThrow('Storage identity has conflicting sync states') + + const args = { + syncStateId: registered.syncState.syncStateId, + identityKey, + maxRoughSize: 1000, + maxItems: 1000, + offsets: [], + since: undefined, + fromStorageIdentityKey: sourceIdentityKey, + toStorageIdentityKey: 'remote-storage' + } + const chunk = { + fromStorageIdentityKey: sourceIdentityKey, + toStorageIdentityKey: 'remote-storage', + userIdentityKey: identityKey + } + + await expect(storage.processSyncChunk(args, chunk)).resolves.toMatchObject({ + inserts: 0, + updates: 0 + }) + await expect(storage.processSyncChunk({ ...args, syncStateId: undefined }, chunk)) + .rejects.toThrow('Result must exist and be unique') + } finally { + await resetStorage(storage) + } + }) + + test('renames a registered storage provider without duplicating its sync state', async () => { + const storage = await makeStorage() + try { + const identityKey = '07'.repeat(33) + const userId = await insertUser(storage, identityKey) + const auth = { identityKey, userId } + const sourceIdentityKey = 'stable-storage-identity' + + const first = await storage.findOrInsertSyncStateAuth(auth, sourceIdentityKey, 'Old local name') + const renamed = await storage.findOrInsertSyncStateAuth(auth, sourceIdentityKey, 'New local name') + + expect(first.isNew).toBe(true) + expect(renamed.isNew).toBe(false) + expect(renamed.syncState.syncStateId).toBe(first.syncState.syncStateId) + expect(renamed.syncState.storageName).toBe('New local name') + await expect(storage.findSyncStates({ partial: { userId, storageIdentityKey: sourceIdentityKey } })) + .resolves.toHaveLength(1) + } finally { + await resetStorage(storage) + } + }) + test('preserves the original error when aborting an IndexedDB transaction', async () => { const storage = await makeStorage() try { diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntitySyncState.ts b/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntitySyncState.ts index 3ec500533..a7a899399 100644 --- a/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntitySyncState.ts +++ b/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntitySyncState.ts @@ -306,6 +306,7 @@ export class EntitySyncState extends EntityBase { maxItems?: number ): RequestSyncChunkArgs { const a: RequestSyncChunkArgs = { + syncStateId: this.id, identityKey: forIdentityKey, maxRoughSize: maxRoughSize || 10000000, maxItems: maxItems || 1000, diff --git a/packages/wallet/wallet-toolbox/test/wallet/sync/Wallet.sync.test.ts b/packages/wallet/wallet-toolbox/test/wallet/sync/Wallet.sync.test.ts index 3d796da96..17df5cdcb 100644 --- a/packages/wallet/wallet-toolbox/test/wallet/sync/Wallet.sync.test.ts +++ b/packages/wallet/wallet-toolbox/test/wallet/sync/Wallet.sync.test.ts @@ -108,6 +108,32 @@ describe('Wallet sync tests', () => { await ctx.storage.destroy() }) + + test('1c keeps the original active while adding a fresh local backup', async () => { + const ctx = await _tu.createLegacyWalletSQLiteCopy('walletSyncTest1cSource') + const localSQLiteFile = await _tu.newTmpFile('walletSyncTest1cLocal.sqlite', true, false, false) + const localStorage = new StorageKnex({ + ...StorageKnex.defaultOptions(), + chain: env.chain, + knex: _tu.createLocalSQLite(localSQLiteFile) + }) + + try { + const localStorageIdentityKey = `02${'08'.repeat(32)}` + await localStorage.migrate('BSV Desktop Wallet', localStorageIdentityKey) + await localStorage.makeAvailable() + + const originalStorageIdentityKey = ctx.activeStorage.getSettings().storageIdentityKey + await ctx.storage.addWalletStorageProvider(localStorage) + await ctx.storage.setActive(originalStorageIdentityKey) + + expect(ctx.storage.getActiveStore()).toBe(originalStorageIdentityKey) + expect(ctx.storage.isActiveEnabled).toBe(true) + expect(ctx.storage.getBackupStores()).toEqual([localStorageIdentityKey]) + } finally { + await ctx.storage.destroy() + } + }) }) async function setActiveTwice( From 246d7fc88d22137bd9af95b833113355bfbf0f7d Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Tue, 18 Aug 2026 16:00:58 -0700 Subject: [PATCH 3/9] refactor(wallet): simplify sync state selection --- .../src/storage/StorageReaderWriter.ts | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts b/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts index eeaa38a86..ad771bf67 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts @@ -34,6 +34,24 @@ import { } from './methods/managedChangePolicy' import { WERR_INVALID_OPERATION } from '../sdk/WERR_errors' +function selectSyncStateForRegistration ( + matches: TableSyncState[], + storageName: string +): TableSyncState | undefined { + if (matches.length <= 1) return matches[0] + + // Older releases included storageName in the lookup and could create + // duplicate rows when a provider was renamed or two apps reused a provider + // identity. Preserve exact-name access so upgraded clients can identify and + // repair those rows without guessing a checkpoint. + const exactMatches = matches.filter(s => s.storageName === storageName) + if (exactMatches.length === 1) return exactMatches[0] + + throw new WERR_INVALID_OPERATION( + 'Storage identity has conflicting sync states. Use a unique identity for each storage provider.' + ) +} + export abstract class StorageReaderWriter extends StorageReader { abstract dropAllData (): Promise abstract migrate (storageName: string, storageIdentityKey: string): Promise @@ -367,22 +385,7 @@ export abstract class StorageReaderWriter extends StorageReader { try { const now = new Date() const matches = await this.findSyncStates({ partial }) - let syncState: TableSyncState | undefined - if (matches.length === 1) { - syncState = matches[0] - } else if (matches.length > 1) { - // Older releases included storageName in the lookup and could create - // duplicate rows when a provider was renamed or two apps reused a - // provider identity. Preserve exact-name access so upgraded clients - // can identify and repair those rows without guessing a checkpoint. - const exactMatches = matches.filter(s => s.storageName === storageName) - if (exactMatches.length !== 1) { - throw new WERR_INVALID_OPERATION( - 'Storage identity has conflicting sync states. Use a unique identity for each storage provider.' - ) - } - syncState = exactMatches[0] - } + let syncState = selectSyncStateForRegistration(matches, storageName) if (syncState == null) { syncState = { ...partial, From bb212391b2cfdf9c371315318e270df491226141 Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Tue, 18 Aug 2026 16:11:11 -0700 Subject: [PATCH 4/9] fix(wallet): keep sync state selection bundle-neutral --- .../src/storage/StorageReaderWriter.ts | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts b/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts index ad771bf67..ca76226c2 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts @@ -34,24 +34,6 @@ import { } from './methods/managedChangePolicy' import { WERR_INVALID_OPERATION } from '../sdk/WERR_errors' -function selectSyncStateForRegistration ( - matches: TableSyncState[], - storageName: string -): TableSyncState | undefined { - if (matches.length <= 1) return matches[0] - - // Older releases included storageName in the lookup and could create - // duplicate rows when a provider was renamed or two apps reused a provider - // identity. Preserve exact-name access so upgraded clients can identify and - // repair those rows without guessing a checkpoint. - const exactMatches = matches.filter(s => s.storageName === storageName) - if (exactMatches.length === 1) return exactMatches[0] - - throw new WERR_INVALID_OPERATION( - 'Storage identity has conflicting sync states. Use a unique identity for each storage provider.' - ) -} - export abstract class StorageReaderWriter extends StorageReader { abstract dropAllData (): Promise abstract migrate (storageName: string, storageIdentityKey: string): Promise @@ -385,7 +367,20 @@ export abstract class StorageReaderWriter extends StorageReader { try { const now = new Date() const matches = await this.findSyncStates({ partial }) - let syncState = selectSyncStateForRegistration(matches, storageName) + let syncState = matches[0] + if (matches.length > 1) { + // Older releases included storageName in the lookup and could create + // duplicate rows when a provider was renamed or two apps reused a + // provider identity. Preserve exact-name access so upgraded clients + // can identify and repair those rows without guessing a checkpoint. + const exactMatches = matches.filter(s => s.storageName === storageName) + if (exactMatches.length !== 1) { + throw new WERR_INVALID_OPERATION( + 'Storage identity has conflicting sync states. Use a unique identity for each storage provider.' + ) + } + syncState = exactMatches[0] + } if (syncState == null) { syncState = { ...partial, From 4e49816876a92d41c05aea53a6f3bbbb0d4fdef6 Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Tue, 18 Aug 2026 16:19:53 -0700 Subject: [PATCH 5/9] chore(wallet): record sync bundle growth --- docs/reference/package-api-migrations.md | 6 +++--- governance/package-release-notes.json | 12 ++++++------ .../wallet-toolbox/client/platform-budget.json | 2 +- .../wallet-toolbox/mobile/platform-budget.json | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index ac5db67d6..82c6ee669 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -52,9 +52,9 @@ and clean-consumer tests remain the executable type authority. | `@bsv/verifast` | `0.3.0` | `0.3.4` | patch | [API and usage](../packages/sdk/verifast.md) | No consumer migration is required; exports, verification behavior, worker protocols, package paths, and runtime defaults are unchanged. | | `@bsv/wallet-helper` | `0.1.1` | `0.1.6` | patch | [API and usage](../packages/helpers/wallet-helper.md) | No consumer migration is required; fluent builder APIs and transaction semantics are unchanged. | | `@bsv/wallet-relay` | `0.2.2` | `0.3.5` | minor | [API and usage](../packages/wallet/wallet-relay.md) | No wallet RPC migration is required; upgrade to @bsv/sdk 2.4.1 or later. Existing relay sessions and number arrays remain valid, and host applications continue to provide their matching Express runtime and type graph. | -| `@bsv/wallet-toolbox` | `2.10.2` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox.md) | No consumer migration is required. Existing sync requests and providers remain wire-compatible. Clients may optionally set includeTotals; capable providers return totals, older providers ignore the hint, and SQL storage creates the new indexes through its normal migration path. | -| `@bsv/wallet-toolbox-client` | `2.10.2` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-client.md) | No consumer migration is required. Existing browser sync calls remain valid; applications may optionally request and consume source totals, while older remote providers continue to work without them. | -| `@bsv/wallet-toolbox-mobile` | `2.10.2` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | No consumer migration is required. Existing mobile sync calls remain valid; applications may optionally request and consume source totals, while older remote providers continue to work without them. | +| `@bsv/wallet-toolbox` | `2.10.2` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox.md) | No consumer migration is required. Existing sync requests and providers remain wire-compatible. Clients may optionally set includeTotals; capable providers return totals, older providers ignore the hint, and upgraded clients automatically carry a writer-local sync-state identifier that upgraded providers use when legacy checkpoints collide. SQL storage creates the new indexes through its normal migration path. | +| `@bsv/wallet-toolbox-client` | `2.10.2` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-client.md) | No consumer migration is required. Existing browser sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, and older remote providers continue to work without either addition. | +| `@bsv/wallet-toolbox-mobile` | `2.10.2` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | No consumer migration is required. Existing mobile sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, and older remote providers continue to work without either addition. | | `create-bsv-app` | `1.0.2` | `1.1.0` | minor | [API and usage](../packages/helpers/create-bsv-app.md) | Existing mainnet and testnet scaffolds are unchanged. New TTN projects pass --network ttn or select TerraTestNet in the configurator. | `none` means the source manifest matches the recorded npm baseline. Any other diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index 254b33136..7cf06d4d7 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -196,22 +196,22 @@ "name": "@bsv/wallet-toolbox", "publishedVersion": "2.10.2", "releaseType": "minor", - "summary": "Accelerates wallet-storage replication with adaptive size-aware source reads and user-scoped SQL proof indexes, and adds opt-in source record totals for exact progress reporting.", - "migration": "No consumer migration is required. Existing sync requests and providers remain wire-compatible. Clients may optionally set includeTotals; capable providers return totals, older providers ignore the hint, and SQL storage creates the new indexes through its normal migration path." + "summary": "Accelerates wallet-storage replication with adaptive size-aware source reads and user-scoped SQL proof indexes, adds opt-in source record totals for exact progress reporting, and disambiguates legacy duplicate sync checkpoints.", + "migration": "No consumer migration is required. Existing sync requests and providers remain wire-compatible. Clients may optionally set includeTotals; capable providers return totals, older providers ignore the hint, and upgraded clients automatically carry a writer-local sync-state identifier that upgraded providers use when legacy checkpoints collide. SQL storage creates the new indexes through its normal migration path." }, { "name": "@bsv/wallet-toolbox-client", "publishedVersion": "2.10.2", "releaseType": "minor", - "summary": "Carries adaptive size-aware wallet sync reads and the backward-compatible includeTotals progress contract for browser and WebView wallets.", - "migration": "No consumer migration is required. Existing browser sync calls remain valid; applications may optionally request and consume source totals, while older remote providers continue to work without them." + "summary": "Carries adaptive size-aware wallet sync reads, the backward-compatible includeTotals progress contract, and duplicate-checkpoint disambiguation for browser and WebView wallets.", + "migration": "No consumer migration is required. Existing browser sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, and older remote providers continue to work without either addition." }, { "name": "@bsv/wallet-toolbox-mobile", "publishedVersion": "2.10.2", "releaseType": "minor", - "summary": "Carries adaptive size-aware wallet sync reads and the backward-compatible includeTotals progress contract for React Native wallets.", - "migration": "No consumer migration is required. Existing mobile sync calls remain valid; applications may optionally request and consume source totals, while older remote providers continue to work without them." + "summary": "Carries adaptive size-aware wallet sync reads, the backward-compatible includeTotals progress contract, and duplicate-checkpoint disambiguation for React Native wallets.", + "migration": "No consumer migration is required. Existing mobile sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, and older remote providers continue to work without either addition." }, { "name": "create-bsv-app", diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 6d27cf99a..13371f9a5 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -2,7 +2,7 @@ "profile": "browser", "maximumBytes": { "vite": { - "raw": 1607400, + "raw": 1608100, "gzip": 378900, "brotli": 297000 }, diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index ed26d710e..9232916c9 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -7,7 +7,7 @@ "brotli": 360000 }, "hermes": { - "raw": 3367000, + "raw": 3367800, "gzip": 1366000, "brotli": 1070000 } From de312597670cd844a31e9419f5260a48e2611beb Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Tue, 18 Aug 2026 16:32:59 -0700 Subject: [PATCH 6/9] chore(wallet): align platform budgets with clean builds --- packages/wallet/wallet-toolbox/client/platform-budget.json | 4 ++-- packages/wallet/wallet-toolbox/mobile/platform-budget.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 13371f9a5..32d599a39 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -3,11 +3,11 @@ "maximumBytes": { "vite": { "raw": 1608100, - "gzip": 378900, + "gzip": 379200, "brotli": 297000 }, "esbuild": { - "raw": 1252900, + "raw": 1253400, "gzip": 345300, "brotli": 277300 } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index 9232916c9..e3880202c 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -7,7 +7,7 @@ "brotli": 360000 }, "hermes": { - "raw": 3367800, + "raw": 3367900, "gzip": 1366000, "brotli": 1070000 } From 15d17bceeccc8e0929fbaf3d95f82b4cc8ffc14b Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Tue, 18 Aug 2026 16:54:27 -0700 Subject: [PATCH 7/9] chore(wallet): account for Linux gzip output --- packages/wallet/wallet-toolbox/client/platform-budget.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 32d599a39..6b78c1120 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -8,7 +8,7 @@ }, "esbuild": { "raw": 1253400, - "gzip": 345300, + "gzip": 345500, "brotli": 277300 } } From e998189969a443205a196cbbd0ad848b5ed2e988 Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Tue, 18 Aug 2026 17:39:20 -0700 Subject: [PATCH 8/9] fix(wallet): retry oversized sync responses --- docs/reference/package-api-migrations.md | 18 ++--- governance/package-release-notes.json | 12 +-- packages/wallet/wallet-toolbox/CHANGELOG.md | 15 ++-- packages/wallet/wallet-toolbox/README.md | 5 +- .../client/platform-budget.json | 10 +-- .../mobile/platform-budget.json | 2 +- .../src/storage/remoting/StorageClientBase.ts | 29 +++++++- .../StorageClientBase.syncRetry.test.ts | 74 +++++++++++++++++++ 8 files changed, 135 insertions(+), 30 deletions(-) create mode 100644 packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.syncRetry.test.ts diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index 82c6ee669..7ee9d54a4 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -52,9 +52,9 @@ and clean-consumer tests remain the executable type authority. | `@bsv/verifast` | `0.3.0` | `0.3.4` | patch | [API and usage](../packages/sdk/verifast.md) | No consumer migration is required; exports, verification behavior, worker protocols, package paths, and runtime defaults are unchanged. | | `@bsv/wallet-helper` | `0.1.1` | `0.1.6` | patch | [API and usage](../packages/helpers/wallet-helper.md) | No consumer migration is required; fluent builder APIs and transaction semantics are unchanged. | | `@bsv/wallet-relay` | `0.2.2` | `0.3.5` | minor | [API and usage](../packages/wallet/wallet-relay.md) | No wallet RPC migration is required; upgrade to @bsv/sdk 2.4.1 or later. Existing relay sessions and number arrays remain valid, and host applications continue to provide their matching Express runtime and type graph. | -| `@bsv/wallet-toolbox` | `2.10.2` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox.md) | No consumer migration is required. Existing sync requests and providers remain wire-compatible. Clients may optionally set includeTotals; capable providers return totals, older providers ignore the hint, and upgraded clients automatically carry a writer-local sync-state identifier that upgraded providers use when legacy checkpoints collide. SQL storage creates the new indexes through its normal migration path. | -| `@bsv/wallet-toolbox-client` | `2.10.2` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-client.md) | No consumer migration is required. Existing browser sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, and older remote providers continue to work without either addition. | -| `@bsv/wallet-toolbox-mobile` | `2.10.2` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | No consumer migration is required. Existing mobile sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, and older remote providers continue to work without either addition. | +| `@bsv/wallet-toolbox` | `2.10.2` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox.md) | No consumer migration is required. Existing sync requests and providers remain wire-compatible. Clients may optionally set includeTotals; capable providers return totals, older providers ignore the hint, and upgraded clients automatically carry a writer-local sync-state identifier that upgraded providers use when legacy checkpoints collide. Remote clients transparently retry HTTP 413 sync reads with a smaller response budget. SQL storage creates the new indexes through its normal migration path. | +| `@bsv/wallet-toolbox-client` | `2.10.2` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-client.md) | No consumer migration is required. Existing browser sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, oversized read-only sync responses retry with a smaller remembered budget, and older remote providers continue to work without either addition. | +| `@bsv/wallet-toolbox-mobile` | `2.10.2` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | No consumer migration is required. Existing mobile sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, oversized read-only sync responses retry with a smaller remembered budget, and older remote providers continue to work without either addition. | | `create-bsv-app` | `1.0.2` | `1.1.0` | minor | [API and usage](../packages/helpers/create-bsv-app.md) | Existing mainnet and testnet scaffolds are unchanged. New TTN projects pass --network ttn or select TerraTestNet in the configurator. | `none` means the source manifest matches the recorded npm baseline. Any other @@ -477,8 +477,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox.md](../packages/wallet/wallet-toolbox.md) - Source: [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) -- Release note: Accelerates wallet-storage replication with adaptive size-aware source reads and user-scoped SQL proof indexes, adds opt-in source record totals for exact progress reporting, and disambiguates legacy duplicate sync checkpoints. -- Migration: No consumer migration is required. Existing sync requests and providers remain wire-compatible. Clients may optionally set includeTotals; capable providers return totals, older providers ignore the hint, and upgraded clients automatically carry a writer-local sync-state identifier that upgraded providers use when legacy checkpoints collide. SQL storage creates the new indexes through its normal migration path. +- Release note: Accelerates wallet-storage replication with adaptive size-aware source reads and user-scoped SQL proof indexes, adds opt-in source record totals for exact progress reporting, disambiguates legacy duplicate sync checkpoints, and retries oversized read-only sync responses with a smaller remembered page budget. +- Migration: No consumer migration is required. Existing sync requests and providers remain wire-compatible. Clients may optionally set includeTotals; capable providers return totals, older providers ignore the hint, and upgraded clients automatically carry a writer-local sync-state identifier that upgraded providers use when legacy checkpoints collide. Remote clients transparently retry HTTP 413 sync reads with a smaller response budget. SQL storage creates the new indexes through its normal migration path. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | ---------------------------------------------------- | -------------------------- | @@ -491,8 +491,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-client.md](../packages/wallet/wallet-toolbox-client.md) - Source: [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) -- Release note: Carries adaptive size-aware wallet sync reads, the backward-compatible includeTotals progress contract, and duplicate-checkpoint disambiguation for browser and WebView wallets. -- Migration: No consumer migration is required. Existing browser sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, and older remote providers continue to work without either addition. +- Release note: Carries adaptive size-aware wallet sync reads, HTTP 413 response-budget retries, the backward-compatible includeTotals progress contract, and duplicate-checkpoint disambiguation for browser and WebView wallets. +- Migration: No consumer migration is required. Existing browser sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, oversized read-only sync responses retry with a smaller remembered budget, and older remote providers continue to work without either addition. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | @@ -503,8 +503,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-mobile.md](../packages/wallet/wallet-toolbox-mobile.md) - Source: [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) -- Release note: Carries adaptive size-aware wallet sync reads, the backward-compatible includeTotals progress contract, and duplicate-checkpoint disambiguation for React Native wallets. -- Migration: No consumer migration is required. Existing mobile sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, and older remote providers continue to work without either addition. +- Release note: Carries adaptive size-aware wallet sync reads, HTTP 413 response-budget retries, the backward-compatible includeTotals progress contract, and duplicate-checkpoint disambiguation for React Native wallets. +- Migration: No consumer migration is required. Existing mobile sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, oversized read-only sync responses retry with a smaller remembered budget, and older remote providers continue to work without either addition. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index 7cf06d4d7..60d7fcbb8 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -196,22 +196,22 @@ "name": "@bsv/wallet-toolbox", "publishedVersion": "2.10.2", "releaseType": "minor", - "summary": "Accelerates wallet-storage replication with adaptive size-aware source reads and user-scoped SQL proof indexes, adds opt-in source record totals for exact progress reporting, and disambiguates legacy duplicate sync checkpoints.", - "migration": "No consumer migration is required. Existing sync requests and providers remain wire-compatible. Clients may optionally set includeTotals; capable providers return totals, older providers ignore the hint, and upgraded clients automatically carry a writer-local sync-state identifier that upgraded providers use when legacy checkpoints collide. SQL storage creates the new indexes through its normal migration path." + "summary": "Accelerates wallet-storage replication with adaptive size-aware source reads and user-scoped SQL proof indexes, adds opt-in source record totals for exact progress reporting, disambiguates legacy duplicate sync checkpoints, and retries oversized read-only sync responses with a smaller remembered page budget.", + "migration": "No consumer migration is required. Existing sync requests and providers remain wire-compatible. Clients may optionally set includeTotals; capable providers return totals, older providers ignore the hint, and upgraded clients automatically carry a writer-local sync-state identifier that upgraded providers use when legacy checkpoints collide. Remote clients transparently retry HTTP 413 sync reads with a smaller response budget. SQL storage creates the new indexes through its normal migration path." }, { "name": "@bsv/wallet-toolbox-client", "publishedVersion": "2.10.2", "releaseType": "minor", - "summary": "Carries adaptive size-aware wallet sync reads, the backward-compatible includeTotals progress contract, and duplicate-checkpoint disambiguation for browser and WebView wallets.", - "migration": "No consumer migration is required. Existing browser sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, and older remote providers continue to work without either addition." + "summary": "Carries adaptive size-aware wallet sync reads, HTTP 413 response-budget retries, the backward-compatible includeTotals progress contract, and duplicate-checkpoint disambiguation for browser and WebView wallets.", + "migration": "No consumer migration is required. Existing browser sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, oversized read-only sync responses retry with a smaller remembered budget, and older remote providers continue to work without either addition." }, { "name": "@bsv/wallet-toolbox-mobile", "publishedVersion": "2.10.2", "releaseType": "minor", - "summary": "Carries adaptive size-aware wallet sync reads, the backward-compatible includeTotals progress contract, and duplicate-checkpoint disambiguation for React Native wallets.", - "migration": "No consumer migration is required. Existing mobile sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, and older remote providers continue to work without either addition." + "summary": "Carries adaptive size-aware wallet sync reads, HTTP 413 response-budget retries, the backward-compatible includeTotals progress contract, and duplicate-checkpoint disambiguation for React Native wallets.", + "migration": "No consumer migration is required. Existing mobile sync calls remain valid; applications may optionally request and consume source totals, upgraded clients automatically carry the registered sync-state identifier, oversized read-only sync responses retry with a smaller remembered budget, and older remote providers continue to work without either addition." }, { "name": "create-bsv-app", diff --git a/packages/wallet/wallet-toolbox/CHANGELOG.md b/packages/wallet/wallet-toolbox/CHANGELOG.md index 93937717b..405c2eae9 100644 --- a/packages/wallet/wallet-toolbox/CHANGELOG.md +++ b/packages/wallet/wallet-toolbox/CHANGELOG.md @@ -30,12 +30,15 @@ attention to changes that materially alter behavior or extend functionality. add composite SQL indexes for user-scoped proof lookups. Sync clients may request optional source record totals for exact progress and ETA displays; older clients and providers remain wire-compatible and do not incur count - queries unless totals are requested. The authoritative Linux Vite fixture is - 1,607,393 raw bytes and the local gzip fixture is 378,833 bytes; those - ceilings advance by 400 and 100 bytes to 1,607,400 and 378,900. The measured - authoritative Linux esbuild fixture is 1,252,871 raw and 345,202 gzip bytes; - those ceilings advance by 400 and 300 bytes to 1,252,900 and 345,300. Other - browser compressed and mobile ceilings remain unchanged. + queries unless totals are requested. Remote clients also recover from a + provider's HTTP 413 response ceiling by retrying the read-only sync request + with a smaller chunk budget and reusing the working limit for later pages. + Clean macOS fixtures measure Vite at 1,608,681 raw, 379,257 gzip, and 297,202 + Brotli bytes; esbuild at 1,253,841 raw, 344,592 gzip, and 277,563 Brotli + bytes; and Hermes at 3,369,326 raw bytes. The reviewed ceilings for the + complete sync feature advance to 1,608,800/379,400/297,300 for Vite, + 1,253,900/345,500/277,700 for esbuild, and 3,369,500 raw bytes for Hermes. + Metro and the compressed mobile ceilings remain unchanged. - Make verified phone changes interruption-safe by staging the replacement key in WAB, publishing the UMP rotation, and then finalizing WAB. Authentication diff --git a/packages/wallet/wallet-toolbox/README.md b/packages/wallet/wallet-toolbox/README.md index 92546590f..fbcc9786a 100644 --- a/packages/wallet/wallet-toolbox/README.md +++ b/packages/wallet/wallet-toolbox/README.md @@ -55,7 +55,10 @@ for exact progress reporting. Older providers ignore the hint, and totals are not counted unless requested. New clients also send the writer-local sync-state identifier selected during provider registration. New providers use it to disambiguate legacy duplicate checkpoints, while either side remains compatible -with older protocol peers. +with older protocol peers. When a provider rejects a sync page because its +serialized RPC response exceeds the service ceiling, remote clients retry the +read-only request with a smaller chunk budget and remember the working limit +for the rest of the session. ### UMP account continuity and phone changes diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 6b78c1120..8f2ee9bf7 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -2,14 +2,14 @@ "profile": "browser", "maximumBytes": { "vite": { - "raw": 1608100, - "gzip": 379200, - "brotli": 297000 + "raw": 1608800, + "gzip": 379400, + "brotli": 297300 }, "esbuild": { - "raw": 1253400, + "raw": 1253900, "gzip": 345500, - "brotli": 277300 + "brotli": 277700 } } } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index e3880202c..b767c9cbb 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -7,7 +7,7 @@ "brotli": 360000 }, "hermes": { - "raw": 3367900, + "raw": 3369500, "gzip": 1366000, "brotli": 1070000 } diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/StorageClientBase.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/StorageClientBase.ts index f80d2eb33..20994bdfd 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/StorageClientBase.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/StorageClientBase.ts @@ -69,6 +69,13 @@ import { } from '../../utility/actionBatchPack' import { pruneBeefForTxids } from '../../utility/beefForTxids' +const syncChunkResponseRetryLimit = 4 +const minimumSyncChunkRoughSize = 64 * 1024 + +function isSyncChunkResponseTooLarge (error: unknown): boolean { + return error instanceof Error && /WalletStorageClient rpcCall: network error 413(?:\s|$)/.test(error.message) +} + export interface StorageClientOptions { /** * Send compact tagged binary request values after the server advertises @@ -97,6 +104,7 @@ export abstract class StorageClientBase implements WalletStorageProvider { protected serverSupportsBinary = false protected readonly binaryRequests: boolean protected readonly telemetry: Telemetry + private syncChunkRoughSizeLimit?: number // Track ephemeral (in-memory) "settings" if you wish to align with isAvailable() checks public settings?: TableSettings @@ -603,8 +611,25 @@ export abstract class StorageClientBase implements WalletStorageProvider { * @returns the next "chunk" of replication data */ async getSyncChunk(args: RequestSyncChunkArgs): Promise { - const r = await this.rpcCall('getSyncChunk', [args]) - return validateSyncChunkEntities(r) + let requestArgs = { ...args } + if (this.syncChunkRoughSizeLimit != null) { + requestArgs.maxRoughSize = Math.min(requestArgs.maxRoughSize, this.syncChunkRoughSizeLimit) + } + + for (let retries = 0; ; retries++) { + try { + const r = await this.rpcCall('getSyncChunk', [requestArgs]) + if (requestArgs.maxRoughSize < args.maxRoughSize) { + this.syncChunkRoughSizeLimit = requestArgs.maxRoughSize + } + return validateSyncChunkEntities(r) + } catch (error: unknown) { + if (!isSyncChunkResponseTooLarge(error) || retries >= syncChunkResponseRetryLimit) throw error + const nextRoughSize = Math.max(minimumSyncChunkRoughSize, Math.floor(requestArgs.maxRoughSize / 2)) + if (nextRoughSize >= requestArgs.maxRoughSize) throw error + requestArgs = { ...requestArgs, maxRoughSize: nextRoughSize } + } + } } /** diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.syncRetry.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.syncRetry.test.ts new file mode 100644 index 000000000..61a3d5ab0 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.syncRetry.test.ts @@ -0,0 +1,74 @@ +import { type WalletInterface } from '@bsv/sdk' +import { type RequestSyncChunkArgs, type SyncChunk } from '../../../sdk/WalletStorage.interfaces' +import { StorageClientBase } from '../StorageClientBase' + +class RetryingStorageClient extends StorageClientBase { + calls: RequestSyncChunkArgs[] = [] + failuresRemaining: number + readonly failure: Error + + constructor (failures: number, failure = new Error('WalletStorageClient rpcCall: network error 413 413')) { + super({} as WalletInterface, 'https://storage.example.test') + this.failuresRemaining = failures + this.failure = failure + } + + protected async rpcCall (method: string, params: unknown[]): Promise { + expect(method).toBe('getSyncChunk') + const args = params[0] as RequestSyncChunkArgs + this.calls.push({ ...args }) + if (this.failuresRemaining-- > 0) throw this.failure + return { + fromStorageIdentityKey: args.fromStorageIdentityKey, + toStorageIdentityKey: args.toStorageIdentityKey, + userIdentityKey: args.identityKey + } as T + } +} + +function makeArgs (): RequestSyncChunkArgs { + return { + identityKey: `02${'11'.repeat(32)}`, + fromStorageIdentityKey: `02${'22'.repeat(32)}`, + toStorageIdentityKey: `02${'33'.repeat(32)}`, + maxItems: 1000, + maxRoughSize: 10_000_000, + offsets: [] + } +} + +describe('StorageClientBase sync response retry', () => { + test('halves an oversized response budget and remembers the working limit', async () => { + const client = new RetryingStorageClient(1) + const args = makeArgs() + + const first = await client.getSyncChunk(args) + const second = await client.getSyncChunk(args) + + expect(first).toMatchObject>({ userIdentityKey: args.identityKey }) + expect(second).toMatchObject>({ userIdentityKey: args.identityKey }) + expect(client.calls.map(call => call.maxRoughSize)).toEqual([10_000_000, 5_000_000, 5_000_000]) + expect(args.maxRoughSize).toBe(10_000_000) + }) + + test('does not retry unrelated network failures', async () => { + const failure = new Error('WalletStorageClient rpcCall: network error 503 Service Unavailable') + const client = new RetryingStorageClient(1, failure) + + await expect(client.getSyncChunk(makeArgs())).rejects.toBe(failure) + expect(client.calls).toHaveLength(1) + }) + + test('stops after the bounded number of oversized-response retries', async () => { + const client = new RetryingStorageClient(10) + + await expect(client.getSyncChunk(makeArgs())).rejects.toThrow('network error 413') + expect(client.calls.map(call => call.maxRoughSize)).toEqual([ + 10_000_000, + 5_000_000, + 2_500_000, + 1_250_000, + 625_000 + ]) + }) +}) From 74ee8521c92baeee73499ee68ee0637f93ac57ff Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Tue, 18 Aug 2026 17:44:51 -0700 Subject: [PATCH 9/9] chore(wallet): account for hosted compression --- packages/wallet/wallet-toolbox/CHANGELOG.md | 7 ++++--- packages/wallet/wallet-toolbox/client/platform-budget.json | 2 +- packages/wallet/wallet-toolbox/mobile/platform-budget.json | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/wallet/wallet-toolbox/CHANGELOG.md b/packages/wallet/wallet-toolbox/CHANGELOG.md index 405c2eae9..4d2853d12 100644 --- a/packages/wallet/wallet-toolbox/CHANGELOG.md +++ b/packages/wallet/wallet-toolbox/CHANGELOG.md @@ -35,10 +35,11 @@ attention to changes that materially alter behavior or extend functionality. with a smaller chunk budget and reusing the working limit for later pages. Clean macOS fixtures measure Vite at 1,608,681 raw, 379,257 gzip, and 297,202 Brotli bytes; esbuild at 1,253,841 raw, 344,592 gzip, and 277,563 Brotli - bytes; and Hermes at 3,369,326 raw bytes. The reviewed ceilings for the + bytes; and Hermes at 3,369,326 raw bytes locally and 1,366,854 gzip bytes on + hosted Linux. The reviewed ceilings for the complete sync feature advance to 1,608,800/379,400/297,300 for Vite, - 1,253,900/345,500/277,700 for esbuild, and 3,369,500 raw bytes for Hermes. - Metro and the compressed mobile ceilings remain unchanged. + 1,253,900/345,600/277,700 for esbuild, and 3,369,500/1,367,000 raw/gzip + bytes for Hermes. Metro and the mobile Brotli ceiling remain unchanged. - Make verified phone changes interruption-safe by staging the replacement key in WAB, publishing the UMP rotation, and then finalizing WAB. Authentication diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 8f2ee9bf7..c43defdfa 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -8,7 +8,7 @@ }, "esbuild": { "raw": 1253900, - "gzip": 345500, + "gzip": 345600, "brotli": 277700 } } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index b767c9cbb..8da3135b2 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -8,7 +8,7 @@ }, "hermes": { "raw": 3369500, - "gzip": 1366000, + "gzip": 1367000, "brotli": 1070000 } }