From e8d3ef6b0d01e1475c24e3b32d7f66d966776a6f Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Tue, 3 Nov 2026 19:16:49 +0100 Subject: [PATCH 01/18] test: add explicit E2E inventory and run-bound reporting --- .github/workflows/inventory.yml | 40 + docs/inventory-reporting.md | 75 ++ package-lock.json | 1 + package.json | 3 + scripts/e2e-inventory.ts | 87 ++ test/helpers/suite.ts | 3 + test/inventory.json | 1577 +++++++++++++++++++++++++ test/reporting/inventory.ts | 190 +++ test/reporting/report.ts | 234 ++++ test/reporting/reporter.ts | 82 ++ test/specs/receive-ln-payments.e2e.ts | 2 +- test/unit/inventory.test.ts | 111 ++ test/unit/reporting.test.ts | 178 +++ tsconfig.json | 2 +- wdio.conf.ts | 3 +- 15 files changed, 2585 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/inventory.yml create mode 100644 docs/inventory-reporting.md create mode 100644 scripts/e2e-inventory.ts create mode 100644 test/inventory.json create mode 100644 test/reporting/inventory.ts create mode 100644 test/reporting/report.ts create mode 100644 test/reporting/reporter.ts create mode 100644 test/unit/inventory.test.ts create mode 100644 test/unit/reporting.test.ts diff --git a/.github/workflows/inventory.yml b/.github/workflows/inventory.yml new file mode 100644 index 00000000..86260a03 --- /dev/null +++ b/.github/workflows/inventory.yml @@ -0,0 +1,40 @@ +name: E2E inventory and reporting + +on: + pull_request: + paths: + - 'test/**' + - 'scripts/e2e-inventory.ts' + - 'wdio.conf.ts' + - 'package*.json' + - '.github/workflows/inventory.yml' + push: + branches: [main] + paths: + - 'test/**' + - 'scripts/e2e-inventory.ts' + - 'wdio.conf.ts' + - 'package*.json' + - '.github/workflows/inventory.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: inventory-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + inventory: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: '22' + cache: npm + - run: npm ci --ignore-scripts + - run: npm run inventory:check + - run: npm run test:reporting diff --git a/docs/inventory-reporting.md b/docs/inventory-reporting.md new file mode 100644 index 00000000..be823711 --- /dev/null +++ b/docs/inventory-reporting.md @@ -0,0 +1,75 @@ +# Inventory and reporting rollout (#264) + +The versioned `test/inventory.json` accounts for each source case on iOS and Android, +including setup utilities, conditionally registered migrations, currently skipped +cases and unsupported automation. It describes intended coverage, not proof that all +nightly jobs currently exist. A `gap` remains incomplete coverage; an inventory check +passing means the inventory is consistent with source, not that the tests passed. + +Run `npm run inventory:check` with Node 22.18+ or Node 24. Discovery reads TypeScript +syntax without importing specs or executing hooks, and is independent of CI greps. +Case titles start with one stable `@case_id`. Literal templates can interpolate +finite loop values or literal factory arguments in that ID. Unknown dynamic IDs, +untagged cases, duplicate IDs and focused tests fail validation. Keep each new/renamed +case and its two platform assignments in the same change. Cases must be declared +with `ciIt`/`it` rather than aliases or computed registration functions. + +The initial destinations retain iOS remote-regtest UI, local money/mining and +hardware jobs, Android local UI/money and staging hardware, staging Pubky/Paykit, +migration variants and private mainnet orchestration. `settings_10` and `onchain_3` +need local execution on both platforms. Mainnet iOS support and currently skipped +cases are explicit gaps linked to #264, not invented platform exemptions. +Fixture flags in the manifest are requirements, not secrets or actual fixture values. +Migration source/target versions must be supplied in each run's `E2E_VARIANT`. + +## Opt-in execution + +Existing invocations and gates keep their current selectors and verdicts. Reporting +is enabled only when `E2E_PLAN` points to a prepared plan. Use a fresh directory for +each run; plans and worker files cannot overwrite an earlier run. For example: + +```sh +export E2E_RUN_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-ios-smoke" +export E2E_APP_SHA='' +export E2E_ARTIFACT_DIGEST='' +export E2E_BACKEND_REVISION='regtest:stag0' +export E2E_VARIANT='default' +export E2E_PLAN="$PWD/artifacts/reporting/plan.json" +node --experimental-strip-types scripts/e2e-inventory.ts plan \ + --platform ios --backend regtest --pack core-ui --plan "$E2E_PLAN" +BACKEND=regtest ATTEMPT=1 ./ci_run_ios.sh \ + --mochaOpts.grep "$(cat artifacts/reporting/grep.txt)" +# Run even when the runner failed. The report command exits nonzero on incomplete coverage. +node --experimental-strip-types scripts/e2e-inventory.ts report --plan "$E2E_PLAN" +``` + +Retain the plan, worker JSON and summary files alongside failure diagnostics. The +report checks exact case IDs, source/build/backend/run identity, missing workers or +cases, unexpected skips, duplicate execution and interrupted runs. Runner failure +after passing cases (including teardown failure) still fails the report. JSON keeps +per-case timing and attempt history; Markdown can be appended to the Actions summary. +These are test execution durations, not billed Mac/Linux usage. + +One attempt is the default. An explicit `E2E_MAX_ATTEMPTS` of 2 or 3 allows a bounded +retry policy; recovered failures remain visible. Inventory-enabled runs bypass the +legacy `/tmp/lock` mechanism and rerun selected cases. A receipt from another night, +source, build, backend or migration variant cannot satisfy the current plan. Existing +non-inventory invocations retain their legacy retry behavior. + +`--pack nightly` selects all applicable obligations for one platform/backend, +including known gaps. It is a building block for the later full-nightly orchestration; +it neither starts a schedule nor declares a platform's whole nightly complete. That +orchestrator must reconcile every required backend/job/variant and detect a missing +scheduled batch. It must run on both platforms every night even for unchanged inputs. +An empty selection fails rather than becoming an empty green run. + +## Stages + +1. Land the inventory/reporting producer and its validation checks first. +2. Add the iOS smoke workflow in shadow using the producer's companion branch or a + revision containing these files. Keep the current required gates and schedules. +3. Measure real smoke duration, concurrent-run isolation and first-pass reliability + before proposing any required-check change. The iOS two-shard pilot, automatic PR + risk selection and full nightly orchestration remain separate stages. + +No ruleset change or existing check-name replacement is part of these first stages. diff --git a/package-lock.json b/package-lock.json index 310d12c4..3053ca31 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "@wdio/cli": "^9.18.2", "@wdio/local-runner": "^9.18.1", "@wdio/mocha-framework": "^9.18.0", + "@wdio/reporter": "9.18.0", "@wdio/spec-reporter": "^9.18.0", "appium": "3.0.0", "appium-uiautomator2-driver": "5.0.0", diff --git a/package.json b/package.json index c13ddf4e..05fc3a90 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,8 @@ "name": "bitkit-e2e-tests", "version": "1.0.0", "scripts": { + "inventory:check": "node --experimental-strip-types scripts/e2e-inventory.ts check", + "test:reporting": "node --experimental-strip-types --test test/unit/inventory.test.ts test/unit/reporting.test.ts", "e2e:android": "PLATFORM=android wdio run ./wdio.conf.ts", "e2e:ios": "PLATFORM=ios wdio run ./wdio.conf.ts", "lint": "eslint './**/*.{ts,js}'", @@ -18,6 +20,7 @@ "@wdio/cli": "^9.18.2", "@wdio/local-runner": "^9.18.1", "@wdio/mocha-framework": "^9.18.0", + "@wdio/reporter": "9.18.0", "@wdio/spec-reporter": "^9.18.0", "appium": "3.0.0", "appium-uiautomator2-driver": "5.0.0", diff --git a/scripts/e2e-inventory.ts b/scripts/e2e-inventory.ts new file mode 100644 index 00000000..b241dd10 --- /dev/null +++ b/scripts/e2e-inventory.ts @@ -0,0 +1,87 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { parseArgs } from 'node:util'; +import { + loadInventory, + validateInventory, + type Backend, + type Platform, +} from '../test/reporting/inventory.ts'; +import { + createPlan, + digest, + markdownReport, + reconcile, + writeJson, + type Plan, + type WorkerReport, +} from '../test/reporting/report.ts'; + +const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + platform: { type: 'string' }, + backend: { type: 'string' }, + pack: { type: 'string' }, + plan: { type: 'string', default: 'artifacts/reporting/plan.json' }, + }, +}); +const root = process.cwd(); +const command = positionals[0]; +try { + if (command === 'check' || command === 'plan') { + const inventory = loadInventory(root); + const discovered = validateInventory(root, inventory); + console.info(`Inventory accounts for ${discovered.length} cases on both platforms.`); + if (command === 'plan') { + if (fs.existsSync(values.plan!)) + throw new Error('Plan already exists; use a fresh run directory'); + const plan = createPlan( + inventory.cases, + { + runId: process.env.E2E_RUN_ID!, + platform: values.platform as Platform, + backend: values.backend as Backend, + appSha: process.env.E2E_APP_SHA!, + e2eSha: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), + artifactDigest: process.env.E2E_ARTIFACT_DIGEST!, + backendRevision: process.env.E2E_BACKEND_REVISION!, + variant: process.env.E2E_VARIANT || 'default', + }, + digest(fs.readFileSync('test/inventory.json', 'utf8')), + values.pack!, + Number(process.env.E2E_MAX_ATTEMPTS || '1') + ); + writeJson(values.plan!, plan); + const grep = `(?:^| )@(?:${plan.expected.join('|')})(?=[ ,]|$)`; + fs.writeFileSync(path.join(path.dirname(values.plan!), 'grep.txt'), grep + '\n'); + console.info(`Planned ${plan.expected.length} cases; plan ${plan.id}`); + } + } else if (command === 'report') { + const plan: Plan = JSON.parse(fs.readFileSync(values.plan!, 'utf8')); + const directory = path.join(path.dirname(values.plan!), 'workers'); + const reports: WorkerReport[] = fs.existsSync(directory) + ? fs + .readdirSync(directory) + .filter((file) => file.endsWith('.json')) + .sort() + .map((file) => JSON.parse(fs.readFileSync(path.join(directory, file), 'utf8'))) + : []; + const result = reconcile(plan, reports); + writeJson(path.join(path.dirname(values.plan!), 'summary.json'), result); + const markdown = markdownReport(result); + fs.writeFileSync(path.join(path.dirname(values.plan!), 'summary.md'), markdown); + if (process.env.GITHUB_STEP_SUMMARY) + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown); + console.info(markdown); + if (result.status !== 'passed') process.exitCode = 1; + } else { + throw new Error( + 'Usage: e2e-inventory.ts check | plan --platform ios|android --backend local|regtest|mainnet --pack NAME [--plan FILE] | report [--plan FILE]' + ); + } +} catch (error) { + console.error(String(error)); + process.exitCode = 1; +} diff --git a/test/helpers/suite.ts b/test/helpers/suite.ts index e4436191..94a99921 100644 --- a/test/helpers/suite.ts +++ b/test/helpers/suite.ts @@ -42,10 +42,13 @@ ciIt.only = (name: string, fn?: () => Promise) => { }; export function checkComplete(name: string): boolean { + // Inventory runs use run-bound receipts, never shared or previous-night locks. + if (process.env.E2E_PLAN) return false; return ON_CI && fs.existsSync(lockFile(name)); } export function markComplete(name: string) { + if (process.env.E2E_PLAN) return; if (!ON_CI) return; fs.mkdirSync(LOCK_PATH, { recursive: true }); fs.writeFileSync(lockFile(name), '1'); diff --git a/test/inventory.json b/test/inventory.json new file mode 100644 index 00000000..5bbb5b10 --- /dev/null +++ b/test/inventory.json @@ -0,0 +1,1577 @@ +{ + "schemaVersion": 1, + "cases": [ + { + "id": "backup_1", + "file": "test/specs/backup.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "boost_1", + "file": "test/specs/boost.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "boost_2", + "file": "test/specs/boost.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "hardware_wallet_1", + "file": "test/specs/hardware-wallet.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "hardware", + "disposition": "required", + "fixtures": ["fresh-wallet", "trezor-emulator", "trezor-bridge-build"] + }, + "android": { + "backend": "regtest", + "pack": "hardware", + "disposition": "required", + "fixtures": ["fresh-wallet", "trezor-emulator", "trezor-bridge-build"] + } + } + }, + { + "id": "hardware_wallet_2", + "file": "test/specs/hardware-wallet.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "hardware", + "disposition": "required", + "fixtures": ["fresh-wallet", "trezor-emulator", "trezor-bridge-build"] + }, + "android": { + "backend": "regtest", + "pack": "hardware", + "disposition": "required", + "fixtures": ["fresh-wallet", "trezor-emulator", "trezor-bridge-build"] + } + } + }, + { + "id": "hardware_wallet_3", + "file": "test/specs/hardware-wallet.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "hardware", + "disposition": "gap", + "fixtures": ["fresh-wallet", "trezor-emulator", "trezor-bridge-build"], + "reason": "Currently skipped after Blocktank channel-order retirement; retire or replace its assertions explicitly, rather than counting a skip as a pass.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "regtest", + "pack": "hardware", + "disposition": "gap", + "fixtures": ["fresh-wallet", "trezor-emulator", "trezor-bridge-build"], + "reason": "Currently skipped after Blocktank channel-order retirement; retire or replace its assertions explicitly, rather than counting a skip as a pass.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + } + } + }, + { + "id": "hardware_wallet_4", + "file": "test/specs/hardware-wallet.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "hardware", + "disposition": "required", + "fixtures": ["fresh-wallet", "trezor-emulator", "trezor-bridge-build"] + }, + "android": { + "backend": "regtest", + "pack": "hardware", + "disposition": "required", + "fixtures": ["fresh-wallet", "trezor-emulator", "trezor-bridge-build"] + } + } + }, + { + "id": "lightning_1", + "file": "test/specs/lightning.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "lnurl_1", + "file": "test/specs/lnurl.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "blocktank_sunset_1", + "file": "test/specs/mainnet/blocktank-sunset.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "gap", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "geocheck-enabled", + "BLOCKTANK_SUNSET_UI" + ], + "reason": "The current spec rejects iOS; equivalent applicable iOS behavior needs its own validated fixture/spec.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "required", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "geocheck-enabled", + "BLOCKTANK_SUNSET_UI" + ] + } + } + }, + { + "id": "blocktank_sunset_2", + "file": "test/specs/mainnet/blocktank-sunset.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "gap", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "geocheck-enabled", + "BLOCKTANK_SUNSET_UI" + ], + "reason": "The current spec rejects iOS; equivalent applicable iOS behavior needs its own validated fixture/spec.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "required", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "geocheck-enabled", + "BLOCKTANK_SUNSET_UI" + ] + } + } + }, + { + "id": "blocktank_sunset_3", + "file": "test/specs/mainnet/blocktank-sunset.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "gap", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "geocheck-enabled", + "BLOCKTANK_SUNSET_UI" + ], + "reason": "The current spec rejects iOS; equivalent applicable iOS behavior needs its own validated fixture/spec.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "required", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "geocheck-enabled", + "BLOCKTANK_SUNSET_UI" + ] + } + } + }, + { + "id": "strike_1", + "file": "test/specs/mainnet/ln.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "gap", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "funded-lightning-wallet", + "private-receiver-or-probe-targets" + ], + "reason": "iOS mainnet fixture, devtools and private nightly job support has not been validated.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "required", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "funded-lightning-wallet", + "private-receiver-or-probe-targets" + ] + } + } + }, + { + "id": "wos_1", + "file": "test/specs/mainnet/ln.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "gap", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "funded-lightning-wallet", + "private-receiver-or-probe-targets" + ], + "reason": "iOS mainnet fixture, devtools and private nightly job support has not been validated.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "required", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "funded-lightning-wallet", + "private-receiver-or-probe-targets" + ] + } + } + }, + { + "id": "probe_mainnet_1", + "file": "test/specs/mainnet/probe.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "gap", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "funded-lightning-wallet", + "private-receiver-or-probe-targets" + ], + "reason": "iOS mainnet fixture, devtools and private nightly job support has not been validated.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "required", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "funded-lightning-wallet", + "private-receiver-or-probe-targets" + ] + } + } + }, + { + "id": "migration_native_restore", + "file": "test/specs/migration.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "previous-native-artifact", + "target-native-artifact", + "isolated-funded-wallet" + ] + }, + "android": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "previous-native-artifact", + "target-native-artifact", + "isolated-funded-wallet" + ] + } + } + }, + { + "id": "migration_native_upgrade", + "file": "test/specs/migration.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "previous-native-artifact", + "target-native-artifact", + "isolated-funded-wallet" + ] + }, + "android": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "previous-native-artifact", + "target-native-artifact", + "isolated-funded-wallet" + ] + } + } + }, + { + "id": "migration_setup_standard", + "file": "test/specs/migration.e2e.ts", + "kind": "fixture", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration-fixture", + "disposition": "inapplicable", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_SETUP_WALLET" + ], + "reason": "Fixture preparation runs on Android; iOS migration consumes its result, not this Android-only setup case.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "regtest", + "pack": "migration-fixture", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_SETUP_WALLET" + ] + } + } + }, + { + "id": "migration_setup_passphrase", + "file": "test/specs/migration.e2e.ts", + "kind": "fixture", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration-fixture", + "disposition": "inapplicable", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_SETUP_WALLET" + ], + "reason": "Fixture preparation runs on Android; iOS migration consumes its result, not this Android-only setup case.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "regtest", + "pack": "migration-fixture", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_SETUP_WALLET" + ] + } + } + }, + { + "id": "migration_setup_sweep", + "file": "test/specs/migration.e2e.ts", + "kind": "fixture", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration-fixture", + "disposition": "inapplicable", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_SETUP_WALLET" + ], + "reason": "Fixture preparation runs on Android; iOS migration consumes its result, not this Android-only setup case.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "regtest", + "pack": "migration-fixture", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_SETUP_WALLET" + ] + } + } + }, + { + "id": "migration_rn_restore", + "file": "test/specs/migration.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "android-prepared-RN_MNEMONIC-and-RN_BALANCE" + ] + }, + "android": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": ["rn-artifact", "target-native-artifact", "isolated-funded-wallet"] + } + } + }, + { + "id": "migration_rn_upgrade", + "file": "test/specs/migration.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "android-prepared-RN_MNEMONIC-and-RN_BALANCE" + ] + }, + "android": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": ["rn-artifact", "target-native-artifact", "isolated-funded-wallet"] + } + } + }, + { + "id": "migration_3", + "file": "test/specs/migration.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_EXTENDED=true", + "android-prepared-RN_MNEMONIC-and-RN_BALANCE" + ] + }, + "android": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_EXTENDED=true" + ] + } + } + }, + { + "id": "migration_4", + "file": "test/specs/migration.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_EXTENDED=true", + "android-prepared-RN_MNEMONIC-and-RN_BALANCE" + ] + }, + "android": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_EXTENDED=true" + ] + } + } + }, + { + "id": "multi_address_1", + "file": "test/specs/multiaddress.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "multi_address_2", + "file": "test/specs/multiaddress.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "gap", + "fixtures": ["fresh-wallet", "blocktank-channel-orders"], + "reason": "Currently skipped after Blocktank channel-order retirement; retire or replace its assertions explicitly, rather than counting a skip as a pass.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "gap", + "fixtures": ["fresh-wallet", "blocktank-channel-orders"], + "reason": "Currently skipped after Blocktank channel-order retirement; retire or replace its assertions explicitly, rather than counting a skip as a pass.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + } + } + }, + { + "id": "multi_address_3", + "file": "test/specs/multiaddress.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "multi_address_4", + "file": "test/specs/multiaddress.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "numberpad_1", + "file": "test/specs/numberpad.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "numberpad_2", + "file": "test/specs/numberpad.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "numberpad_3", + "file": "test/specs/numberpad.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "numberpad_4", + "file": "test/specs/numberpad.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "onboarding_1", + "file": "test/specs/onboarding.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "onboarding_2", + "file": "test/specs/onboarding.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "onchain_1", + "file": "test/specs/onchain.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "onchain_2", + "file": "test/specs/onchain.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "onchain_3", + "file": "test/specs/onchain.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-local", + "disposition": "required", + "fixtures": ["fresh-wallet", "bitcoin-electrum"] + }, + "android": { + "backend": "local", + "pack": "core-local", + "disposition": "required", + "fixtures": ["fresh-wallet", "bitcoin-electrum"] + } + } + }, + { + "id": "paykit_1", + "file": "test/specs/paykit.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + } + } + }, + { + "id": "pubky_profile_1", + "file": "test/specs/pubky-profile.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + } + } + }, + { + "id": "pubky_profile_2", + "file": "test/specs/pubky-profile.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + } + } + }, + { + "id": "pubky_profile_3", + "file": "test/specs/pubky-profile.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + } + } + }, + { + "id": "pubky_profile_4", + "file": "test/specs/pubky-profile.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + } + } + }, + { + "id": "pubky_profile_5", + "file": "test/specs/pubky-profile.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + } + } + }, + { + "id": "utility_receive_ln_1", + "file": "test/specs/receive-ln-payments.e2e.ts", + "kind": "utility", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "manual", + "disposition": "manual", + "fixtures": ["already-prepared-wallet"], + "reason": "Interactive payment utility; does not provision its own wallet and accepts partial payment success. Not an autonomous regression case.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "regtest", + "pack": "manual", + "disposition": "manual", + "fixtures": ["already-prepared-wallet"], + "reason": "Interactive payment utility; does not provision its own wallet and accepts partial payment success. Not an autonomous regression case.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + } + } + }, + { + "id": "receive_1", + "file": "test/specs/receive.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "security_1", + "file": "test/specs/security.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "send_1", + "file": "test/specs/send.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "send_2", + "file": "test/specs/send.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "send_3", + "file": "test/specs/send.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "settings_01", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_02", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_03", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_04", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_05", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_06", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_07", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_08", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "gap", + "fixtures": ["fresh-wallet"], + "reason": "Currently skipped with an address-switching-unavailable comment; applicability needs reassessment before restoring or retiring this case.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "gap", + "fixtures": ["fresh-wallet"], + "reason": "Currently skipped with an address-switching-unavailable comment; applicability needs reassessment before restoring or retiring this case.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + } + } + }, + { + "id": "settings_09", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_10", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-local", + "disposition": "required", + "fixtures": ["fresh-wallet", "bitcoin-electrum"] + }, + "android": { + "backend": "local", + "pack": "core-local", + "disposition": "required", + "fixtures": ["fresh-wallet", "bitcoin-electrum"] + } + } + }, + { + "id": "settings_11", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_12", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_13", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_14", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "transfer_1", + "file": "test/specs/transfer.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "gap", + "fixtures": ["fresh-wallet", "blocktank-channel-orders"], + "reason": "Currently skipped after Blocktank channel-order retirement; retire or replace its assertions explicitly, rather than counting a skip as a pass.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "gap", + "fixtures": ["fresh-wallet", "blocktank-channel-orders"], + "reason": "Currently skipped after Blocktank channel-order retirement; retire or replace its assertions explicitly, rather than counting a skip as a pass.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + } + } + }, + { + "id": "transfer_max", + "file": "test/specs/transfer.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "gap", + "fixtures": ["fresh-wallet", "blocktank-channel-orders"], + "reason": "Currently skipped after Blocktank channel-order retirement; retire or replace its assertions explicitly, rather than counting a skip as a pass.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "gap", + "fixtures": ["fresh-wallet", "blocktank-channel-orders"], + "reason": "Currently skipped after Blocktank channel-order retirement; retire or replace its assertions explicitly, rather than counting a skip as a pass.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + } + } + }, + { + "id": "transfer_2", + "file": "test/specs/transfer.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "widgets_1", + "file": "test/specs/widgets.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "widgets_2", + "file": "test/specs/widgets.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "widgets_3", + "file": "test/specs/widgets.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + } + ] +} diff --git a/test/reporting/inventory.ts b/test/reporting/inventory.ts new file mode 100644 index 00000000..a2b9efb5 --- /dev/null +++ b/test/reporting/inventory.ts @@ -0,0 +1,190 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import ts from 'typescript'; + +export type Platform = 'ios' | 'android'; +export type Backend = 'local' | 'regtest' | 'mainnet'; +export type Disposition = 'required' | 'gap' | 'inapplicable' | 'manual'; +export type Assignment = { + backend: Backend; + pack: string; + disposition: Disposition; + fixtures: string[]; + reason?: string; + reference?: string; +}; +export type InventoryCase = { + id: string; + file: string; + kind: 'test' | 'fixture' | 'utility'; + platforms: Record; +}; +export type Inventory = { schemaVersion: 1; cases: InventoryCase[] }; +export type DiscoveredCase = { id: string; file: string; skipped: boolean }; + +export function caseId(title: string): string { + const match = /^@([a-z][a-z0-9_]*)(?=[\s,]|$)/.exec(title); + if (!match) throw new Error(`Test title needs a stable leading case ID: ${title}`); + return match[1]; +} + +function uncast(node: ts.Expression): ts.Expression { + return ts.isAsExpression(node) || ts.isParenthesizedExpression(node) + ? uncast(node.expression) + : node; +} + +// Only resolve finite source literals; never import specs or execute their hooks during discovery. +function finiteValues(expr: ts.Expression, owner: ts.Node, source: ts.SourceFile): string[] { + expr = uncast(expr); + if (ts.isStringLiteral(expr)) return [expr.text]; + if (ts.isIdentifier(expr)) { + for (let node: ts.Node | undefined = owner.parent; node; node = node.parent) { + if (!ts.isForOfStatement(node) || !ts.isVariableDeclarationList(node.initializer)) continue; + if (node.initializer.declarations[0].name.getText(source) !== expr.text) continue; + const values = uncast(node.expression); + if (ts.isArrayLiteralExpression(values)) { + return values.elements.flatMap((value) => + finiteValues(value as ts.Expression, node, source) + ); + } + } + } + if (ts.isPropertyAccessExpression(expr) && ts.isIdentifier(expr.expression)) { + const objectName = expr.expression.text; + const propertyName = expr.name.text; + for (let node: ts.Node | undefined = owner.parent; node; node = node.parent) { + if (!ts.isFunctionDeclaration(node) || !node.name) continue; + const index = node.parameters.findIndex((p) => p.name.getText(source) === objectName); + if (index < 0) continue; + const values: string[] = []; + const functionName = node.name.text; + function visit(call: ts.Node) { + if (ts.isCallExpression(call) && call.expression.getText(source) === functionName) { + const arg = call.arguments[index]; + if (!arg || !ts.isObjectLiteralExpression(arg)) { + throw new Error(`Non-literal case factory argument: ${call.getText(source)}`); + } + const property = arg.properties.find( + (p) => ts.isPropertyAssignment(p) && p.name.getText(source) === propertyName + ); + if (!property || !ts.isPropertyAssignment(property)) { + throw new Error(`Missing ${propertyName} in case factory ${functionName}`); + } + values.push(...finiteValues(property.initializer, call, source)); + } + ts.forEachChild(call, visit); + } + visit(source); + if (values.length) return values; + } + } + throw new Error(`Cannot discover dynamic case ID: ${expr.getText(source)}`); +} + +function titleIds(title: ts.Expression, call: ts.Node, source: ts.SourceFile): string[] { + if (ts.isStringLiteralLike(title)) return [caseId(title.text)]; + if (!ts.isTemplateExpression(title)) { + throw new Error(`Cannot discover test title: ${title.getText(source)}`); + } + let prefixes = [title.head.text]; + for (const span of title.templateSpans) { + if (prefixes.every((s) => /^@[a-z][a-z0-9_]*[\s,]/.test(s))) break; + const values = finiteValues(span.expression, call, source); + prefixes = prefixes.flatMap((s) => values.map((value) => s + value + span.literal.text)); + } + return prefixes.map(caseId); +} + +function filesIn(directory: string): string[] { + return fs.readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const file = path.join(directory, entry.name); + return entry.isDirectory() ? filesIn(file) : entry.name.endsWith('.ts') ? [file] : []; + }); +} + +export function discoverCases(root: string): DiscoveredCase[] { + const cases: DiscoveredCase[] = []; + for (const absolute of filesIn(path.join(root, 'test/specs')).sort()) { + const file = path.relative(root, absolute).split(path.sep).join('/'); + const source = ts.createSourceFile( + file, + fs.readFileSync(absolute, 'utf8'), + ts.ScriptTarget.Latest, + true + ); + function visit(node: ts.Node) { + if (ts.isCallExpression(node)) { + const name = node.expression.getText(source); + if (/^(?:ciIt|it|test|xit)(?:\.(?:skip|only))?$/.test(name)) { + if (name.endsWith('.only')) throw new Error(`Focused test is forbidden: ${file}`); + if (!node.arguments[0]) throw new Error(`Missing test title: ${file}`); + let skipped = name.endsWith('.skip') || name === 'xit'; + for (let parent: ts.Node | undefined = node.parent; parent; parent = parent.parent) { + if (ts.isCallExpression(parent)) { + const parentName = parent.expression.getText(source); + if (parentName === 'describe.only') + throw new Error(`Focused suite is forbidden: ${file}`); + if (parentName === 'describe.skip' || parentName === 'xdescribe') skipped = true; + } + } + for (const id of titleIds(node.arguments[0], node, source)) + cases.push({ id, file, skipped }); + } + } + ts.forEachChild(node, visit); + } + visit(source); + } + const seen = new Set(); + for (const item of cases) { + if (seen.has(item.id)) throw new Error(`Duplicate source case ID: ${item.id}`); + seen.add(item.id); + } + return cases; +} + +export function loadInventory(root: string): Inventory { + return JSON.parse(fs.readFileSync(path.join(root, 'test/inventory.json'), 'utf8')); +} + +export function validateInventory(root: string, inventory = loadInventory(root)): DiscoveredCase[] { + if (inventory.schemaVersion !== 1 || !Array.isArray(inventory.cases) || !inventory.cases.length) { + throw new Error('Invalid or empty inventory'); + } + const discovered = discoverCases(root); + const byId = new Map(discovered.map((item) => [item.id, item])); + const seen = new Set(); + for (const item of inventory.cases) { + const source = byId.get(item.id); + if (seen.has(item.id)) throw new Error(`Duplicate inventory case: ${item.id}`); + seen.add(item.id); + if (!source || source.file !== item.file) + throw new Error(`Stale inventory case/file: ${item.id}`); + if (!['test', 'fixture', 'utility'].includes(item.kind)) + throw new Error(`Invalid kind: ${item.id}`); + for (const platform of ['ios', 'android'] as const) { + const entry = item.platforms?.[platform]; + if ( + !entry || + !['local', 'regtest', 'mainnet'].includes(entry.backend) || + !entry.pack || + !['required', 'gap', 'inapplicable', 'manual'].includes(entry.disposition) || + !Array.isArray(entry.fixtures) || + !entry.fixtures.every((v) => typeof v === 'string') + ) { + throw new Error(`Invalid ${platform} assignment: ${item.id}`); + } + if (entry.disposition !== 'required' && (!entry.reason || !entry.reference)) { + throw new Error(`Unexplained ${platform} disposition: ${item.id}`); + } + if (source.skipped && entry.disposition === 'required') { + throw new Error(`Required case is statically skipped: ${platform}/${item.id}`); + } + } + } + const missing = discovered.filter((item) => !seen.has(item.id)); + if (missing.length) + throw new Error(`Uninventoried cases: ${missing.map((item) => item.id).join(', ')}`); + return discovered; +} diff --git a/test/reporting/report.ts b/test/reporting/report.ts new file mode 100644 index 00000000..f563d65d --- /dev/null +++ b/test/reporting/report.ts @@ -0,0 +1,234 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import type { Backend, InventoryCase, Platform } from './inventory.ts'; + +export type Identity = { + runId: string; + platform: Platform; + backend: Backend; + appSha: string; + e2eSha: string; + artifactDigest: string; + backendRevision: string; + variant: string; +}; +export type Plan = { + schemaVersion: 1; + id: string; + identity: Identity; + inventoryDigest: string; + pack: string; + maxAttempts: number; + cases: InventoryCase[]; + expected: string[]; +}; +export type Event = { + id: string; + status: 'passed' | 'failed' | 'skipped'; + durationMs: number; +}; +export type WorkerReport = { + schemaVersion: 1; + planId: string; + attempt: number; + cid: string; + startedAt: string; + finishedAt?: string; + failures: number; + errors: string[]; + events: Event[]; +}; + +export function digest(value: string): string { + return crypto.createHash('sha256').update(value).digest('hex'); +} + +export function writeJson(file: string, value: unknown): void { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const temporary = `${file}.${process.pid}.tmp`; + fs.writeFileSync(temporary, JSON.stringify(value, null, 2) + '\n'); + fs.renameSync(temporary, file); +} + +export function createPlan( + cases: InventoryCase[], + identity: Identity, + inventoryDigest: string, + pack: string, + maxAttempts = 1 +): Plan { + if ( + !['ios', 'android'].includes(identity.platform) || + !['local', 'regtest', 'mainnet'].includes(identity.backend) || + Object.values(identity).some((value) => typeof value !== 'string' || !value.trim()) || + !/^[a-f0-9]{40}$/.test(identity.appSha) || + !/^[a-f0-9]{40}$/.test(identity.e2eSha) || + !/^[a-f0-9]{64}$/.test(identity.artifactDigest) || + !/^[a-f0-9]{64}$/.test(inventoryDigest) || + !Number.isInteger(maxAttempts) || + maxAttempts < 1 || + maxAttempts > 3 + ) { + throw new Error('Invalid or incomplete run identity/attempt policy'); + } + const selected = cases.filter((item) => { + const assignment = item.platforms[identity.platform]; + return ( + assignment.backend === identity.backend && (pack === 'nightly' || assignment.pack === pack) + ); + }); + const expected = selected + .filter((item) => ['required', 'gap'].includes(item.platforms[identity.platform].disposition)) + .map((item) => item.id) + .sort(); + if (!expected.length) + throw new Error(`No expected cases for ${identity.platform}/${identity.backend}/${pack}`); + if (new Set(expected).size !== expected.length) throw new Error('Duplicate expected case ID'); + const data = { + schemaVersion: 1 as const, + identity, + inventoryDigest, + pack, + maxAttempts, + cases: selected, + expected, + }; + return { ...data, id: digest(JSON.stringify(data)) }; +} + +export function validatePlan(plan: Plan): void { + const rebuilt = createPlan( + plan.cases, + plan.identity, + plan.inventoryDigest, + plan.pack, + plan.maxAttempts + ); + if ( + rebuilt.id !== plan.id || + JSON.stringify(rebuilt.expected) !== JSON.stringify(plan.expected) + ) { + throw new Error('Run plan changed or is invalid'); + } +} + +export function reconcile(plan: Plan, reports: WorkerReport[]) { + validatePlan(plan); + const problems: string[] = []; + const expected = new Set(plan.expected); + const observations = new Map(); + const workers = new Set(); + const executions = new Set(); + let firstFailure = false; + const finalAttempt = Math.max(1, ...reports.map((r) => r.attempt)); + if (!reports.length) + problems.push('No worker reports: tests never started or reports are missing'); + for (const report of reports) { + if ( + report.schemaVersion !== 1 || + report.planId !== plan.id || + !Number.isInteger(report.attempt) || + report.attempt < 1 || + report.attempt > plan.maxAttempts || + !report.cid || + !Array.isArray(report.events) || + !Array.isArray(report.errors) || + !Number.isInteger(report.failures) || + report.failures < 0 + ) { + problems.push('Invalid worker report or mismatched run/platform/backend/source identity'); + continue; + } + const worker = `${report.attempt}/${report.cid}`; + if (workers.has(worker)) problems.push(`Duplicate worker report: ${worker}`); + workers.add(worker); + if (!report.finishedAt) problems.push(`Interrupted worker: ${worker}`); + firstFailure ||= report.failures > 0 || report.errors.length > 0; + if (report.attempt === finalAttempt && (report.failures || report.errors.length)) { + problems.push(`Final attempt failed: ${worker} (${report.errors.join('; ')})`); + } + for (const event of report.events) { + if ( + !expected.has(event.id) || + !['passed', 'failed', 'skipped'].includes(event.status) || + !Number.isFinite(event.durationMs) || + event.durationMs < 0 + ) { + problems.push(`Unexpected or invalid test event: ${event.id}`); + continue; + } + const execution = `${report.attempt}/${event.id}`; + if (executions.has(execution)) problems.push(`Duplicate case execution: ${execution}`); + executions.add(execution); + const history = observations.get(event.id) ?? []; + history.push({ attempt: report.attempt, event }); + observations.set(event.id, history); + } + } + const results = plan.expected.map((id) => { + const history = (observations.get(id) ?? []).sort((a, b) => a.attempt - b.attempt); + let status = 'missing'; + let previouslyPassed = false; + for (const { attempt, event } of history) { + firstFailure ||= event.status === 'failed'; + if (event.status === 'skipped') { + if (!previouslyPassed) { + problems.push(`Unexpected skip: ${id} (attempt ${attempt})`); + status = 'unexpected-skip'; + } + } else { + status = event.status; + previouslyPassed = event.status === 'passed'; + } + } + if (status !== 'passed') problems.push(`${id}: ${status}`); + return { id, status, firstAttempt: history[0]?.event.status ?? 'missing', history }; + }); + const dispositions = plan.cases.filter( + (item) => item.platforms[plan.identity.platform].disposition !== 'required' + ); + for (const item of dispositions) { + if (item.platforms[plan.identity.platform].disposition === 'gap') + problems.push(`Unresolved coverage gap: ${item.id}`); + } + return { + schemaVersion: 1, + planId: plan.id, + identity: plan.identity, + pack: plan.pack, + status: problems.length ? 'incomplete' : 'passed', + firstFailure, + expectedCount: plan.expected.length, + passedCount: results.filter((r) => r.status === 'passed').length, + durationMs: reports + .flatMap((r) => r.events ?? []) + .reduce((sum, e) => sum + (e.durationMs || 0), 0), + problems, + results, + dispositions, + }; +} + +export function markdownReport(result: ReturnType): string { + return [ + `## E2E ${result.pack}: ${result.status}`, + '', + `${result.identity.platform} / ${result.identity.backend} — ${result.passedCount}/${result.expectedCount} expected cases passed.`, + `First-attempt failure observed: ${result.firstFailure ? 'yes' : 'no'}.`, + '', + `Run: \`${result.identity.runId}\`; app: \`${result.identity.appSha}\`; tests: \`${result.identity.e2eSha}\`.`, + '', + '| Case | Result | First attempt |', + '| --- | --- | --- |', + ...result.results.map((r) => `| ${r.id} | ${r.status} | ${r.firstAttempt} |`), + '', + ...result.problems.map((p) => `- ${p}`), + '', + ...result.dispositions.map((item) => { + const entry = item.platforms[result.identity.platform]; + return `- ${item.id}: **${entry.disposition}** — ${entry.reason} (${entry.reference})`; + }), + '', + ].join('\n'); +} diff --git a/test/reporting/reporter.ts b/test/reporting/reporter.ts new file mode 100644 index 00000000..b7105b27 --- /dev/null +++ b/test/reporting/reporter.ts @@ -0,0 +1,82 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import WDIOReporter, { type RunnerStats, type TestStats } from '@wdio/reporter'; +import { caseId } from './inventory.ts'; +import { digest, validatePlan, writeJson, type Plan, type WorkerReport } from './report.ts'; + +export default class InventoryReporter extends WDIOReporter { + private plan: Plan; + private report?: WorkerReport; + private file?: string; + + constructor(options: Record) { + super({ ...options, stdout: false }); + this.plan = JSON.parse(fs.readFileSync(process.env.E2E_PLAN!, 'utf8')); + validatePlan(this.plan); + if ( + this.plan.identity.platform !== process.env.PLATFORM || + this.plan.identity.backend !== (process.env.BACKEND || 'local') || + this.plan.inventoryDigest !== digest(fs.readFileSync('test/inventory.json', 'utf8')) + ) { + throw new Error('Run plan does not match the executing platform/backend/inventory'); + } + } + + onRunnerStart(runner: RunnerStats) { + const attempt = Number(process.env.ATTEMPT || '1'); + if (!Number.isInteger(attempt) || attempt < 1 || attempt > this.plan.maxAttempts) { + throw new Error('Attempt is outside the declared run policy'); + } + if (!/^[\w-]+$/.test(runner.cid)) throw new Error('Invalid worker ID'); + this.file = path.join( + path.dirname(process.env.E2E_PLAN!), + 'workers', + `attempt-${attempt}-${runner.cid}.json` + ); + if (fs.existsSync(this.file)) + throw new Error('Refusing to overwrite an existing worker report'); + this.report = { + schemaVersion: 1, + planId: this.plan.id, + attempt, + cid: runner.cid, + startedAt: new Date().toISOString(), + failures: 0, + errors: [], + events: [], + }; + this.flush(); + } + + private record(test: TestStats, status: 'passed' | 'failed' | 'skipped') { + if (!this.report) throw new Error('Test event arrived before runner start'); + try { + this.report.events.push({ id: caseId(test.title), status, durationMs: test.duration || 0 }); + } catch (error) { + this.report.errors.push(String(error)); + } + this.flush(); + } + + onTestPass(test: TestStats) { + this.record(test, 'passed'); + } + onTestFail(test: TestStats) { + this.record(test, 'failed'); + } + onTestSkip(test: TestStats) { + this.record(test, 'skipped'); + } + + onRunnerEnd(runner: RunnerStats) { + if (!this.report) throw new Error('Runner ended without starting'); + this.report.failures = runner.failures ?? 0; + if (runner.error) this.report.errors.push(runner.error); + this.report.finishedAt = new Date().toISOString(); + this.flush(); + } + + private flush() { + if (this.file && this.report) writeJson(this.file, this.report); + } +} diff --git a/test/specs/receive-ln-payments.e2e.ts b/test/specs/receive-ln-payments.e2e.ts index cf5aa5d2..7677ab13 100644 --- a/test/specs/receive-ln-payments.e2e.ts +++ b/test/specs/receive-ln-payments.e2e.ts @@ -43,7 +43,7 @@ describe('Receive LN payments (utility)', () => { await sleep(3000); }); - it(`should receive ${PAYMENT_COUNT} Lightning payments`, async () => { + it(`@utility_receive_ln_1 - should receive ${PAYMENT_COUNT} Lightning payments`, async () => { let succeeded = 0; let failed = 0; diff --git a/test/unit/inventory.test.ts b/test/unit/inventory.test.ts new file mode 100644 index 00000000..154ba2f0 --- /dev/null +++ b/test/unit/inventory.test.ts @@ -0,0 +1,111 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { + discoverCases, + loadInventory, + validateInventory, + type Inventory, +} from '../reporting/inventory.ts'; + +function fixture(t: test.TestContext, source: string) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'inventory-test-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + fs.mkdirSync(path.join(root, 'test/specs'), { recursive: true }); + fs.writeFileSync(path.join(root, 'test/specs/example.ts'), source); + return root; +} +function inventory(id = 'example_1'): Inventory { + const assignment = { + backend: 'regtest' as const, + pack: 'core', + disposition: 'required' as const, + fixtures: [], + }; + return { + schemaVersion: 1, + cases: [ + { + id, + file: 'test/specs/example.ts', + kind: 'test', + platforms: { ios: { ...assignment }, android: { ...assignment } }, + }, + ], + }; +} + +test('repository inventory covers every source case, including skipped and conditional cases', () => { + const root = process.cwd(); + const cases = validateInventory(root); + assert.equal(cases.length, loadInventory(root).cases.length); + assert.ok(cases.some((c) => c.id === 'migration_setup_sweep')); + assert.ok(cases.some((c) => c.id === 'hardware_wallet_3' && c.skipped)); +}); + +test('discovery does not execute imports, hooks or runtime conditions', (t) => { + const root = fixture( + t, + "import dangerous from 'missing-network-module'; before(() => dangerous()); if (process.env.EXTRA) { ciIt('@example_1 - hidden', async () => {}); }" + ); + assert.equal(validateInventory(root, inventory()).length, 1); +}); + +test('dynamic IDs expand from finite factory arguments and loop values', (t) => { + const root = fixture( + t, + ` + function suite(config) { describe(config.label, () => { ciIt(\`\${config.tag} - scenario\`, async () => {}); }); } + suite({label:'first',tag:'@first_1'}); suite({label:'second',tag:'@second_1'}); + for (const method of ['restore','upgrade'] as const) { ciIt(\`@native_\${method} - \${process.env.VERSION}\`, async () => {}); } + ` + ); + assert.deepEqual( + discoverCases(root).map((c) => c.id), + ['first_1', 'second_1', 'native_restore', 'native_upgrade'] + ); +}); + +for (const [name, source, pattern] of [ + ['untagged case', "it('untracked test', () => {})", /stable leading case ID/], + [ + 'unknown dynamic ID', + 'ciIt(`@case_${process.env.NAME} - test`, () => {})', + /Cannot discover dynamic/, + ], + [ + 'duplicate ID', + "ciIt('@example_1 - a', () => {}); ciIt('@example_1 - b', () => {})", + /Duplicate source/, + ], + ['focused test', "ciIt.only('@example_1 - a', () => {})", /Focused test/], + [ + 'focused suite', + "describe.only('a', () => { ciIt('@example_1 - a', () => {}); })", + /Focused suite/, + ], +] as const) + test(`rejects ${name}`, (t) => assert.throws(() => discoverCases(fixture(t, source)), pattern)); + +test('new source cases cannot be hidden by workflow selection', (t) => { + const root = fixture(t, "ciIt('@example_1 - a', () => {}); ciIt('@new_1 - b', () => {})"); + assert.throws(() => validateInventory(root, inventory()), /Uninventoried cases: new_1/); +}); +test('removed or renamed tests and stale file mappings fail', (t) => { + const root = fixture(t, "ciIt('@renamed_1 - a', () => {})"); + assert.throws(() => validateInventory(root, inventory()), /Stale inventory/); +}); +test('every case requires both platforms and an explained exception', (t) => { + const root = fixture(t, "ciIt('@example_1 - a', () => {})"); + const data = inventory(); + data.cases[0].platforms.ios.disposition = 'inapplicable'; + assert.throws(() => validateInventory(root, data), /Unexplained ios disposition/); + delete (data.cases[0].platforms as Partial<(typeof data.cases)[0]['platforms']>).ios; + assert.throws(() => validateInventory(root, data), /Invalid ios assignment/); +}); +test('skip does not silently remove a required case', (t) => { + const root = fixture(t, "describe.skip('suite',()=>{ciIt('@example_1 - a', () => {})})"); + assert.throws(() => validateInventory(root, inventory()), /Required case is statically skipped/); +}); diff --git a/test/unit/reporting.test.ts b/test/unit/reporting.test.ts new file mode 100644 index 00000000..e11084ff --- /dev/null +++ b/test/unit/reporting.test.ts @@ -0,0 +1,178 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { execFileSync } from 'node:child_process'; +import { + createPlan, + reconcile, + validatePlan, + type Identity, + type WorkerReport, +} from '../reporting/report.ts'; +import type { InventoryCase } from '../reporting/inventory.ts'; + +const assignment = { + backend: 'regtest' as const, + pack: 'smoke', + disposition: 'required' as const, + fixtures: [], +}; +const cases: InventoryCase[] = ['wallet_1', 'payment_1'].map((id) => ({ + id, + file: 'test/specs/example.ts', + kind: 'test', + platforms: { ios: { ...assignment }, android: { ...assignment } }, +})); +const identity: Identity = { + runId: 'night-2026-11-03', + platform: 'ios', + backend: 'regtest', + appSha: 'a'.repeat(40), + e2eSha: 'b'.repeat(40), + artifactDigest: 'c'.repeat(64), + backendRevision: 'stag0', + variant: 'default', +}; +function plan() { + return createPlan(cases, identity, 'd'.repeat(64), 'smoke', 3); +} +function report( + events: WorkerReport['events'], + overrides: Partial = {} +): WorkerReport { + return { + schemaVersion: 1, + planId: plan().id, + attempt: 1, + cid: '0-0', + startedAt: '2026-11-03T00:00:00Z', + finishedAt: '2026-11-03T00:01:00Z', + failures: 0, + errors: [], + events, + ...overrides, + }; +} +const passes = cases.map((c) => ({ id: c.id, status: 'passed' as const, durationMs: 10 })); + +test('exact expected cases pass and retain timing', () => { + const result = reconcile(plan(), [report(passes)]); + assert.equal(result.status, 'passed'); + assert.equal(result.passedCount, 2); + assert.equal(result.durationMs, 20); +}); +test('same inputs on the following night require fresh receipts', () => { + const next = createPlan( + cases, + { ...identity, runId: 'night-2026-11-04' }, + 'd'.repeat(64), + 'smoke', + 3 + ); + assert.notEqual(next.id, plan().id); + assert.equal(reconcile(next, [report(passes)]).status, 'incomplete'); +}); +for (const change of [ + { platform: 'android' }, + { appSha: 'e'.repeat(40) }, + { backendRevision: 'other' }, + { variant: 'rn-upgrade' }, + { artifactDigest: 'f'.repeat(64) }, +]) { + test(`receipts cannot cross identity boundary ${Object.keys(change)[0]}`, () => { + const next = createPlan( + cases, + { ...identity, ...change } as Identity, + 'd'.repeat(64), + 'smoke', + 3 + ); + assert.equal(reconcile(next, [report(passes)]).status, 'incomplete'); + }); +} +test('unknown/empty packs and incomplete build identity cannot create a green plan', () => { + assert.throws(() => createPlan(cases, identity, 'd'.repeat(64), 'typo'), /No expected cases/); + assert.throws( + () => createPlan(cases, { ...identity, appSha: '' }, 'd'.repeat(64), 'smoke'), + /identity/ + ); +}); +test('tampered expected cases invalidate the plan', () => { + const changed = plan(); + changed.expected.pop(); + assert.throws(() => validatePlan(changed), /changed or is invalid/); +}); +for (const [name, reports] of [ + ['no started worker', []], + ['missing case', [report(passes.slice(0, 1))]], + ['unexpected skip', [report([{ ...passes[0], status: 'skipped' }, passes[1]])]], + ['wrong case with same total count', [report([passes[0], { ...passes[1], id: 'different_1' }])]], + ['duplicate report', [report(passes), report(passes)]], + ['duplicate case across workers', [report(passes), report([passes[0]], { cid: '0-1' })]], + ['worker interrupted after assertions', [report(passes, { finishedAt: undefined })]], + ['setup failure', [report([], { failures: 1 })]], + ['teardown failure after passing cases', [report(passes, { failures: 1 })]], +] as [string, WorkerReport[]][]) + test(`${name} cannot report full coverage`, () => + assert.equal(reconcile(plan(), reports).status, 'incomplete')); + +test('a recovered retry keeps first-attempt failure evidence', () => { + const result = reconcile(plan(), [ + report([{ ...passes[0], status: 'failed' }, passes[1]], { failures: 1 }), + report(passes, { attempt: 2 }), + ]); + assert.equal(result.status, 'passed'); + assert.equal(result.firstFailure, true); + assert.equal(result.results.find((r) => r.id === 'wallet_1')?.firstAttempt, 'failed'); +}); +test('a skipped retry is accepted only with a verified same-run earlier pass', () => { + const retry = report([{ ...passes[0], status: 'skipped' }, passes[1]], { attempt: 2 }); + assert.equal(reconcile(plan(), [report(passes), retry]).status, 'passed'); + assert.equal(reconcile(plan(), [retry]).status, 'incomplete'); +}); +test('an old pass cannot mask a later failure', () => { + assert.equal( + reconcile(plan(), [ + report(passes), + report([{ ...passes[0], status: 'failed' }, passes[1]], { attempt: 2, failures: 1 }), + ]).status, + 'incomplete' + ); +}); +test('single-attempt smoke rejects retry receipts', () => { + const single = createPlan(cases, identity, 'd'.repeat(64), 'smoke'); + assert.equal( + reconcile(single, [report(passes, { planId: single.id, attempt: 2 })]).status, + 'incomplete' + ); +}); +test('known coverage gaps remain visible even if a case happened to pass', () => { + const data = structuredClone(cases); + data[0].platforms.ios = { + ...assignment, + disposition: 'gap', + reason: 'missing fixture validation', + reference: '#264', + }; + const p = createPlan(data, identity, 'd'.repeat(64), 'smoke'); + const result = reconcile(p, [report(passes, { planId: p.id })]); + assert.equal(result.status, 'incomplete'); + assert.match(result.problems.join(' '), /Unresolved coverage gap/); +}); +test('inventory-enabled runs bypass legacy completion locks without changing legacy mode', (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'report-lock-test-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const helper = path.resolve('test/helpers/suite.ts'); + const output = execFileSync( + process.execPath, + [ + '--experimental-strip-types', + '-e', + `const fs=require('node:fs'); fs.existsSync=()=>true; fs.mkdirSync=()=>{throw Error('must not touch legacy locks')}; fs.writeFileSync=()=>{throw Error('must not touch legacy locks')}; const s=require(${JSON.stringify(helper)}); if(s.checkComplete('case'))throw Error('stale pass'); s.markComplete('case'); delete process.env.E2E_PLAN; if(!s.checkComplete('case'))throw Error('legacy behavior changed'); console.log('ok');`, + ], + { env: { ...process.env, CI: '1', E2E_PLAN: path.join(root, 'plan.json') }, encoding: 'utf8' } + ); + assert.match(output, /ok/); +}); diff --git a/tsconfig.json b/tsconfig.json index 07de3e38..5d19c29f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -22,5 +22,5 @@ "noFallthroughCasesInSwitch": true, "esModuleInterop": true }, - "include": ["test", "test/types/**/*.d.ts", "wdio.conf.ts"] + "include": ["test", "test/types/**/*.d.ts", "wdio.conf.ts", "scripts/e2e-inventory.ts"] } diff --git a/wdio.conf.ts b/wdio.conf.ts index bdb546b7..53abf89c 100644 --- a/wdio.conf.ts +++ b/wdio.conf.ts @@ -2,6 +2,7 @@ import path from 'node:path'; import fs from 'node:fs'; import { grantIOSCameraPermission } from './test/helpers/setup'; +import InventoryReporter from './test/reporting/reporter'; const appiumPort = Number(process.env.APPIUM_PORT ?? '4723'); const isAndroid = process.env.PLATFORM === 'android'; @@ -193,7 +194,7 @@ export const config: WebdriverIO.Config = { // Test reporter for stdout. // The only one supported by default is 'dot' // see also: https://webdriver.io/docs/dot-reporter - reporters: ['spec'], + reporters: process.env.E2E_PLAN ? ['spec', [InventoryReporter, {}]] : ['spec'], // Options to be passed to Mocha. // See the full list at http://mochajs.org/ From 5f3f28be7d8bcbf44698e1748befb6eecfde8eac Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Tue, 3 Nov 2026 19:24:17 +0100 Subject: [PATCH 02/18] test: add focused regtest smoke and independent payment evidence --- docs/inventory-reporting.md | 17 ++++ package-lock.json | 1 + package.json | 3 +- test/helpers/smoke-oracle.ts | 163 ++++++++++++++++++++++++++++++ test/inventory.json | 57 +++++++++++ test/reporting/reporter.ts | 18 +++- test/specs/smoke.e2e.ts | 109 ++++++++++++++++++++ test/unit/reporter-events.test.ts | 118 +++++++++++++++++++++ test/unit/smoke-oracle.test.ts | 118 +++++++++++++++++++++ wdio.conf.ts | 14 +++ 10 files changed, 616 insertions(+), 2 deletions(-) create mode 100644 test/helpers/smoke-oracle.ts create mode 100644 test/specs/smoke.e2e.ts create mode 100644 test/unit/reporter-events.test.ts create mode 100644 test/unit/smoke-oracle.test.ts diff --git a/docs/inventory-reporting.md b/docs/inventory-reporting.md index be823711..ae99b17b 100644 --- a/docs/inventory-reporting.md +++ b/docs/inventory-reporting.md @@ -73,3 +73,20 @@ An empty selection fails rather than becoming an empty green run. risk selection and full nightly orchestration remain separate stages. No ruleset change or existing check-name replacement is part of these first stages. + +## Shadow smoke producer + +The `smoke` pack contains three fresh-wallet regtest cases: `smoke_wallet_1` +(Home), `smoke_receive_1` (receive QR address and exact entered amount), and +`smoke_onchain_1` (fund and send). The payment oracle queries Electrum directly, +recomputes transaction IDs from raw transactions, checks the exact recipient and +amount, and requires the outgoing payment to spend this case's funding transaction. +A fresh recipient prevents another concurrent run's payment from satisfying the +assertion. The oracle uses bounded requests and stores raw transaction evidence. +These cases do not match the existing `@ios_gate` or Android feature selectors. + +The iOS consumer is `.github/workflows/e2e-smoke-shadow.yml`. Land this producer +before enabling that workflow; for paired branches, use the same branch name in +both repositories. A missing producer fails visibly. Reporting remains opt-in: +existing automatic gates and schedules continue unchanged. This stage does not +activate full nightly coverage, risk packs or the two-shard experiment. diff --git a/package-lock.json b/package-lock.json index 3053ca31..cc8aca94 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@radar/lnrpc": "^0.11.1-beta.1", "bip21": "^3.0.0", "bitcoin-json-rpc": "^1.4.1", + "bitcoinjs-lib": "6.1.7", "lnurl": "0.26.2", "rn-electrum-client": "^0.0.22" }, diff --git a/package.json b/package.json index 05fc3a90..d451fa81 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "scripts": { "inventory:check": "node --experimental-strip-types scripts/e2e-inventory.ts check", - "test:reporting": "node --experimental-strip-types --test test/unit/inventory.test.ts test/unit/reporting.test.ts", + "test:reporting": "node --experimental-strip-types --test test/unit/inventory.test.ts test/unit/reporting.test.ts test/unit/reporter-events.test.ts test/unit/smoke-oracle.test.ts", "e2e:android": "PLATFORM=android wdio run ./wdio.conf.ts", "e2e:ios": "PLATFORM=ios wdio run ./wdio.conf.ts", "lint": "eslint './**/*.{ts,js}'", @@ -35,6 +35,7 @@ "@radar/lnrpc": "^0.11.1-beta.1", "bip21": "^3.0.0", "bitcoin-json-rpc": "^1.4.1", + "bitcoinjs-lib": "6.1.7", "lnurl": "0.26.2", "rn-electrum-client": "^0.0.22" }, diff --git a/test/helpers/smoke-oracle.ts b/test/helpers/smoke-oracle.ts new file mode 100644 index 00000000..19e6282c --- /dev/null +++ b/test/helpers/smoke-oracle.ts @@ -0,0 +1,163 @@ +import crypto from 'node:crypto'; +import net from 'node:net'; +import { address, networks, payments, Transaction } from 'bitcoinjs-lib'; + +export type Endpoint = { host: string; port: number }; +export type PaymentEvidence = { + txid: string; + address: string; + sats: number; + rawTransaction: string; +}; + +export function freshRegtestRecipient(): string { + const key = crypto.createECDH('secp256k1'); + key.generateKeys(); + return payments.p2wpkh({ + pubkey: key.getPublicKey(undefined, 'compressed'), + network: networks.regtest, + }).address!; +} + +export function scriptHash(recipient: string): string { + return crypto + .createHash('sha256') + .update(address.toOutputScript(recipient, networks.regtest)) + .digest() + .reverse() + .toString('hex'); +} + +export function electrumRequest( + endpoint: Endpoint, + method: string, + params: unknown[], + timeoutMs = 10_000 +): Promise { + return new Promise((resolve, reject) => { + const socket = net.createConnection(endpoint); + let buffer = ''; + let settled = false; + const finish = (error?: Error, result?: unknown) => { + if (settled) return; + settled = true; + clearTimeout(timer); + socket.destroy(); + if (error) reject(error); + else resolve(result); + }; + const timer = setTimeout( + () => finish(new Error(`Smoke backend unavailable: ${method} timed out`)), + timeoutMs + ); + socket.setEncoding('utf8'); + socket.on('connect', () => + socket.write(JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }) + '\n') + ); + socket.on('error', (error) => finish(new Error(`Smoke backend unavailable: ${error.message}`))); + socket.on('close', () => + finish(new Error(`Smoke backend closed before responding to ${method}`)) + ); + socket.on('data', (chunk) => { + buffer += chunk; + if (buffer.length > 2_000_000) return finish(new Error('Oversized Electrum response')); + while (buffer.includes('\n')) { + const end = buffer.indexOf('\n'); + const line = buffer.slice(0, end); + buffer = buffer.slice(end + 1); + try { + const response = JSON.parse(line); + if (response.id !== 1) continue; + if (response.error) + return finish( + new Error(`Smoke backend rejected ${method}: ${JSON.stringify(response.error)}`) + ); + if (!Object.hasOwn(response, 'result')) + return finish(new Error('Malformed Electrum response')); + finish(undefined, response.result); + } catch (error) { + finish(error instanceof Error ? error : new Error(String(error))); + } + } + }); + }); +} + +export async function history( + endpoint: Endpoint, + recipient: string +): Promise<{ tx_hash: string; height: number }[]> { + const result = await electrumRequest(endpoint, 'blockchain.scripthash.get_history', [ + scriptHash(recipient), + ]); + if ( + !Array.isArray(result) || + result.some((r) => !r || !/^[a-f0-9]{64}$/.test(r.tx_hash) || !Number.isInteger(r.height)) + ) { + throw new Error('Malformed Electrum address history'); + } + return result; +} + +export function verifyPayment( + raw: string, + txid: string, + recipient: string, + sats: number, + spendsTxid?: string +): PaymentEvidence { + const tx = Transaction.fromHex(raw); + if (tx.getId() !== txid) throw new Error('Transaction identity does not match its raw bytes'); + const script = address.toOutputScript(recipient, networks.regtest); + const outputs = tx.outs.filter((output) => output.script.equals(script)); + if (outputs.length !== 1 || outputs[0].value !== sats) + throw new Error('Transaction recipient/amount does not match the requested payment'); + if ( + spendsTxid && + !tx.ins.some((input) => Buffer.from(input.hash).reverse().toString('hex') === spendsTxid) + ) { + throw new Error("Payment does not spend this test wallet's funding transaction"); + } + return { txid, address: recipient, sats, rawTransaction: raw }; +} + +export async function waitForPayment( + endpoint: Endpoint, + recipient: string, + sats: number, + options: { txid?: string; spendsTxid?: string; timeoutMs?: number } = {} +): Promise { + const deadline = Date.now() + (options.timeoutMs ?? 120_000); + while (Date.now() < deadline) { + const transactions = await history(endpoint, recipient); + const candidates = options.txid + ? transactions.filter((tx) => tx.tx_hash === options.txid) + : transactions; + if (candidates.length > 1) + throw new Error('Fresh recipient has multiple transactions; fixture isolation is broken'); + if (candidates.length === 1) { + const txid = candidates[0].tx_hash; + const raw = await electrumRequest(endpoint, 'blockchain.transaction.get', [txid, false]); + if (typeof raw !== 'string') throw new Error('Electrum did not return a raw transaction'); + return verifyPayment(raw, txid, recipient, sats, options.spendsTxid); + } + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + throw new Error(`Expected payment was not observed within ${options.timeoutMs ?? 120_000}ms`); +} + +export async function regtestOperation( + baseUrl: string, + operation: 'deposit' | 'mine', + body: Record +): Promise { + const response = await fetch(`${baseUrl}/regtest/chain/${operation}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) + throw new Error(`Smoke regtest ${operation} unavailable: HTTP ${response.status}`); + return (await response.text()).trim(); +} diff --git a/test/inventory.json b/test/inventory.json index 5bbb5b10..f92facb7 100644 --- a/test/inventory.json +++ b/test/inventory.json @@ -1572,6 +1572,63 @@ "fixtures": ["fresh-wallet"] } } + }, + { + "id": "smoke_wallet_1", + "file": "test/specs/smoke.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "smoke", + "disposition": "required", + "fixtures": ["fresh-wallet", "regtest-funding-api", "independent-electrum-oracle"] + }, + "android": { + "backend": "regtest", + "pack": "smoke", + "disposition": "required", + "fixtures": ["fresh-wallet", "regtest-funding-api", "independent-electrum-oracle"] + } + } + }, + { + "id": "smoke_receive_1", + "file": "test/specs/smoke.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "smoke", + "disposition": "required", + "fixtures": ["fresh-wallet", "regtest-funding-api", "independent-electrum-oracle"] + }, + "android": { + "backend": "regtest", + "pack": "smoke", + "disposition": "required", + "fixtures": ["fresh-wallet", "regtest-funding-api", "independent-electrum-oracle"] + } + } + }, + { + "id": "smoke_onchain_1", + "file": "test/specs/smoke.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "smoke", + "disposition": "required", + "fixtures": ["fresh-wallet", "regtest-funding-api", "independent-electrum-oracle"] + }, + "android": { + "backend": "regtest", + "pack": "smoke", + "disposition": "required", + "fixtures": ["fresh-wallet", "regtest-funding-api", "independent-electrum-oracle"] + } + } } ] } diff --git a/test/reporting/reporter.ts b/test/reporting/reporter.ts index b7105b27..1c1af7c3 100644 --- a/test/reporting/reporter.ts +++ b/test/reporting/reporter.ts @@ -1,6 +1,6 @@ import fs from 'node:fs'; import path from 'node:path'; -import WDIOReporter, { type RunnerStats, type TestStats } from '@wdio/reporter'; +import WDIOReporter, { type HookStats, type RunnerStats, type TestStats } from '@wdio/reporter'; import { caseId } from './inventory.ts'; import { digest, validatePlan, writeJson, type Plan, type WorkerReport } from './report.ts'; @@ -68,6 +68,22 @@ export default class InventoryReporter extends WDIOReporter { this.record(test, 'skipped'); } + onTestPending(test: TestStats) { + this.record(test, 'skipped'); + } + + onTestRetry(test: TestStats) { + this.report?.errors.push(`Undeclared framework retry: ${test.title}`); + this.flush(); + } + + onHookEnd(hook: HookStats) { + if (hook.error || hook.errors?.length || hook.state === 'failed') { + this.report?.errors.push(`Hook failed: ${hook.title}`); + this.flush(); + } + } + onRunnerEnd(runner: RunnerStats) { if (!this.report) throw new Error('Runner ended without starting'); this.report.failures = runner.failures ?? 0; diff --git a/test/specs/smoke.e2e.ts b/test/specs/smoke.e2e.ts new file mode 100644 index 00000000..a3592256 --- /dev/null +++ b/test/specs/smoke.e2e.ts @@ -0,0 +1,109 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { + acknowledgeReceivedPaymentIfPresent, + completeOnboarding, + dismissBackupTimedSheet, + dragOnElement, + elementById, + enterAddress, + enterAmount, + expectSavingsBalance, + expectTotalBalance, + getReceiveAddress, + getUriFromQRCode, + swipeFullScreen, + tap, +} from '../helpers/actions'; +import { blocktankURL, electrumHost, electrumPort, getBackend } from '../helpers/constants'; +import { reinstallApp } from '../helpers/setup'; +import { ciIt } from '../helpers/suite'; +import { + freshRegtestRecipient, + history, + regtestOperation, + scriptHash, + waitForPayment, +} from '../helpers/smoke-oracle'; +import { writeJson } from '../reporting/report'; + +describe('@ios_smoke - Focused wallet smoke (shadow)', () => { + const endpoint = { host: electrumHost, port: electrumPort }; + before(() => { + if (getBackend() !== 'regtest') + throw new Error('Smoke requires a matching regtest app build and BACKEND=regtest'); + }); + beforeEach(async () => { + await reinstallApp(); + await completeOnboarding(); + }); + + ciIt('@smoke_wallet_1 - A fresh wallet reaches usable Home', async () => { + await expectTotalBalance(0); + await elementById('Receive').waitForDisplayed(); + await elementById('Send').waitForDisplayed(); + }); + + ciIt('@smoke_receive_1 - Receive details encode the requested amount', async () => { + const recipient = await getReceiveAddress(); + assert.ok(recipient.startsWith('bcrt1')); + scriptHash(recipient); + await tap('ShowDetails'); + await elementById('ReceiveScreen').waitForDisplayed(); + await tap('QRCode'); + await tap('SpecifyInvoiceButton'); + await tap('ReceiveNumberPadTextField'); + await tap('N1'); + await tap('N2'); + await tap('N3'); + await tap('ReceiveNumberPadSubmit'); + await tap('ShowQrReceive'); + const uri = await getUriFromQRCode(); + const [encodedAddress, query] = uri.replace(/^bitcoin:/, '').split('?'); + assert.equal(encodedAddress, recipient); + const amount = new URLSearchParams(query).get('amount'); + assert.ok(amount, 'Receive QR must encode the specified amount'); + assert.equal(Number(amount) * 100_000_000, 123); + }); + + ciIt('@smoke_onchain_1 - Receive and send with independent transaction evidence', async () => { + const walletAddress = await getReceiveAddress(); + assert.equal((await history(endpoint, walletAddress)).length, 0, 'Wallet must be fresh'); + await swipeFullScreen('down'); + const depositResult = await regtestOperation(blocktankURL, 'deposit', { + address: walletAddress, + amountSat: 100_000, + }); + const fundingTxid = depositResult.startsWith('"') ? JSON.parse(depositResult) : depositResult; + assert.match(fundingTxid, /^[a-f0-9]{64}$/); + await regtestOperation(blocktankURL, 'mine', { count: 1 }); + const received = await waitForPayment(endpoint, walletAddress, 100_000, { txid: fundingTxid }); + await acknowledgeReceivedPaymentIfPresent(); + await expectTotalBalance(100_000); + await expectSavingsBalance(100_000); + await dismissBackupTimedSheet({ triggerTimedSheet: true }); + + const recipient = freshRegtestRecipient(); + assert.equal( + (await history(endpoint, recipient)).length, + 0, + 'Recipient must be unique to this case' + ); + await enterAddress(recipient); + await enterAmount(10_000); + await tap('ContinueAmount'); + await dragOnElement('GRAB', 'right', 0.95); + await elementById('SendSuccess').waitForDisplayed(); + await tap('Close'); + await regtestOperation(blocktankURL, 'mine', { count: 1 }); + const sent = await waitForPayment(endpoint, recipient, 10_000, { spendsTxid: fundingTxid }); + await expectSavingsBalance(100_000, { condition: 'lt' }); + const directory = process.env.E2E_PLAN + ? path.dirname(process.env.E2E_PLAN) + : 'artifacts/reporting'; + writeJson(path.join(directory, `payment-attempt-${Number(process.env.ATTEMPT || 1)}.json`), { + received, + sent, + }); + }); +}); diff --git a/test/unit/reporter-events.test.ts b/test/unit/reporter-events.test.ts new file mode 100644 index 00000000..73bd7100 --- /dev/null +++ b/test/unit/reporter-events.test.ts @@ -0,0 +1,118 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import InventoryReporter from '../reporting/reporter.ts'; +import { loadInventory } from '../reporting/inventory.ts'; +import { + createPlan, + digest, + reconcile, + writeJson, + type WorkerReport, +} from '../reporting/report.ts'; + +function fixture(t: test.TestContext) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'reporter-events-')); + const previous = { + E2E_PLAN: process.env.E2E_PLAN, + PLATFORM: process.env.PLATFORM, + BACKEND: process.env.BACKEND, + ATTEMPT: process.env.ATTEMPT, + }; + t.after(() => { + for (const [key, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + fs.rmSync(directory, { recursive: true, force: true }); + }); + const plan = createPlan( + loadInventory(process.cwd()).cases, + { + runId: 'reporter-integration', + platform: 'ios', + backend: 'regtest', + appSha: 'a'.repeat(40), + e2eSha: 'b'.repeat(40), + artifactDigest: 'c'.repeat(64), + backendRevision: 'stag0', + variant: 'default', + }, + digest(fs.readFileSync('test/inventory.json', 'utf8')), + 'smoke' + ); + process.env.E2E_PLAN = path.join(directory, 'plan.json'); + process.env.PLATFORM = 'ios'; + process.env.BACKEND = 'regtest'; + process.env.ATTEMPT = '1'; + writeJson(process.env.E2E_PLAN, plan); + const reporter = new InventoryReporter({ writeStream: { write: () => true } }); + reporter.emit('runner:start', { + cid: '0-0', + capabilities: { platformName: 'iOS' }, + config: {}, + specs: ['test/specs/smoke.e2e.ts'], + }); + const receipt = () => + JSON.parse( + fs.readFileSync(path.join(directory, 'workers/attempt-1-0-0.json'), 'utf8') + ) as WorkerReport; + const passAll = () => { + for (const id of plan.expected) { + const e = { uid: id, cid: '0-0', title: `@${id} - test`, fullTitle: `smoke @${id} - test` }; + reporter.emit('test:start', e); + reporter.emit('test:pass', e); + reporter.emit('test:end', e); + } + }; + return { reporter, receipt, plan, passAll }; +} + +test('actual WDIO reporter events produce a complete exact-case receipt', (t) => { + const f = fixture(t); + f.passAll(); + f.reporter.emit('runner:end', { failures: 0 }); + assert.equal(reconcile(f.plan, [f.receipt()]).status, 'passed'); +}); +test('Mocha pending events are reported as unexpected skips', (t) => { + const f = fixture(t); + const id = f.plan.expected[0]; + f.reporter.emit('test:pending', { + uid: id, + cid: '0-0', + title: `@${id} - skipped`, + fullTitle: `@${id} - skipped`, + state: 'pending', + }); + f.reporter.emit('runner:end', { failures: 0 }); + const result = reconcile(f.plan, [f.receipt()]); + assert.match(result.problems.join(' '), /Unexpected skip/); +}); +test('hook failure is recorded even if runner failure count is zero', (t) => { + const f = fixture(t); + f.passAll(); + f.reporter.emit('hook:start', { uid: 'after', cid: '0-0', title: 'after hook' }); + f.reporter.emit('hook:end', { uid: 'after', errors: [new Error('teardown failed')] }); + f.reporter.emit('runner:end', { failures: 0 }); + const result = reconcile(f.plan, [f.receipt()]); + assert.equal(result.status, 'incomplete'); + assert.match(result.problems.join(' '), /Hook failed/); +}); +test('interruption retains incremental case evidence without a successful finish', (t) => { + const f = fixture(t); + f.passAll(); + assert.equal(f.receipt().events.length, 3); + assert.equal(reconcile(f.plan, [f.receipt()]).status, 'incomplete'); +}); +test('framework retries cannot silently turn one-attempt smoke green', (t) => { + const f = fixture(t); + const id = f.plan.expected[0]; + const event = { uid: id, cid: '0-0', title: `@${id} - retry` }; + f.reporter.emit('test:start', event); + f.reporter.emit('test:retry', { ...event, errors: [new Error('first attempt')] }); + f.passAll(); + f.reporter.emit('runner:end', { failures: 0 }); + assert.equal(reconcile(f.plan, [f.receipt()]).status, 'incomplete'); +}); diff --git a/test/unit/smoke-oracle.test.ts b/test/unit/smoke-oracle.test.ts new file mode 100644 index 00000000..dc411f3a --- /dev/null +++ b/test/unit/smoke-oracle.test.ts @@ -0,0 +1,118 @@ +import assert from 'node:assert/strict'; +import net from 'node:net'; +import test from 'node:test'; +import { address, networks, Transaction } from 'bitcoinjs-lib'; +import { + electrumRequest, + freshRegtestRecipient, + history, + scriptHash, + verifyPayment, + waitForPayment, +} from '../helpers/smoke-oracle.ts'; + +function transaction(recipient: string, sats = 10_000, funding = 'ab'.repeat(32)) { + const tx = new Transaction(); + tx.addInput(Buffer.from(funding, 'hex').reverse(), 0); + tx.addOutput(address.toOutputScript(recipient, networks.regtest), sats); + return tx; +} +async function server( + t: test.TestContext, + handle: (socket: net.Socket, request: { id: number; method: string; params: unknown[] }) => void +) { + const sockets = new Set(); + const server = net.createServer((socket) => { + sockets.add(socket); + socket.on('close', () => sockets.delete(socket)); + let data = ''; + socket.on('data', (chunk) => { + data += chunk; + if (data.includes('\n')) handle(socket, JSON.parse(data)); + }); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + t.after(async () => { + for (const s of sockets) s.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + }); + return { host: '127.0.0.1', port: (server.address() as net.AddressInfo).port }; +} +test('fresh recipients are distinct valid regtest addresses', () => { + const one = freshRegtestRecipient(), + two = freshRegtestRecipient(); + assert.notEqual(one, two); + assert.match(one, /^bcrt1/); + assert.match(scriptHash(one), /^[a-f0-9]{64}$/); +}); +test('oracle verifies raw transaction identity, exact output and wallet funding input', () => { + const recipient = freshRegtestRecipient(), + tx = transaction(recipient); + assert.equal( + verifyPayment(tx.toHex(), tx.getId(), recipient, 10_000, 'ab'.repeat(32)).txid, + tx.getId() + ); + assert.throws(() => verifyPayment(tx.toHex(), 'cd'.repeat(32), recipient, 10_000), /identity/); + assert.throws( + () => verifyPayment(tx.toHex(), tx.getId(), freshRegtestRecipient(), 10_000), + /recipient\/amount/ + ); + assert.throws(() => verifyPayment(tx.toHex(), tx.getId(), recipient, 9_999), /recipient\/amount/); + assert.throws( + () => verifyPayment(tx.toHex(), tx.getId(), recipient, 10_000, 'cd'.repeat(32)), + /funding transaction/ + ); +}); +test('unrelated concurrent transactions cannot satisfy another case', () => { + const first = freshRegtestRecipient(), + second = freshRegtestRecipient(); + const firstTx = transaction(first), + secondTx = transaction(second, 10_000, 'cd'.repeat(32)); + assert.throws( + () => verifyPayment(secondTx.toHex(), secondTx.getId(), first, 10_000, 'ab'.repeat(32)), + /recipient/ + ); + assert.equal( + verifyPayment(firstTx.toHex(), firstTx.getId(), first, 10_000, 'ab'.repeat(32)).address, + first + ); +}); +test('split TCP responses and notifications are handled without accepting another request ID', async (t) => { + const endpoint = await server(t, (socket) => { + socket.write('{"id":2,"result":"wrong"}\n{"id":1,"res'); + socket.end('ult":"correct"}\n'); + }); + assert.equal(await electrumRequest(endpoint, 'server.version', []), 'correct'); +}); +test('a nonresponding backend is bounded and closes the connection', async (t) => { + const endpoint = await server(t, () => {}); + await assert.rejects(electrumRequest(endpoint, 'server.version', [], 30), /timed out/); +}); +test('closed connections and backend errors cannot become successful readiness', async (t) => { + const closed = await server(t, (socket) => socket.end()); + await assert.rejects(electrumRequest(closed, 'server.version', []), /closed before/); + const error = await server(t, (socket) => + socket.end('{"id":1,"error":{"code":-1,"message":"unavailable"}}\n') + ); + await assert.rejects(electrumRequest(error, 'server.version', []), /backend rejected/); +}); +test('malformed history fails rather than looking like an empty ready wallet', async (t) => { + const endpoint = await server(t, (socket) => socket.end('{"id":1,"result":null}\n')); + await assert.rejects(history(endpoint, freshRegtestRecipient()), /Malformed.*history/); +}); +test('end-to-end oracle reads history then independently decodes the payment', async (t) => { + const recipient = freshRegtestRecipient(), + tx = transaction(recipient); + const endpoint = await server(t, (socket, request) => + socket.end( + JSON.stringify({ + id: 1, + result: request.method.endsWith('get_history') + ? [{ tx_hash: tx.getId(), height: 1 }] + : tx.toHex(), + }) + '\n' + ) + ); + const result = await waitForPayment(endpoint, recipient, 10_000, { spendsTxid: 'ab'.repeat(32) }); + assert.equal(result.txid, tx.getId()); +}); diff --git a/wdio.conf.ts b/wdio.conf.ts index 53abf89c..e8a69069 100644 --- a/wdio.conf.ts +++ b/wdio.conf.ts @@ -382,6 +382,20 @@ export const config: WebdriverIO.Config = { console.log(`🧪 Start: ${test.parent} - ${test.title}`); }, + afterHook: async function (_test, _context, { error }) { + if (!process.env.E2E_PLAN || !error) return; + try { + const directory = path.join(path.dirname(process.env.E2E_PLAN), 'hook-failures'); + fs.mkdirSync(directory, { recursive: true }); + const screenshot = await driver.takeScreenshot(); + fs.writeFileSync(path.join(directory, `${Date.now()}.png`), screenshot, 'base64'); + } catch { + console.warn( + 'Hook failed before a screenshot could be collected; retain runner and simulator logs.' + ); + } + }, + afterTest: async function (test, _context, { error }) { const saveDiagnostics = Boolean(error) || process.env.SAVE_DIAGNOSTICS === 'true'; From 9d08c2f4f853d993ca3a3ffa8285d1bf9cc106d2 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Tue, 3 Nov 2026 19:30:37 +0100 Subject: [PATCH 03/18] test: fix smoke precision, TLS and execution identity boundaries --- docs/inventory-reporting.md | 2 + package.json | 2 +- scripts/e2e-inventory.ts | 14 ++----- test/helpers/smoke-oracle.ts | 23 ++++++++++-- test/reporting/identity.ts | 31 ++++++++++++++++ test/reporting/reporter.ts | 2 + test/specs/smoke.e2e.ts | 5 ++- test/unit/reporter-events.test.ts | 59 ++++++++++++++++++++++++----- test/unit/reporting-cli.test.ts | 62 +++++++++++++++++++++++++++++++ test/unit/smoke-oracle.test.ts | 38 ++++++++++++++++++- 10 files changed, 210 insertions(+), 28 deletions(-) create mode 100644 test/reporting/identity.ts create mode 100644 test/unit/reporting-cli.test.ts diff --git a/docs/inventory-reporting.md b/docs/inventory-reporting.md index ae99b17b..4cbb1808 100644 --- a/docs/inventory-reporting.md +++ b/docs/inventory-reporting.md @@ -29,6 +29,8 @@ is enabled only when `E2E_PLAN` points to a prepared plan. Use a fresh directory each run; plans and worker files cannot overwrite an earlier run. For example: ```sh +export PLATFORM=ios +export BACKEND=regtest export E2E_RUN_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-ios-smoke" export E2E_APP_SHA='' export E2E_ARTIFACT_DIGEST='' diff --git a/package.json b/package.json index d451fa81..932c3868 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "scripts": { "inventory:check": "node --experimental-strip-types scripts/e2e-inventory.ts check", - "test:reporting": "node --experimental-strip-types --test test/unit/inventory.test.ts test/unit/reporting.test.ts test/unit/reporter-events.test.ts test/unit/smoke-oracle.test.ts", + "test:reporting": "node --experimental-strip-types --test test/unit/inventory.test.ts test/unit/reporting.test.ts test/unit/reporting-cli.test.ts test/unit/reporter-events.test.ts test/unit/smoke-oracle.test.ts", "e2e:android": "PLATFORM=android wdio run ./wdio.conf.ts", "e2e:ios": "PLATFORM=ios wdio run ./wdio.conf.ts", "lint": "eslint './**/*.{ts,js}'", diff --git a/scripts/e2e-inventory.ts b/scripts/e2e-inventory.ts index b241dd10..65ed4b67 100644 --- a/scripts/e2e-inventory.ts +++ b/scripts/e2e-inventory.ts @@ -1,6 +1,6 @@ import fs from 'node:fs'; import path from 'node:path'; -import { execFileSync } from 'node:child_process'; +import { assertExecutionIdentity, executionIdentity } from '../test/reporting/identity.ts'; import { parseArgs } from 'node:util'; import { loadInventory, @@ -39,16 +39,7 @@ try { throw new Error('Plan already exists; use a fresh run directory'); const plan = createPlan( inventory.cases, - { - runId: process.env.E2E_RUN_ID!, - platform: values.platform as Platform, - backend: values.backend as Backend, - appSha: process.env.E2E_APP_SHA!, - e2eSha: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), - artifactDigest: process.env.E2E_ARTIFACT_DIGEST!, - backendRevision: process.env.E2E_BACKEND_REVISION!, - variant: process.env.E2E_VARIANT || 'default', - }, + executionIdentity(values.platform as Platform, values.backend as Backend), digest(fs.readFileSync('test/inventory.json', 'utf8')), values.pack!, Number(process.env.E2E_MAX_ATTEMPTS || '1') @@ -60,6 +51,7 @@ try { } } else if (command === 'report') { const plan: Plan = JSON.parse(fs.readFileSync(values.plan!, 'utf8')); + assertExecutionIdentity(plan); const directory = path.join(path.dirname(values.plan!), 'workers'); const reports: WorkerReport[] = fs.existsSync(directory) ? fs diff --git a/test/helpers/smoke-oracle.ts b/test/helpers/smoke-oracle.ts index 19e6282c..1db59992 100644 --- a/test/helpers/smoke-oracle.ts +++ b/test/helpers/smoke-oracle.ts @@ -1,8 +1,9 @@ import crypto from 'node:crypto'; import net from 'node:net'; +import tls from 'node:tls'; import { address, networks, payments, Transaction } from 'bitcoinjs-lib'; -export type Endpoint = { host: string; port: number }; +export type Endpoint = { host: string; port: number; protocol: 'tls' | 'tcp' }; export type PaymentEvidence = { txid: string; address: string; @@ -35,7 +36,14 @@ export function electrumRequest( timeoutMs = 10_000 ): Promise { return new Promise((resolve, reject) => { - const socket = net.createConnection(endpoint); + const socket = + endpoint.protocol === 'tls' + ? tls.connect({ + host: endpoint.host, + port: endpoint.port, + servername: net.isIP(endpoint.host) ? undefined : endpoint.host, + }) + : net.createConnection({ host: endpoint.host, port: endpoint.port }); let buffer = ''; let settled = false; const finish = (error?: Error, result?: unknown) => { @@ -51,7 +59,7 @@ export function electrumRequest( timeoutMs ); socket.setEncoding('utf8'); - socket.on('connect', () => + socket.on(endpoint.protocol === 'tls' ? 'secureConnect' : 'connect', () => socket.write(JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }) + '\n') ); socket.on('error', (error) => finish(new Error(`Smoke backend unavailable: ${error.message}`))); @@ -161,3 +169,12 @@ export async function regtestOperation( throw new Error(`Smoke regtest ${operation} unavailable: HTTP ${response.status}`); return (await response.text()).trim(); } + +/** Parse BIP21 decimal BTC exactly; never round fractional satoshis into a pass. */ +export function bitcoinAmountToSats(value: string): number { + const match = /^(0|[1-9][0-9]*)(?:\.([0-9]{1,8}))?$/.exec(value); + if (!match) throw new Error('Invalid BIP21 amount'); + const sats = BigInt(match[1]) * 100_000_000n + BigInt((match[2] || '').padEnd(8, '0')); + if (sats > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error('BIP21 amount exceeds safe range'); + return Number(sats); +} diff --git a/test/reporting/identity.ts b/test/reporting/identity.ts new file mode 100644 index 00000000..64ebd7a1 --- /dev/null +++ b/test/reporting/identity.ts @@ -0,0 +1,31 @@ +import { execFileSync } from 'node:child_process'; +import type { Identity, Plan } from './report.ts'; + +/** Current execution inputs supplied by the builder plus the actual checked-out E2E revision. */ +export function executionIdentity( + platform: Identity['platform'], + backend: Identity['backend'] +): Identity { + return { + runId: process.env.E2E_RUN_ID!, + platform, + backend, + appSha: process.env.E2E_APP_SHA!, + e2eSha: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), + artifactDigest: process.env.E2E_ARTIFACT_DIGEST!, + backendRevision: process.env.E2E_BACKEND_REVISION!, + variant: process.env.E2E_VARIANT || 'default', + }; +} + +export function assertExecutionIdentity(plan: Plan): void { + const current = executionIdentity( + process.env.PLATFORM as Identity['platform'], + (process.env.BACKEND || 'local') as Identity['backend'] + ); + for (const key of Object.keys(current) as (keyof Identity)[]) { + if (current[key] !== plan.identity[key]) { + throw new Error(`Run plan does not match executing ${key}`); + } + } +} diff --git a/test/reporting/reporter.ts b/test/reporting/reporter.ts index 1c1af7c3..a7298db3 100644 --- a/test/reporting/reporter.ts +++ b/test/reporting/reporter.ts @@ -1,6 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; import WDIOReporter, { type HookStats, type RunnerStats, type TestStats } from '@wdio/reporter'; +import { assertExecutionIdentity } from './identity.ts'; import { caseId } from './inventory.ts'; import { digest, validatePlan, writeJson, type Plan, type WorkerReport } from './report.ts'; @@ -13,6 +14,7 @@ export default class InventoryReporter extends WDIOReporter { super({ ...options, stdout: false }); this.plan = JSON.parse(fs.readFileSync(process.env.E2E_PLAN!, 'utf8')); validatePlan(this.plan); + assertExecutionIdentity(this.plan); if ( this.plan.identity.platform !== process.env.PLATFORM || this.plan.identity.backend !== (process.env.BACKEND || 'local') || diff --git a/test/specs/smoke.e2e.ts b/test/specs/smoke.e2e.ts index a3592256..5a9d4c3f 100644 --- a/test/specs/smoke.e2e.ts +++ b/test/specs/smoke.e2e.ts @@ -19,6 +19,7 @@ import { blocktankURL, electrumHost, electrumPort, getBackend } from '../helpers import { reinstallApp } from '../helpers/setup'; import { ciIt } from '../helpers/suite'; import { + bitcoinAmountToSats, freshRegtestRecipient, history, regtestOperation, @@ -28,7 +29,7 @@ import { import { writeJson } from '../reporting/report'; describe('@ios_smoke - Focused wallet smoke (shadow)', () => { - const endpoint = { host: electrumHost, port: electrumPort }; + const endpoint = { host: electrumHost, port: electrumPort, protocol: 'tls' as const }; before(() => { if (getBackend() !== 'regtest') throw new Error('Smoke requires a matching regtest app build and BACKEND=regtest'); @@ -63,7 +64,7 @@ describe('@ios_smoke - Focused wallet smoke (shadow)', () => { assert.equal(encodedAddress, recipient); const amount = new URLSearchParams(query).get('amount'); assert.ok(amount, 'Receive QR must encode the specified amount'); - assert.equal(Number(amount) * 100_000_000, 123); + assert.equal(bitcoinAmountToSats(amount), 123); }); ciIt('@smoke_onchain_1 - Receive and send with independent transaction evidence', async () => { diff --git a/test/unit/reporter-events.test.ts b/test/unit/reporter-events.test.ts index 73bd7100..289663aa 100644 --- a/test/unit/reporter-events.test.ts +++ b/test/unit/reporter-events.test.ts @@ -3,6 +3,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import test from 'node:test'; +import { executionIdentity } from '../reporting/identity.ts'; import InventoryReporter from '../reporting/reporter.ts'; import { loadInventory } from '../reporting/inventory.ts'; import { @@ -20,6 +21,11 @@ function fixture(t: test.TestContext) { PLATFORM: process.env.PLATFORM, BACKEND: process.env.BACKEND, ATTEMPT: process.env.ATTEMPT, + E2E_RUN_ID: process.env.E2E_RUN_ID, + E2E_APP_SHA: process.env.E2E_APP_SHA, + E2E_ARTIFACT_DIGEST: process.env.E2E_ARTIFACT_DIGEST, + E2E_BACKEND_REVISION: process.env.E2E_BACKEND_REVISION, + E2E_VARIANT: process.env.E2E_VARIANT, }; t.after(() => { for (const [key, value] of Object.entries(previous)) { @@ -28,18 +34,16 @@ function fixture(t: test.TestContext) { } fs.rmSync(directory, { recursive: true, force: true }); }); + Object.assign(process.env, { + E2E_RUN_ID: 'reporter-integration', + E2E_APP_SHA: 'a'.repeat(40), + E2E_ARTIFACT_DIGEST: 'c'.repeat(64), + E2E_BACKEND_REVISION: 'stag0', + E2E_VARIANT: 'default', + }); const plan = createPlan( loadInventory(process.cwd()).cases, - { - runId: 'reporter-integration', - platform: 'ios', - backend: 'regtest', - appSha: 'a'.repeat(40), - e2eSha: 'b'.repeat(40), - artifactDigest: 'c'.repeat(64), - backendRevision: 'stag0', - variant: 'default', - }, + executionIdentity('ios', 'regtest'), digest(fs.readFileSync('test/inventory.json', 'utf8')), 'smoke' ); @@ -116,3 +120,38 @@ test('framework retries cannot silently turn one-attempt smoke green', (t) => { f.reporter.emit('runner:end', { failures: 0 }); assert.equal(reconcile(f.plan, [f.receipt()]).status, 'incomplete'); }); + +test('old plans reject changed current run, app, artifact, backend, platform or variant inputs', (t) => { + fixture(t); + for (const key of [ + 'E2E_RUN_ID', + 'E2E_APP_SHA', + 'E2E_ARTIFACT_DIGEST', + 'E2E_BACKEND_REVISION', + 'E2E_VARIANT', + 'BACKEND', + 'PLATFORM', + ]) { + const value = process.env[key]; + process.env[key] = 'changed'; + assert.throws( + () => new InventoryReporter({ writeStream: { write: () => true } }), + /Run plan does not match executing/ + ); + process.env[key] = value; + } +}); +test('old plan rejects changed actual E2E source revision even with unchanged inventory', (t) => { + const f = fixture(t); + const stale = createPlan( + f.plan.cases, + { ...f.plan.identity, e2eSha: 'f'.repeat(40) }, + f.plan.inventoryDigest, + f.plan.pack + ); + writeJson(process.env.E2E_PLAN!, stale); + assert.throws( + () => new InventoryReporter({ writeStream: { write: () => true } }), + /executing e2eSha/ + ); +}); diff --git a/test/unit/reporting-cli.test.ts b/test/unit/reporting-cli.test.ts new file mode 100644 index 00000000..aa4fd6f4 --- /dev/null +++ b/test/unit/reporting-cli.test.ts @@ -0,0 +1,62 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import test from 'node:test'; +import { writeJson } from '../reporting/report.ts'; + +test('real CLI rejects a retained successful plan under a replacement execution', (t) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'reporting-cli-')); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const planPath = path.join(directory, 'plan.json'); + const env = { + ...process.env, + PLATFORM: 'ios', + BACKEND: 'regtest', + E2E_RUN_ID: 'cli-original', + E2E_APP_SHA: 'a'.repeat(40), + E2E_ARTIFACT_DIGEST: 'b'.repeat(64), + E2E_BACKEND_REVISION: 'stag0', + E2E_VARIANT: 'default', + GITHUB_STEP_SUMMARY: path.join(directory, 'github.md'), + }; + const invoke = (args: string[]) => + spawnSync( + process.execPath, + ['--experimental-strip-types', 'scripts/e2e-inventory.ts', ...args, '--plan', planPath], + { env, encoding: 'utf8' } + ); + const preparation = invoke([ + 'plan', + '--platform', + 'ios', + '--backend', + 'regtest', + '--pack', + 'smoke', + ]); + assert.equal(preparation.status, 0, preparation.stderr); + const plan = JSON.parse(fs.readFileSync(planPath, 'utf8')); + writeJson(path.join(directory, 'workers/attempt-1-0-0.json'), { + schemaVersion: 1, + planId: plan.id, + attempt: 1, + cid: '0-0', + startedAt: new Date().toISOString(), + finishedAt: new Date().toISOString(), + failures: 0, + errors: [], + events: plan.expected.map((id: string) => ({ id, status: 'passed', durationMs: 1 })), + }); + const success = invoke(['report']); + assert.equal(success.status, 0, success.stderr); + assert.equal( + JSON.parse(fs.readFileSync(path.join(directory, 'summary.json'), 'utf8')).status, + 'passed' + ); + env.E2E_RUN_ID = 'cli-replacement'; + const stale = invoke(['report']); + assert.equal(stale.status, 1); + assert.match(stale.stderr, /does not match executing runId/); +}); diff --git a/test/unit/smoke-oracle.test.ts b/test/unit/smoke-oracle.test.ts index dc411f3a..3933c6d3 100644 --- a/test/unit/smoke-oracle.test.ts +++ b/test/unit/smoke-oracle.test.ts @@ -3,6 +3,7 @@ import net from 'node:net'; import test from 'node:test'; import { address, networks, Transaction } from 'bitcoinjs-lib'; import { + bitcoinAmountToSats, electrumRequest, freshRegtestRecipient, history, @@ -36,7 +37,11 @@ async function server( for (const s of sockets) s.destroy(); await new Promise((resolve) => server.close(() => resolve())); }); - return { host: '127.0.0.1', port: (server.address() as net.AddressInfo).port }; + return { + host: '127.0.0.1', + port: (server.address() as net.AddressInfo).port, + protocol: 'tcp' as const, + }; } test('fresh recipients are distinct valid regtest addresses', () => { const one = freshRegtestRecipient(), @@ -116,3 +121,34 @@ test('end-to-end oracle reads history then independently decodes the payment', a const result = await waitForPayment(endpoint, recipient, 10_000, { spendsTxid: 'ab'.repeat(32) }); assert.equal(result.txid, tx.getId()); }); + +test('BIP21 amounts compare exact satoshis without floating-point rounding', () => { + assert.equal(bitcoinAmountToSats('0.00000123'), 123); + assert.equal(bitcoinAmountToSats('0.00000001'), 1); + assert.equal(bitcoinAmountToSats('1.1'), 110_000_000); + assert.notEqual(bitcoinAmountToSats('0.00000124'), 123); + for (const value of ['1e-8', '-0.1', '0.000000001', 'NaN', '', '1.2x', '999999999999999']) { + assert.throws(() => bitcoinAmountToSats(value), /BIP21/); + } +}); +test('TLS endpoints send a TLS handshake instead of plaintext RPC', async (t) => { + let firstByte: number | undefined; + const listener = net.createServer((socket) => + socket.once('data', (data) => { + firstByte = data[0]; + socket.destroy(); + }) + ); + await new Promise((resolve) => listener.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise((resolve) => listener.close(() => resolve()))); + await assert.rejects( + electrumRequest( + { host: '127.0.0.1', port: (listener.address() as net.AddressInfo).port, protocol: 'tls' }, + 'server.version', + [], + 1000 + ), + /Smoke backend/ + ); + assert.equal(firstByte, 0x16, 'First record must be TLS handshake, not plaintext JSON'); +}); From b49b22bc03f46ca7bce00d67fd14c077e48d705d Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Tue, 3 Nov 2026 19:59:47 +0100 Subject: [PATCH 04/18] docs: validate smoke manually before automatic gate cutover --- docs/inventory-reporting.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/inventory-reporting.md b/docs/inventory-reporting.md index 4cbb1808..b2265be5 100644 --- a/docs/inventory-reporting.md +++ b/docs/inventory-reporting.md @@ -68,8 +68,9 @@ An empty selection fails rather than becoming an empty green run. ## Stages 1. Land the inventory/reporting producer and its validation checks first. -2. Add the iOS smoke workflow in shadow using the producer's companion branch or a - revision containing these files. Keep the current required gates and schedules. +2. Add the iOS smoke workflow as manual-only validation using the producer's + companion branch or a revision containing these files. Keep the current required + gates and schedules; do not run both gates automatically on each PR. 3. Measure real smoke duration, concurrent-run isolation and first-pass reliability before proposing any required-check change. The iOS two-shard pilot, automatic PR risk selection and full nightly orchestration remain separate stages. @@ -92,3 +93,7 @@ before enabling that workflow; for paired branches, use the same branch name in both repositories. A missing producer fails visibly. Reporting remains opt-in: existing automatic gates and schedules continue unchanged. This stage does not activate full nightly coverage, risk packs or the two-shard experiment. + +The iOS smoke consumer has only a manual dispatch trigger during validation. After +validation, switch the automatic PR gate in one change, without an automatic +overlap period. The lightweight shared inventory check can continue automatically. From 49d99b37d3bb97132632b9af302e093fad8f50ec Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Tue, 3 Nov 2026 20:55:02 +0100 Subject: [PATCH 05/18] ci: select iOS risk coverage and retain retry evidence --- .github/workflows/inventory.yml | 2 + AGENTS.md | 24 +-- docs/inventory-reporting.md | 146 +++++++----------- package.json | 2 +- scripts/ci-run.ts | 242 ++++++++++++++++++++++++++++++ scripts/ci-selection.cjs | 198 ++++++++++++++++++++++++ test/helpers/smoke-oracle.ts | 180 ---------------------- test/inventory.json | 115 ++++---------- test/reporting/report.ts | 60 +++++++- test/specs/smoke.e2e.ts | 110 -------------- test/unit/ci-retry.test.ts | 166 ++++++++++++++++++++ test/unit/ci-selection.test.cjs | 77 ++++++++++ test/unit/reporter-events.test.ts | 14 +- test/unit/reporting-cli.test.ts | 6 +- test/unit/reporting.test.ts | 16 +- test/unit/smoke-oracle.test.ts | 154 ------------------- 16 files changed, 846 insertions(+), 666 deletions(-) create mode 100644 scripts/ci-run.ts create mode 100644 scripts/ci-selection.cjs delete mode 100644 test/helpers/smoke-oracle.ts delete mode 100644 test/specs/smoke.e2e.ts create mode 100644 test/unit/ci-retry.test.ts create mode 100644 test/unit/ci-selection.test.cjs delete mode 100644 test/unit/smoke-oracle.test.ts diff --git a/.github/workflows/inventory.yml b/.github/workflows/inventory.yml index 86260a03..4df92bb7 100644 --- a/.github/workflows/inventory.yml +++ b/.github/workflows/inventory.yml @@ -5,6 +5,7 @@ on: paths: - 'test/**' - 'scripts/e2e-inventory.ts' + - 'scripts/ci-*' - 'wdio.conf.ts' - 'package*.json' - '.github/workflows/inventory.yml' @@ -13,6 +14,7 @@ on: paths: - 'test/**' - 'scripts/e2e-inventory.ts' + - 'scripts/ci-*' - 'wdio.conf.ts' - 'package*.json' - '.github/workflows/inventory.yml' diff --git a/AGENTS.md b/AGENTS.md index c4cf190c..87ad379a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -169,26 +169,10 @@ Implication for feature work: - Keep Android/iOS platform differences behind helpers in `test/helpers/`. - Prefer extracting shared flows into `test/helpers/` over copying logic between specs. Keep helpers small and reuse existing ones before adding new test code. -## Local vs staging tags +## iOS CI selection -The app merge gate (`e2e.yml` / `e2e-tests.yml`) runs `BACKEND=local` (docker Electrum/LND). Staging (`e2e-staging.yml`) and migration (`e2e_migration.yml`) run `BACKEND=regtest` against stag0. +The iOS consumer selects cases from `test/inventory.json` using `scripts/ci-selection.cjs`, not the historical `@ios_gate` and `@ios_nightly` tags. Its base cases are `send_1`, `send_2`, `lightning_1`, `transfer_2`, `boost_2`; automatic risk selection expands coverage. Nightly runs all applicable local/staging/migration cases. See [inventory and reporting](docs/inventory-reporting.md). -Capability tags (`@settings`, `@boost`, `@send`, …) stay on titles so Android local can keep grepping them. iOS queue tags are disjoint from `@staging` and from `*_staging` (mocha grep is a regex substring). +Keep capability tags for Android's existing selectors and manual execution. Android workflows and selections are unchanged. Do not rename tags merely to match the iOS policy. -| Tag | Android local | iOS Mini (`e2e-tests.yml`) | Staging (`e2e-staging.yml`) | -|---|---|---|---| -| `@ios_gate` | ignored | merge gate grep | no | -| `@ios_nightly` | ignored (still run via `@settings` etc.) | no | extra iOS shard (`ios_nightly_ui`), GitHub mac, not Mini | -| `@transfer_1`, `@transfer_max` | no | no | `@transfer_staging` (keep `@transfer_1` / `@transfer_max` / `@transfer`). `@transfer_1` is not in app `e2e-staging.yml` yet — add the transfer shard once it is 3× green. | -| `@multi_address_2` | no | no | `@multi_address_staging` only | -| `@pubky` / `@paykit` / `@pubky_profile` | no | no | `@pubky_staging` (keep capability tags for other greps) | -| `@boost`, `@multi_address_3` | yes | `@ios_gate` (needs exclusive miner) | no | -| `@send_2`, `@send_3`, `@lightning`, `@lnurl`, `@transfer_2`, `@multi_address_4` | yes | `@ios_gate` (local LND) | no | -| `@hardware_wallet` | no (staging only) | `@ios_gate` | Android staging greps `@hardware_wallet`; do **not** add `@hardware_wallet_staging`. iOS Mini greps `@ios_gate` on this describe. No Trezor on GitHub mac. | -| `@onboarding`, `@backup`, `@receive`, `@onchain_1`, `@onchain_2`, `@send_1`, `@multi_address_1`, `@settings`, `@numberpad`, `@widgets`, `@security` | yes | `@ios_nightly` | iOS nightly shard | -| `@onchain_3` | yes (`@onchain`) | no | no | -| `@migration_*` | migration workflow (nightly, dispatch, `release-*` PRs) | same | no | - -Put `@ios_gate` / `@ios_nightly` on `describe` when the whole file is one iOS queue; on `ciIt` when siblings split (`@send_1` vs `@send_2`/`_3`, `@multi_address_1` vs `_3`/`_4`). Local-LND specs (`@send_3`, `@multi_address_4`) stay `@ios_gate` — they cannot run on the regtest nightly shard. Mini keeps boost/RBF, local-LND money path, and hardware wallet. - -Poke staging with `gh workflow run e2e-staging.yml` on the app repo (does not queue the iOS Mini). Optional Slack post to `#bitkit-staging-nightly` via dispatch `post_to_slack`. +Inventory-enabled iOS execution uses `scripts/ci-run.ts`: up to three attempts, complete selected pack per attempt, stop after exact successful coverage, first-failure evidence retained. This bypasses legacy `/tmp/lock`; Android non-inventory retries retain their existing behavior. Plans and receipts must identify the app, test source, artifact, backend and run. Never reuse a previous night's pass as current execution. diff --git a/docs/inventory-reporting.md b/docs/inventory-reporting.md index b2265be5..acd67c12 100644 --- a/docs/inventory-reporting.md +++ b/docs/inventory-reporting.md @@ -1,99 +1,55 @@ -# Inventory and reporting rollout (#264) - -The versioned `test/inventory.json` accounts for each source case on iOS and Android, -including setup utilities, conditionally registered migrations, currently skipped -cases and unsupported automation. It describes intended coverage, not proof that all -nightly jobs currently exist. A `gap` remains incomplete coverage; an inventory check -passing means the inventory is consistent with source, not that the tests passed. - -Run `npm run inventory:check` with Node 22.18+ or Node 24. Discovery reads TypeScript -syntax without importing specs or executing hooks, and is independent of CI greps. -Case titles start with one stable `@case_id`. Literal templates can interpolate -finite loop values or literal factory arguments in that ID. Unknown dynamic IDs, -untagged cases, duplicate IDs and focused tests fail validation. Keep each new/renamed -case and its two platform assignments in the same change. Cases must be declared -with `ciIt`/`it` rather than aliases or computed registration functions. - -The initial destinations retain iOS remote-regtest UI, local money/mining and -hardware jobs, Android local UI/money and staging hardware, staging Pubky/Paykit, -migration variants and private mainnet orchestration. `settings_10` and `onchain_3` -need local execution on both platforms. Mainnet iOS support and currently skipped -cases are explicit gaps linked to #264, not invented platform exemptions. -Fixture flags in the manifest are requirements, not secrets or actual fixture values. -Migration source/target versions must be supplied in each run's `E2E_VARIANT`. - -## Opt-in execution - -Existing invocations and gates keep their current selectors and verdicts. Reporting -is enabled only when `E2E_PLAN` points to a prepared plan. Use a fresh directory for -each run; plans and worker files cannot overwrite an earlier run. For example: +# iOS E2E selection and reporting + +Issue: https://github.com/synonymdev/bitkit-e2e-tests/issues/264 + +The iOS `e2e-tests.yml` entry point owns PR selection, manual validation and the nightly schedule. Existing staging and migration workflows are reusable execution lanes. There is no shadow gate. Android workflow triggers, test selection, backend assignments and legacy retry behavior remain unchanged. + +## PR selection + +Every iOS app-code PR starts with these existing cases, unchanged: + +- `send_1`, `send_2`, `lightning_1`, `transfer_2`, `boost_2`. + +`scripts/ci-selection.cjs` adds cases by changed path. Hardware adds active hardware tests; LNURL adds its case; address/coin-selection adds multi-address, on-chain and boost tests; fees add boost/on-chain/send; onboarding/backup/recovery adds its tests and migration; security adds security and payment checks; Pubky/Paykit/profile/contact changes add the staging pack; receive/send/lightning/settings/widgets add the corresponding cases. Rules are additive and IDs are deduplicated. + +Shared services, repositories, models, dependencies, build/CI configuration, USDT changes and unclassified code paths select all applicable local, staging and migration packs. Both sides of renamed files are considered. Documentation-only changes skip device jobs. The selection artifact records the tested app SHA, E2E SHA, cases, reasons, lanes and explicit coverage gaps. + +PRs stacked on another branch retain the existing automatic exclusion. Manual dispatch with `pr` checks the PR's current merge against its actual base/head, including stacked PRs. The workflow source is the branch selected for dispatch. `selection=auto` uses the PR diff; `base` runs the five cases; `full` runs all applicable lanes. Automatic PRs cannot narrow selection with a manual input. + +## Nightly + +The main iOS workflow runs nightly at 03:00 UTC, without a changed-revision shortcut: + +| Lane | Active existing cases | +| --- | --- | +| Local | 42: all core UI, local money, on-chain, multi-address and active hardware cases | +| Staging | 6: `pubky_profile_1` through `pubky_profile_5`, `paykit_1` | +| Migration | 6: native restore/upgrade, RN restore/upgrade, passphrase and legacy-address migration | + +The 54 cases use the appropriate local or remote-regtest build and fixtures. Migration setup entries are fixtures, not extra regression passes. Five existing skipped cases and six unsupported iOS mainnet cases remain explicit gaps; the interactive Lightning payment utility remains manual. This does not invent missing USDT, allowance or subscription automation. + +## Attempts and evidence + +`scripts/ci-run.ts` prepares a plan tied to the actual E2E checkout, selected app SHA, downloaded artifact digest, platform, backend, variant and run identity. Selection rejects unknown, skipped or wrong-backend cases. Only selected spec files load. + +A run permits up to three attempts and stops after the first runner exit-zero with exact complete case coverage. Each attempt retains worker events, process outcome and timing. The local/staging wrapper owns retries; migration keeps its existing per-attempt prepared-wallet setup and invokes the same wrapper one attempt at a time. An inventory-enabled attempt reruns the complete selected pack, avoiding dependence on the machine's legacy `/tmp/lock`. Android's non-inventory execution retains its existing retry locks. + +A recovered failure is reported as a recovery, with the first failed/missing case results and earlier errors retained. Missing reports, setup/teardown errors, incomplete workers, unexpected skips, stale identities or a nonzero final runner cannot produce a pass. The parent `e2e-status` requires every selected local/staging/migration job and build to succeed. + +## Cost controls and manual validation + +Automatic runs use one local shard. Only manual dispatch can request two, partitioned by spec so a spec's fixture lifecycle stays together. Namespace supplies a separate Mac and backend instance per shard; every shard destroys its backend in cleanup. The local app is built once and reused. Staging and migration share one remote-regtest app build in the combined run. Builds have a 40-minute limit; execution jobs 135 minutes; individual runner attempts 40 minutes. Artifacts are retained for seven days and superseded runs cancel. + +Compare `selection=full, shards=1` and `selection=full, shards=2` on the same app and E2E revisions. Inspect elapsed/build/setup/test time, first-pass success and billed Mac plus backend usage. Two shards are an experiment, not an assumed cost saving or a new automatic default. + +Merge the E2E producer before the iOS consumer. Before merging, dispatch the existing iOS workflow from the development branch and explicitly select the E2E development branch. Validate base/full/automatic selection, failures/recovery and the optional shard experiment. The default branch's gate stays intact until the consumer is merged. No parallel shadow workflow is installed. + +## Local checks ```sh -export PLATFORM=ios -export BACKEND=regtest -export E2E_RUN_ID="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-ios-smoke" -export E2E_APP_SHA='' -export E2E_ARTIFACT_DIGEST='' -export E2E_BACKEND_REVISION='regtest:stag0' -export E2E_VARIANT='default' -export E2E_PLAN="$PWD/artifacts/reporting/plan.json" -node --experimental-strip-types scripts/e2e-inventory.ts plan \ - --platform ios --backend regtest --pack core-ui --plan "$E2E_PLAN" -BACKEND=regtest ATTEMPT=1 ./ci_run_ios.sh \ - --mochaOpts.grep "$(cat artifacts/reporting/grep.txt)" -# Run even when the runner failed. The report command exits nonzero on incomplete coverage. -node --experimental-strip-types scripts/e2e-inventory.ts report --plan "$E2E_PLAN" +npm run inventory:check +npm run test:reporting +npx tsc --noEmit ``` -Retain the plan, worker JSON and summary files alongside failure diagnostics. The -report checks exact case IDs, source/build/backend/run identity, missing workers or -cases, unexpected skips, duplicate execution and interrupted runs. Runner failure -after passing cases (including teardown failure) still fails the report. JSON keeps -per-case timing and attempt history; Markdown can be appended to the Actions summary. -These are test execution durations, not billed Mac/Linux usage. - -One attempt is the default. An explicit `E2E_MAX_ATTEMPTS` of 2 or 3 allows a bounded -retry policy; recovered failures remain visible. Inventory-enabled runs bypass the -legacy `/tmp/lock` mechanism and rerun selected cases. A receipt from another night, -source, build, backend or migration variant cannot satisfy the current plan. Existing -non-inventory invocations retain their legacy retry behavior. - -`--pack nightly` selects all applicable obligations for one platform/backend, -including known gaps. It is a building block for the later full-nightly orchestration; -it neither starts a schedule nor declares a platform's whole nightly complete. That -orchestrator must reconcile every required backend/job/variant and detect a missing -scheduled batch. It must run on both platforms every night even for unchanged inputs. -An empty selection fails rather than becoming an empty green run. - -## Stages - -1. Land the inventory/reporting producer and its validation checks first. -2. Add the iOS smoke workflow as manual-only validation using the producer's - companion branch or a revision containing these files. Keep the current required - gates and schedules; do not run both gates automatically on each PR. -3. Measure real smoke duration, concurrent-run isolation and first-pass reliability - before proposing any required-check change. The iOS two-shard pilot, automatic PR - risk selection and full nightly orchestration remain separate stages. - -No ruleset change or existing check-name replacement is part of these first stages. - -## Shadow smoke producer - -The `smoke` pack contains three fresh-wallet regtest cases: `smoke_wallet_1` -(Home), `smoke_receive_1` (receive QR address and exact entered amount), and -`smoke_onchain_1` (fund and send). The payment oracle queries Electrum directly, -recomputes transaction IDs from raw transactions, checks the exact recipient and -amount, and requires the outgoing payment to spend this case's funding transaction. -A fresh recipient prevents another concurrent run's payment from satisfying the -assertion. The oracle uses bounded requests and stores raw transaction evidence. -These cases do not match the existing `@ios_gate` or Android feature selectors. - -The iOS consumer is `.github/workflows/e2e-smoke-shadow.yml`. Land this producer -before enabling that workflow; for paired branches, use the same branch name in -both repositories. A missing producer fails visibly. Reporting remains opt-in: -existing automatic gates and schedules continue unchanged. This stage does not -activate full nightly coverage, risk packs or the two-shard experiment. - -The iOS smoke consumer has only a manual dispatch trigger during validation. After -validation, switch the automatic PR gate in one change, without an automatic -overlap period. The lightweight shared inventory check can continue automatically. +The CLI workflow tests use controlled runner fixtures; they are not evidence that hosted Appium journeys passed. Actual hosted measurements and device results must name their revisions and environments. diff --git a/package.json b/package.json index 932c3868..30cf446b 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "scripts": { "inventory:check": "node --experimental-strip-types scripts/e2e-inventory.ts check", - "test:reporting": "node --experimental-strip-types --test test/unit/inventory.test.ts test/unit/reporting.test.ts test/unit/reporting-cli.test.ts test/unit/reporter-events.test.ts test/unit/smoke-oracle.test.ts", + "test:reporting": "node --experimental-strip-types --test test/unit/inventory.test.ts test/unit/reporting.test.ts test/unit/reporting-cli.test.ts test/unit/reporter-events.test.ts test/unit/ci-selection.test.cjs test/unit/ci-retry.test.ts", "e2e:android": "PLATFORM=android wdio run ./wdio.conf.ts", "e2e:ios": "PLATFORM=ios wdio run ./wdio.conf.ts", "lint": "eslint './**/*.{ts,js}'", diff --git a/scripts/ci-run.ts b/scripts/ci-run.ts new file mode 100644 index 00000000..bd0918d4 --- /dev/null +++ b/scripts/ci-run.ts @@ -0,0 +1,242 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { + loadInventory, + validateInventory, + type Platform, + type Backend, +} from '../test/reporting/inventory.ts'; +import { executionIdentity, assertExecutionIdentity } from '../test/reporting/identity.ts'; +import { + createPlan, + digest, + writeJson, + reconcile, + reconcileAttempts, + markdownReport, + type Plan, + type WorkerReport, + type AttemptReceipt, +} from '../test/reporting/report.ts'; + +async function runProcess(runner: string, args: string[], env: NodeJS.ProcessEnv, log: string) { + const output = fs.createWriteStream(log); + const child = spawn(runner, args, { env, detached: true, stdio: ['ignore', 'pipe', 'pipe'] }); + let cancelled = false; + let timedOut = false; + let forceKill: ReturnType | undefined; + const kill = (signal: NodeJS.Signals) => { + if (child.pid) { + try { + process.kill(-child.pid, signal); + } catch (error: any) { + if (error.code !== 'ESRCH') throw error; + } + } + }; + const stop = () => { + kill('SIGTERM'); + forceKill = setTimeout(() => kill('SIGKILL'), 5000); + }; + const interrupt = () => { + cancelled = true; + stop(); + }; + process.once('SIGTERM', interrupt); + process.once('SIGINT', interrupt); + const timer = setTimeout( + () => { + timedOut = true; + stop(); + }, + 40 * 60 * 1000 + ); + child.stdout?.on('data', (data) => { + process.stdout.write(data); + output.write(data); + }); + child.stderr?.on('data', (data) => { + process.stderr.write(data); + output.write(data); + }); + child.on('error', (error) => output.write(String(error) + '\n')); + const status = await new Promise((resolve) => + child.on('close', (code) => resolve(code ?? 1)) + ); + clearTimeout(timer); + if (forceKill) clearTimeout(forceKill); + // Also terminate descendants that outlived the runner itself. + kill('SIGKILL'); + process.removeListener('SIGTERM', interrupt); + process.removeListener('SIGINT', interrupt); + await new Promise((resolve) => output.end(resolve)); + return { status: cancelled || timedOut ? 1 : status, cancelled }; +} + +function artifactHash(directory: string): string { + const hash = crypto.createHash('sha256'); + function visit(relative: string) { + const filename = path.join(directory, relative); + const stat = fs.lstatSync(filename); + hash.update(relative + '\0'); + if (stat.isSymbolicLink()) hash.update('link\0' + fs.readlinkSync(filename)); + else if (stat.isDirectory()) + for (const name of fs.readdirSync(filename).sort()) visit(path.join(relative, name)); + else hash.update(fs.readFileSync(filename)); + } + visit(''); + return hash.digest('hex'); +} +const command = process.argv[2]; +const planFile = process.env.E2E_PLAN || 'artifacts/reporting/plan.json'; +const directory = path.dirname(planFile); +function workers(): WorkerReport[] { + const dir = path.join(directory, 'workers'); + return fs.existsSync(dir) + ? fs + .readdirSync(dir) + .filter((f) => f.endsWith('.json')) + .sort() + .map((f) => JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8'))) + : []; +} +function receipts(): AttemptReceipt[] { + return fs.existsSync(directory) + ? fs + .readdirSync(directory) + .filter((f) => /^attempt-\d+\.json$/.test(f)) + .sort() + .map((f) => JSON.parse(fs.readFileSync(path.join(directory, f), 'utf8'))) + : []; +} +function summary(plan: Plan) { + const result = reconcileAttempts(plan, workers(), receipts()); + writeJson(path.join(directory, 'summary.json'), result); + const markdown = + markdownReport(result) + + `\nAttempts: ${result.attemptsUsed}/${result.maxAttempts}.\n\n` + + result.attempts + .map( + (a) => + `- Attempt ${a.attempt}: ${a.passed ? 'passed' : 'failed/incomplete'}; runner exit ${a.exitCode ?? 'missing'}${a.problems.length ? '; ' + a.problems.join('; ') : ''}` + ) + .join('\n') + + '\n'; + fs.writeFileSync(path.join(directory, 'summary.md'), markdown); + console.info(markdown); + if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown); + return result; +} +try { + if (command === 'prepare') { + if (fs.existsSync(planFile)) throw Error('Plan exists: use a fresh run directory'); + const inventory = loadInventory(process.cwd()); + validateInventory(process.cwd(), inventory); + const platform = process.env.PLATFORM as Platform, + backend = (process.env.BACKEND || 'local') as Backend; + const ids: string[] = process.env.E2E_CASE_TAG + ? [process.env.E2E_CASE_TAG.replace(/^@/, '')] + : JSON.parse(process.env.E2E_CASE_IDS || '[]'); + if (!Array.isArray(ids) || !ids.length || new Set(ids).size !== ids.length) + throw Error('Empty or duplicate case selection'); + const cases = inventory.cases.filter((c) => ids.includes(c.id)); + if ( + cases.length !== ids.length || + cases.some( + (c) => + c.kind !== 'test' || + c.platforms[platform]?.disposition !== 'required' || + c.platforms[platform]?.backend !== backend + ) + ) + throw Error('Selection contains unknown, unavailable or wrong-backend cases'); + const metadata = JSON.parse(fs.readFileSync('aut/e2e-build.json', 'utf8')); + if ( + metadata.appSha !== process.env.E2E_APP_SHA || + metadata.backend !== backend || + metadata.platform !== platform + ) + throw Error('Downloaded build does not match selected app/platform/backend'); + const artifact = artifactHash( + path.join( + 'aut', + process.env.AUT_FILENAME || (platform === 'ios' ? 'bitkit.app' : 'bitkit_e2e.apk') + ) + ); + process.env.E2E_ARTIFACT_DIGEST = artifact; + if (process.env.GITHUB_ENV) + fs.appendFileSync(process.env.GITHUB_ENV, `E2E_ARTIFACT_DIGEST=${artifact}\n`); + const plan = createPlan( + cases, + executionIdentity(platform, backend), + digest(fs.readFileSync('test/inventory.json', 'utf8')), + 'selected', + 3 + ); + writeJson(planFile, plan); + fs.writeFileSync( + path.join(directory, 'grep.txt'), + `(?:^| )@(?:${plan.expected.join('|')})(?=[ ,]|$)\n` + ); + // Only load selected spec files, avoiding unrelated fixture hooks and environment requirements. + writeJson(path.join(directory, 'specs.json'), [...new Set(cases.map((c) => c.file))]); + } else { + const plan: Plan = JSON.parse(fs.readFileSync(planFile, 'utf8')); + assertExecutionIdentity(plan); + if (command === 'run' || command === 'attempt') { + const currentAttempt = command === 'attempt' ? Number(process.env.ATTEMPT) : 1; + if ( + !Number.isInteger(currentAttempt) || + currentAttempt < 1 || + currentAttempt > plan.maxAttempts + ) + throw Error('Invalid attempt'); + if (receipts().length !== currentAttempt - 1) + throw Error('Unexpected attempt receipts; refusing to reuse another execution'); + if (currentAttempt > 1 && reconcileAttempts(plan, workers(), receipts()).status === 'passed') + throw Error('Previous attempt already passed'); + const specs: string[] = JSON.parse( + fs.readFileSync(path.join(directory, 'specs.json'), 'utf8') + ); + const grep = fs.readFileSync(path.join(directory, 'grep.txt'), 'utf8').trim(); + const runner = plan.identity.platform === 'ios' ? './ci_run_ios.sh' : './ci_run_android.sh'; + for ( + let attempt = currentAttempt; + attempt <= (command === 'attempt' ? currentAttempt : plan.maxAttempts); + attempt++ + ) { + const receipt: AttemptReceipt = { attempt, startedAt: new Date().toISOString() }; + const file = path.join(directory, `attempt-${attempt}.json`); + writeJson(file, receipt); + const args = specs.flatMap((spec) => ['--spec', spec]).concat(['--mochaOpts.grep', grep]); + const child = await runProcess( + runner, + args, + { ...process.env, ATTEMPT: String(attempt), E2E_PLAN: planFile }, + path.join(directory, `attempt-${attempt}-runner.log`) + ); + receipt.finishedAt = new Date().toISOString(); + receipt.exitCode = child.status ?? 1; + writeJson(file, receipt); + const result = reconcile( + plan, + workers().filter((w) => w.attempt === attempt) + ); + writeJson(path.join(directory, `attempt-${attempt}-summary.json`), { + ...result, + runnerExit: receipt.exitCode, + }); + if (child.cancelled || (receipt.exitCode === 0 && result.status === 'passed')) break; + console.error( + `Attempt ${attempt} failed; evidence retained in ${directory} and artifacts/attempt-${attempt}` + ); + } + } else if (command !== 'report') throw Error('Usage: ci-run.ts prepare|run|attempt|report'); + if (summary(plan).status !== 'passed') process.exitCode = 1; + } +} catch (error) { + console.error(error); + process.exitCode = 1; +} diff --git a/scripts/ci-selection.cjs b/scripts/ci-selection.cjs new file mode 100644 index 00000000..b6cf5a2d --- /dev/null +++ b/scripts/ci-selection.cjs @@ -0,0 +1,198 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); + +const BASE = ['send_1', 'send_2', 'lightning_1', 'transfer_2', 'boost_2']; +const RULES = [ + { name: 'hardware', match: /hardware|trezor|jade/i, ids: /^hardware_wallet_/ }, + { name: 'lnurl', match: /lnurl/i, ids: /^lnurl_/ }, + { + name: 'addresses', + match: /multi.?address|address.?type|address.?viewer|utxo|coin.?selection/i, + ids: /^(multi_address_|onchain_|boost_)/, + }, + { + name: 'fees', + match: /boost|cpfp|rbf|fee|transaction.?speed/i, + ids: /^(boost_|onchain_|send_)/, + }, + { + name: 'onboarding', + match: /onboarding|mnemonic|seedqr|passphrase|backup|restore|recovery|migration/i, + ids: /^(onboarding_|backup_|security_|migration_(?!setup)|settings_07)/, + migration: true, + }, + { + name: 'security', + match: /security|privacy|pin(code|pad)?|biometric|keychain|authcheck/i, + ids: /^(security_|settings_06|settings_07|send_)/, + }, + { + name: 'pubky-paykit', + match: /pubky|paykit|contact|profile|subscription|allowance|payment.?request|marketplace|shop/i, + ids: /^(pubky_profile_|paykit_)/, + staging: true, + }, + { + name: 'receive', + match: /receive|invoice|scanner|qrscann/i, + ids: /^(receive_|numberpad_|send_|onchain_|lnurl_)/, + }, + { + name: 'send', + match: /send|quickpay|amount|numberpad/i, + ids: /^(send_|numberpad_|onchain_|boost_|lnurl_)/, + }, + { + name: 'lightning', + match: /lightning|channel|transfer|connection/i, + ids: /^(lightning_|transfer_|multi_address_|send_|lnurl_)/, + }, + { name: 'settings', match: /settings|electrum|gossip|currency|denomination/i, ids: /^settings_/ }, + { name: 'widgets', match: /widget/i, ids: /^widgets_/ }, +]; +// Shared services, dependencies, routing and unknown paths expand rather than guess. +const BROAD = + /(^|\/)(services?|repositories|managers|data|models|helpers|navigation|utilities|extensions)(\/|$)|appscene|appviewmodel|mainnav|mainactivity|lightningrepo|\.github\/|\.gradle|gradle\/|Package\.(swift|resolved)|project\.pbxproj|\.xcconfig|\.toml$|\.properties$|usdt|bitkit.?core|ldk/i; +const DOCUMENTATION = + /^(docs?\/|journeys\/|changelog\.d\/)|(^|\/)(README|CHANGELOG|AGENTS|CLAUDE)\.md$|\.md$/i; + +function select(cases, { platform, mode = 'auto', files = [], shards = 1, release = false }) { + if (!['ios', 'android'].includes(platform)) throw Error('Invalid platform'); + if (!['auto', 'base', 'full', 'nightly'].includes(mode)) throw Error('Invalid selection mode'); + if (![1, 2].includes(shards)) throw Error('Only one or two shards are supported'); + if (!Array.isArray(files) || files.some((f) => typeof f !== 'string' || !f || f.includes('\0'))) + throw Error('Invalid changed paths'); + const runnable = cases.filter( + (c) => c.kind === 'test' && c.platforms[platform].disposition === 'required' + ); + for (const id of BASE) + if (!runnable.some((c) => c.id === id && c.platforms[platform].backend === 'local')) + throw Error(`Missing base case: ${id}`); + const local = runnable.filter((c) => c.platforms[platform].backend === 'local'); + const selected = new Set(BASE); + const reasons = []; + let broad = ['full', 'nightly'].includes(mode); + let staging = broad, + migration = broad || release; + const changed = files.filter((f) => !DOCUMENTATION.test(f)); + const code = mode !== 'auto' || changed.length > 0 || release; + if (mode === 'auto') + for (const file of changed) { + const rules = RULES.filter((r) => r.match.test(file)); + if (BROAD.test(file) || rules.length === 0) { + broad = true; + reasons.push({ file, rule: 'broad-or-unclassified' }); + } else + for (const rule of rules) { + for (const c of runnable) if (rule.ids.test(c.id)) selected.add(c.id); + staging ||= !!rule.staging; + migration ||= !!rule.migration; + reasons.push({ file, rule: rule.name }); + } + } + if (broad) { + for (const c of local) selected.add(c.id); + staging = true; + migration = true; + } + const ids = code + ? local + .filter((c) => selected.has(c.id)) + .map((c) => c.id) + .sort() + : []; + // Keep a spec's cases together: before hooks can own a wallet shared by sibling cases. + const bins = Array.from({ length: shards }, () => []); + const specs = [...new Set(local.filter((c) => ids.includes(c.id)).map((c) => c.file))].sort(); + for (const file of specs) { + const group = local.filter((c) => c.file === file && ids.includes(c.id)).map((c) => c.id); + bins.sort((a, b) => a.length - b.length); + bins[0].push(...group); + } + const matrix = bins + .filter((b) => b.length) + .map((ids, i) => ({ name: `local-${i + 1}`, ids: ids.sort() })); + return { + schemaVersion: 1, + platform, + mode, + code, + base: BASE, + ids, + matrix, + staging: code && staging, + migration: code && migration, + reasons, + gaps: cases + .filter((c) => c.platforms[platform].disposition === 'gap') + .map((c) => ({ id: c.id, ...c.platforms[platform] })), + }; +} + +if (require.main === module) { + const app = path.resolve(process.env.E2E_APP_ROOT || '.'); + const git = (...args) => execFileSync('git', ['-C', app, ...args], { encoding: 'utf8' }).trim(); + const tested = git('rev-parse', 'HEAD'); + const event = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT_PATH, 'utf8')); + const pr = event.pull_request; + const manualPr = process.env.E2E_MANUAL_PR; + let base = pr?.base?.sha; + if (manualPr) { + if (!/^\d+$/.test(manualPr)) throw Error('Invalid PR number'); + const current = JSON.parse( + execFileSync('gh', ['api', `repos/${process.env.GITHUB_REPOSITORY}/pulls/${manualPr}`], { + encoding: 'utf8', + }) + ); + base = current.base.sha; + if (git('show', '-s', '--format=%P', 'HEAD') !== `${base} ${current.head.sha}`) + throw Error('Manual PR merge does not match current base and head'); + } else if (pr && git('show', '-s', '--format=%P', 'HEAD') !== `${base} ${pr.head.sha}`) + throw Error('PR merge does not match event base and head'); + const files = base + ? execFileSync('git', ['-C', app, 'diff', '--no-renames', '--name-only', '-z', base, tested], { + encoding: 'utf8', + }) + .split('\0') + .filter(Boolean) + : []; + const mode = + process.env.GITHUB_EVENT_NAME === 'schedule' + ? 'nightly' + : process.env.GITHUB_EVENT_NAME === 'pull_request' + ? 'auto' + : process.env.E2E_SELECTION || 'full'; + if (mode === 'auto' && !base) throw Error('Automatic manual selection requires a PR number'); + const shards = + process.env.GITHUB_EVENT_NAME === 'workflow_dispatch' ? Number(process.env.E2E_SHARDS || 1) : 1; + const inventory = JSON.parse( + fs.readFileSync(path.join(__dirname, '../test/inventory.json'), 'utf8') + ); + const result = select(inventory.cases, { + platform: process.env.PLATFORM, + mode, + files, + shards, + release: /^release-/.test(pr?.head?.ref || '') || /^release-/.test(pr?.base?.ref || ''), + }); + result.appSha = tested; + result.e2eSha = execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: path.join(__dirname, '..'), + encoding: 'utf8', + }).trim(); + fs.writeFileSync('selection.json', JSON.stringify(result, null, 2) + '\n'); + if (process.env.GITHUB_OUTPUT) + for (const key of ['code', 'matrix', 'staging', 'migration', 'appSha', 'e2eSha']) + fs.appendFileSync( + process.env.GITHUB_OUTPUT, + `${key}=${typeof result[key] === 'object' ? JSON.stringify(result[key]) : result[key]}\n` + ); + if (process.env.GITHUB_STEP_SUMMARY) + fs.appendFileSync( + process.env.GITHUB_STEP_SUMMARY, + `## E2E selection\n\nApp: ${tested}; tests: ${result.e2eSha}\n\nLocal cases (${result.ids.length}): ${result.ids.join(', ') || 'none'}\n\nStaging: ${result.staging}; migration: ${result.migration}; shards: ${result.matrix.length}\n\n${result.reasons.map((r) => `- ${r.file}: ${r.rule}`).join('\n')}\n\nKnown gaps (not counted as passes): ${result.gaps.map((g) => g.id).join(', ')}\n` + ); + console.log(JSON.stringify(result, null, 2)); +} +module.exports = { BASE, RULES, select }; diff --git a/test/helpers/smoke-oracle.ts b/test/helpers/smoke-oracle.ts deleted file mode 100644 index 1db59992..00000000 --- a/test/helpers/smoke-oracle.ts +++ /dev/null @@ -1,180 +0,0 @@ -import crypto from 'node:crypto'; -import net from 'node:net'; -import tls from 'node:tls'; -import { address, networks, payments, Transaction } from 'bitcoinjs-lib'; - -export type Endpoint = { host: string; port: number; protocol: 'tls' | 'tcp' }; -export type PaymentEvidence = { - txid: string; - address: string; - sats: number; - rawTransaction: string; -}; - -export function freshRegtestRecipient(): string { - const key = crypto.createECDH('secp256k1'); - key.generateKeys(); - return payments.p2wpkh({ - pubkey: key.getPublicKey(undefined, 'compressed'), - network: networks.regtest, - }).address!; -} - -export function scriptHash(recipient: string): string { - return crypto - .createHash('sha256') - .update(address.toOutputScript(recipient, networks.regtest)) - .digest() - .reverse() - .toString('hex'); -} - -export function electrumRequest( - endpoint: Endpoint, - method: string, - params: unknown[], - timeoutMs = 10_000 -): Promise { - return new Promise((resolve, reject) => { - const socket = - endpoint.protocol === 'tls' - ? tls.connect({ - host: endpoint.host, - port: endpoint.port, - servername: net.isIP(endpoint.host) ? undefined : endpoint.host, - }) - : net.createConnection({ host: endpoint.host, port: endpoint.port }); - let buffer = ''; - let settled = false; - const finish = (error?: Error, result?: unknown) => { - if (settled) return; - settled = true; - clearTimeout(timer); - socket.destroy(); - if (error) reject(error); - else resolve(result); - }; - const timer = setTimeout( - () => finish(new Error(`Smoke backend unavailable: ${method} timed out`)), - timeoutMs - ); - socket.setEncoding('utf8'); - socket.on(endpoint.protocol === 'tls' ? 'secureConnect' : 'connect', () => - socket.write(JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }) + '\n') - ); - socket.on('error', (error) => finish(new Error(`Smoke backend unavailable: ${error.message}`))); - socket.on('close', () => - finish(new Error(`Smoke backend closed before responding to ${method}`)) - ); - socket.on('data', (chunk) => { - buffer += chunk; - if (buffer.length > 2_000_000) return finish(new Error('Oversized Electrum response')); - while (buffer.includes('\n')) { - const end = buffer.indexOf('\n'); - const line = buffer.slice(0, end); - buffer = buffer.slice(end + 1); - try { - const response = JSON.parse(line); - if (response.id !== 1) continue; - if (response.error) - return finish( - new Error(`Smoke backend rejected ${method}: ${JSON.stringify(response.error)}`) - ); - if (!Object.hasOwn(response, 'result')) - return finish(new Error('Malformed Electrum response')); - finish(undefined, response.result); - } catch (error) { - finish(error instanceof Error ? error : new Error(String(error))); - } - } - }); - }); -} - -export async function history( - endpoint: Endpoint, - recipient: string -): Promise<{ tx_hash: string; height: number }[]> { - const result = await electrumRequest(endpoint, 'blockchain.scripthash.get_history', [ - scriptHash(recipient), - ]); - if ( - !Array.isArray(result) || - result.some((r) => !r || !/^[a-f0-9]{64}$/.test(r.tx_hash) || !Number.isInteger(r.height)) - ) { - throw new Error('Malformed Electrum address history'); - } - return result; -} - -export function verifyPayment( - raw: string, - txid: string, - recipient: string, - sats: number, - spendsTxid?: string -): PaymentEvidence { - const tx = Transaction.fromHex(raw); - if (tx.getId() !== txid) throw new Error('Transaction identity does not match its raw bytes'); - const script = address.toOutputScript(recipient, networks.regtest); - const outputs = tx.outs.filter((output) => output.script.equals(script)); - if (outputs.length !== 1 || outputs[0].value !== sats) - throw new Error('Transaction recipient/amount does not match the requested payment'); - if ( - spendsTxid && - !tx.ins.some((input) => Buffer.from(input.hash).reverse().toString('hex') === spendsTxid) - ) { - throw new Error("Payment does not spend this test wallet's funding transaction"); - } - return { txid, address: recipient, sats, rawTransaction: raw }; -} - -export async function waitForPayment( - endpoint: Endpoint, - recipient: string, - sats: number, - options: { txid?: string; spendsTxid?: string; timeoutMs?: number } = {} -): Promise { - const deadline = Date.now() + (options.timeoutMs ?? 120_000); - while (Date.now() < deadline) { - const transactions = await history(endpoint, recipient); - const candidates = options.txid - ? transactions.filter((tx) => tx.tx_hash === options.txid) - : transactions; - if (candidates.length > 1) - throw new Error('Fresh recipient has multiple transactions; fixture isolation is broken'); - if (candidates.length === 1) { - const txid = candidates[0].tx_hash; - const raw = await electrumRequest(endpoint, 'blockchain.transaction.get', [txid, false]); - if (typeof raw !== 'string') throw new Error('Electrum did not return a raw transaction'); - return verifyPayment(raw, txid, recipient, sats, options.spendsTxid); - } - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - throw new Error(`Expected payment was not observed within ${options.timeoutMs ?? 120_000}ms`); -} - -export async function regtestOperation( - baseUrl: string, - operation: 'deposit' | 'mine', - body: Record -): Promise { - const response = await fetch(`${baseUrl}/regtest/chain/${operation}`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - signal: AbortSignal.timeout(15_000), - }); - if (!response.ok) - throw new Error(`Smoke regtest ${operation} unavailable: HTTP ${response.status}`); - return (await response.text()).trim(); -} - -/** Parse BIP21 decimal BTC exactly; never round fractional satoshis into a pass. */ -export function bitcoinAmountToSats(value: string): number { - const match = /^(0|[1-9][0-9]*)(?:\.([0-9]{1,8}))?$/.exec(value); - if (!match) throw new Error('Invalid BIP21 amount'); - const sats = BigInt(match[1]) * 100_000_000n + BigInt((match[2] || '').padEnd(8, '0')); - if (sats > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error('BIP21 amount exceeds safe range'); - return Number(sats); -} diff --git a/test/inventory.json b/test/inventory.json index f92facb7..5d5c5c46 100644 --- a/test/inventory.json +++ b/test/inventory.json @@ -7,7 +7,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -634,7 +634,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -716,7 +716,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -735,7 +735,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -754,7 +754,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -773,7 +773,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -792,7 +792,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -811,7 +811,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -830,7 +830,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -849,7 +849,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1086,7 +1086,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1105,7 +1105,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1124,7 +1124,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1181,7 +1181,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1200,7 +1200,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1219,7 +1219,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1238,7 +1238,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1257,7 +1257,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1276,7 +1276,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1295,7 +1295,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1314,7 +1314,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "gap", "fixtures": ["fresh-wallet"], @@ -1339,7 +1339,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1377,7 +1377,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1396,7 +1396,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1415,7 +1415,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1434,7 +1434,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1522,7 +1522,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1541,7 +1541,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1560,7 +1560,7 @@ "kind": "test", "platforms": { "ios": { - "backend": "regtest", + "backend": "local", "pack": "core-ui", "disposition": "required", "fixtures": ["fresh-wallet"] @@ -1572,63 +1572,6 @@ "fixtures": ["fresh-wallet"] } } - }, - { - "id": "smoke_wallet_1", - "file": "test/specs/smoke.e2e.ts", - "kind": "test", - "platforms": { - "ios": { - "backend": "regtest", - "pack": "smoke", - "disposition": "required", - "fixtures": ["fresh-wallet", "regtest-funding-api", "independent-electrum-oracle"] - }, - "android": { - "backend": "regtest", - "pack": "smoke", - "disposition": "required", - "fixtures": ["fresh-wallet", "regtest-funding-api", "independent-electrum-oracle"] - } - } - }, - { - "id": "smoke_receive_1", - "file": "test/specs/smoke.e2e.ts", - "kind": "test", - "platforms": { - "ios": { - "backend": "regtest", - "pack": "smoke", - "disposition": "required", - "fixtures": ["fresh-wallet", "regtest-funding-api", "independent-electrum-oracle"] - }, - "android": { - "backend": "regtest", - "pack": "smoke", - "disposition": "required", - "fixtures": ["fresh-wallet", "regtest-funding-api", "independent-electrum-oracle"] - } - } - }, - { - "id": "smoke_onchain_1", - "file": "test/specs/smoke.e2e.ts", - "kind": "test", - "platforms": { - "ios": { - "backend": "regtest", - "pack": "smoke", - "disposition": "required", - "fixtures": ["fresh-wallet", "regtest-funding-api", "independent-electrum-oracle"] - }, - "android": { - "backend": "regtest", - "pack": "smoke", - "disposition": "required", - "fixtures": ["fresh-wallet", "regtest-funding-api", "independent-electrum-oracle"] - } - } } ] } diff --git a/test/reporting/report.ts b/test/reporting/report.ts index f563d65d..2c43487f 100644 --- a/test/reporting/report.ts +++ b/test/reporting/report.ts @@ -75,7 +75,8 @@ export function createPlan( const selected = cases.filter((item) => { const assignment = item.platforms[identity.platform]; return ( - assignment.backend === identity.backend && (pack === 'nightly' || assignment.pack === pack) + assignment.backend === identity.backend && + (pack === 'selected' || pack === 'nightly' || assignment.pack === pack) ); }); const expected = selected @@ -183,7 +184,12 @@ export function reconcile(plan: Plan, reports: WorkerReport[]) { } } if (status !== 'passed') problems.push(`${id}: ${status}`); - return { id, status, firstAttempt: history[0]?.event.status ?? 'missing', history }; + return { + id, + status, + firstAttempt: history.find((h) => h.attempt === 1)?.event.status ?? 'missing', + history, + }; }); const dispositions = plan.cases.filter( (item) => item.platforms[plan.identity.platform].disposition !== 'required' @@ -232,3 +238,53 @@ export function markdownReport(result: ReturnType): string { '', ].join('\n'); } + +export type AttemptReceipt = { + attempt: number; + startedAt: string; + finishedAt?: string; + exitCode?: number; +}; + +export function reconcileAttempts(plan: Plan, reports: WorkerReport[], attempts: AttemptReceipt[]) { + validatePlan(plan); + const problems: string[] = []; + const perAttempt = attempts.map((attempt, index) => { + if (attempt.attempt !== index + 1 || attempt.attempt > plan.maxAttempts) + problems.push('Invalid, missing or out-of-order attempt receipt'); + const coverage = reconcile( + plan, + reports.filter((r) => r.attempt === attempt.attempt) + ); + const passed = attempt.exitCode === 0 && !!attempt.finishedAt && coverage.status === 'passed'; + return { ...attempt, passed, problems: coverage.problems, coverage }; + }); + if (!perAttempt.length) problems.push('Runner never started'); + if (perAttempt.slice(0, -1).some((a) => a.passed)) + problems.push('Attempt executed after a successful attempt'); + if (reports.some((r) => !attempts.some((a) => a.attempt === r.attempt))) + problems.push('Worker report without matching attempt receipt'); + const last = perAttempt.at(-1); + if (!last?.passed) + problems.push('Final attempt did not complete successfully with exact coverage'); + const fallback = reconcile(plan, []); + const result = last?.coverage ?? fallback; + return { + ...result, + status: problems.length ? 'incomplete' : 'passed', + firstFailure: perAttempt.length > 0 && !perAttempt[0].passed, + attemptsUsed: attempts.length, + maxAttempts: plan.maxAttempts, + attempts: perAttempt.map(({ coverage, ...receipt }) => receipt), + problems: [...problems, ...(last?.passed ? [] : result.problems)], + results: result.results.map((r) => ({ + ...r, + firstAttempt: + perAttempt[0]?.coverage.results.find((first) => first.id === r.id)?.status ?? 'missing', + history: perAttempt.flatMap( + (a) => a.coverage.results.find((c) => c.id === r.id)?.history ?? [] + ), + })), + durationMs: perAttempt.reduce((sum, a) => sum + a.coverage.durationMs, 0), + }; +} diff --git a/test/specs/smoke.e2e.ts b/test/specs/smoke.e2e.ts deleted file mode 100644 index 5a9d4c3f..00000000 --- a/test/specs/smoke.e2e.ts +++ /dev/null @@ -1,110 +0,0 @@ -import assert from 'node:assert/strict'; -import path from 'node:path'; -import { - acknowledgeReceivedPaymentIfPresent, - completeOnboarding, - dismissBackupTimedSheet, - dragOnElement, - elementById, - enterAddress, - enterAmount, - expectSavingsBalance, - expectTotalBalance, - getReceiveAddress, - getUriFromQRCode, - swipeFullScreen, - tap, -} from '../helpers/actions'; -import { blocktankURL, electrumHost, electrumPort, getBackend } from '../helpers/constants'; -import { reinstallApp } from '../helpers/setup'; -import { ciIt } from '../helpers/suite'; -import { - bitcoinAmountToSats, - freshRegtestRecipient, - history, - regtestOperation, - scriptHash, - waitForPayment, -} from '../helpers/smoke-oracle'; -import { writeJson } from '../reporting/report'; - -describe('@ios_smoke - Focused wallet smoke (shadow)', () => { - const endpoint = { host: electrumHost, port: electrumPort, protocol: 'tls' as const }; - before(() => { - if (getBackend() !== 'regtest') - throw new Error('Smoke requires a matching regtest app build and BACKEND=regtest'); - }); - beforeEach(async () => { - await reinstallApp(); - await completeOnboarding(); - }); - - ciIt('@smoke_wallet_1 - A fresh wallet reaches usable Home', async () => { - await expectTotalBalance(0); - await elementById('Receive').waitForDisplayed(); - await elementById('Send').waitForDisplayed(); - }); - - ciIt('@smoke_receive_1 - Receive details encode the requested amount', async () => { - const recipient = await getReceiveAddress(); - assert.ok(recipient.startsWith('bcrt1')); - scriptHash(recipient); - await tap('ShowDetails'); - await elementById('ReceiveScreen').waitForDisplayed(); - await tap('QRCode'); - await tap('SpecifyInvoiceButton'); - await tap('ReceiveNumberPadTextField'); - await tap('N1'); - await tap('N2'); - await tap('N3'); - await tap('ReceiveNumberPadSubmit'); - await tap('ShowQrReceive'); - const uri = await getUriFromQRCode(); - const [encodedAddress, query] = uri.replace(/^bitcoin:/, '').split('?'); - assert.equal(encodedAddress, recipient); - const amount = new URLSearchParams(query).get('amount'); - assert.ok(amount, 'Receive QR must encode the specified amount'); - assert.equal(bitcoinAmountToSats(amount), 123); - }); - - ciIt('@smoke_onchain_1 - Receive and send with independent transaction evidence', async () => { - const walletAddress = await getReceiveAddress(); - assert.equal((await history(endpoint, walletAddress)).length, 0, 'Wallet must be fresh'); - await swipeFullScreen('down'); - const depositResult = await regtestOperation(blocktankURL, 'deposit', { - address: walletAddress, - amountSat: 100_000, - }); - const fundingTxid = depositResult.startsWith('"') ? JSON.parse(depositResult) : depositResult; - assert.match(fundingTxid, /^[a-f0-9]{64}$/); - await regtestOperation(blocktankURL, 'mine', { count: 1 }); - const received = await waitForPayment(endpoint, walletAddress, 100_000, { txid: fundingTxid }); - await acknowledgeReceivedPaymentIfPresent(); - await expectTotalBalance(100_000); - await expectSavingsBalance(100_000); - await dismissBackupTimedSheet({ triggerTimedSheet: true }); - - const recipient = freshRegtestRecipient(); - assert.equal( - (await history(endpoint, recipient)).length, - 0, - 'Recipient must be unique to this case' - ); - await enterAddress(recipient); - await enterAmount(10_000); - await tap('ContinueAmount'); - await dragOnElement('GRAB', 'right', 0.95); - await elementById('SendSuccess').waitForDisplayed(); - await tap('Close'); - await regtestOperation(blocktankURL, 'mine', { count: 1 }); - const sent = await waitForPayment(endpoint, recipient, 10_000, { spendsTxid: fundingTxid }); - await expectSavingsBalance(100_000, { condition: 'lt' }); - const directory = process.env.E2E_PLAN - ? path.dirname(process.env.E2E_PLAN) - : 'artifacts/reporting'; - writeJson(path.join(directory, `payment-attempt-${Number(process.env.ATTEMPT || 1)}.json`), { - received, - sent, - }); - }); -}); diff --git a/test/unit/ci-retry.test.ts b/test/unit/ci-retry.test.ts new file mode 100644 index 00000000..4efeb895 --- /dev/null +++ b/test/unit/ci-retry.test.ts @@ -0,0 +1,166 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import fs from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { + createPlan, + reconcileAttempts, + type Identity, + type WorkerReport, + type AttemptReceipt, +} from '../reporting/report.ts'; +import { loadInventory } from '../reporting/inventory.ts'; +const cases = loadInventory(process.cwd()).cases.filter((c) => ['send_1', 'send_2'].includes(c.id)); +const identity: Identity = { + runId: 'retry', + platform: 'ios', + backend: 'local', + appSha: 'a'.repeat(40), + e2eSha: 'b'.repeat(40), + artifactDigest: 'c'.repeat(64), + backendRevision: 'local', + variant: 'local', +}; +const plan = createPlan(cases, identity, 'd'.repeat(64), 'selected', 3); +function receipt(attempt: number, exitCode = 0): AttemptReceipt { + return { + attempt, + exitCode, + startedAt: '2026-11-03T00:00:00Z', + finishedAt: '2026-11-03T00:01:00Z', + }; +} +function worker(attempt: number, failed = false): WorkerReport { + return { + schemaVersion: 1, + planId: plan.id, + attempt, + cid: '0-0', + startedAt: '2026-11-03T00:00:00Z', + finishedAt: '2026-11-03T00:01:00Z', + failures: failed ? 1 : 0, + errors: [], + events: plan.expected.map((id) => ({ + id, + status: failed ? 'failed' : 'passed', + durationMs: 10, + })), + }; +} +test('first failure remains after third-attempt recovery', () => { + const s = reconcileAttempts( + plan, + [worker(1, true), worker(2, true), worker(3)], + [receipt(1, 1), receipt(2, 1), receipt(3)] + ); + assert.equal(s.status, 'passed'); + assert.equal(s.firstFailure, true); + assert.equal(s.attemptsUsed, 3); + assert.equal(s.results[0].firstAttempt, 'failed'); + assert.equal(s.results[0].history.length, 3); +}); +test('startup failure without workers is retained after recovery', () => { + const s = reconcileAttempts(plan, [worker(2)], [receipt(1, 1), receipt(2)]); + assert.equal(s.status, 'passed'); + assert.equal(s.firstFailure, true); + assert.equal(s.results[0].firstAttempt, 'missing'); +}); +test('interrupted worker may recover only with a complete later attempt', () => { + const w = worker(1); + delete w.finishedAt; + const s = reconcileAttempts(plan, [w, worker(2)], [receipt(1, 1), receipt(2)]); + assert.equal(s.status, 'passed'); + assert.equal(s.firstFailure, true); + assert.ok(s.attempts[0].problems.some((p) => p.includes('Interrupted'))); +}); +for (const [name, reports, attempts] of [ + ['no runner', [], []], + ['runner failure despite passing tests', [worker(1)], [receipt(1, 1)]], + ['zero matches', [], [receipt(1)]], + ['attempt after pass', [worker(1), worker(2)], [receipt(1), receipt(2)]], + ['missing attempt', [worker(2)], [receipt(2)]], + ['fourth attempt', [worker(4)], [receipt(4)]], + ['interrupted process', [worker(1)], [{ ...receipt(1), finishedAt: undefined }]], + ['foreign report', [{ ...worker(1), planId: 'other' }], [receipt(1)]], + [ + 'partial retry', + [worker(1, true), { ...worker(2), events: worker(2).events.slice(0, 1) }], + [receipt(1, 1), receipt(2)], + ], +] as [string, WorkerReport[], AttemptReceipt[]][]) + test(name + ' fails closed', () => + assert.equal(reconcileAttempts(plan, reports, attempts).status, 'incomplete') + ); + +for (const failures of [0, 1, 2, 3]) + test(`actual CI wrapper stops correctly after ${failures} initial failures`, (t) => { + fs.mkdirSync('artifacts', { recursive: true }); + const root = fs.mkdtempSync(path.resolve('artifacts/retry-fixture-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + fs.mkdirSync(path.join(root, 'test/specs'), { recursive: true }); + fs.mkdirSync(path.join(root, 'aut/bitkit.app'), { recursive: true }); + fs.writeFileSync( + path.join(root, 'test/inventory.json'), + JSON.stringify({ schemaVersion: 1, cases }) + ); + fs.writeFileSync( + path.join(root, 'test/specs/send.e2e.ts'), + cases.map((c) => `ciIt('@${c.id} - fixture',async()=>{});`).join('\n') + ); + fs.writeFileSync(path.join(root, 'aut/bitkit.app/app'), 'fixture'); + fs.writeFileSync( + path.join(root, 'aut/e2e-build.json'), + JSON.stringify({ appSha: identity.appSha, platform: 'ios', backend: 'local' }) + ); + fs.writeFileSync( + path.join(root, 'ci_run_ios.sh'), + '#!/usr/bin/env bash\nexec node stub.cjs\n', + { mode: 0o755 } + ); + fs.writeFileSync( + path.join(root, 'stub.cjs'), + `const fs=require('fs');const p=JSON.parse(fs.readFileSync(process.env.E2E_PLAN,'utf8'));const a=Number(process.env.ATTEMPT);const failed=a<=${failures};fs.mkdirSync('artifacts/reporting/workers',{recursive:true});fs.mkdirSync('artifacts/attempt-'+a,{recursive:true});fs.writeFileSync('artifacts/attempt-'+a+'/evidence.txt',failed?'first failure':'pass');fs.writeFileSync('artifacts/reporting/workers/attempt-'+a+'.json',JSON.stringify({schemaVersion:1,planId:p.id,attempt:a,cid:'0-0',startedAt:new Date().toISOString(),finishedAt:new Date().toISOString(),failures:failed?1:0,errors:[],events:p.expected.map(id=>({id,status:failed?'failed':'passed',durationMs:1}))}));process.exitCode=failed?1:0;` + ); + const env = { + ...process.env, + PLATFORM: 'ios', + BACKEND: 'local', + E2E_APP_SHA: identity.appSha, + E2E_RUN_ID: 'wrapper-fixture', + E2E_BACKEND_REVISION: 'local', + E2E_CASE_IDS: JSON.stringify(cases.map((c) => c.id)), + E2E_PLAN: 'artifacts/reporting/plan.json', + GITHUB_ENV: path.join(root, 'env'), + GITHUB_STEP_SUMMARY: path.join(root, 'summary.md'), + }; + const invoke = (cmd: string) => + spawnSync( + process.execPath, + ['--experimental-strip-types', path.resolve('scripts/ci-run.ts'), cmd], + { cwd: root, env, encoding: 'utf8' } + ); + const prep = invoke('prepare'); + assert.equal(prep.status, 0, prep.stderr); + Object.assign( + env, + Object.fromEntries( + fs + .readFileSync(path.join(root, 'env'), 'utf8') + .trim() + .split('\n') + .map((line) => line.split('=')) + ) + ); + const run = invoke('run'); + assert.equal(run.status, failures === 3 ? 1 : 0, run.stderr); + const s = JSON.parse( + fs.readFileSync(path.join(root, 'artifacts/reporting/summary.json'), 'utf8') + ); + assert.equal(s.attemptsUsed, Math.min(failures + 1, 3)); + assert.equal(s.firstFailure, failures > 0); + assert.equal( + fs.readFileSync(path.join(root, 'artifacts/attempt-1/evidence.txt'), 'utf8'), + failures ? 'first failure' : 'pass' + ); + }); diff --git a/test/unit/ci-selection.test.cjs b/test/unit/ci-selection.test.cjs new file mode 100644 index 00000000..500932d1 --- /dev/null +++ b/test/unit/ci-selection.test.cjs @@ -0,0 +1,77 @@ +const test = require('node:test'), + assert = require('node:assert/strict'); +const { select, BASE } = require('../../scripts/ci-selection.cjs'); +const cases = require('../inventory.json').cases; +const pick = (files, extra = {}) => + select(cases, { platform: 'ios', mode: 'auto', files, ...extra }); +test('base is exactly the agreed five existing cases', () => + assert.deepEqual([...pick(['Bitkit/Views/Send/SendAmount.swift']).base].sort(), [ + 'boost_2', + 'lightning_1', + 'send_1', + 'send_2', + 'transfer_2', + ])); +test('documentation-only changes avoid device jobs', () => { + const s = pick(['Docs/guide.md']); + assert.equal(s.code, false); + assert.deepEqual(s.matrix, []); +}); +test('hardware changes add all active hardware cases once', () => { + const s = pick(['Bitkit/Views/Hardware/HardwareSend.swift']); + for (const id of [...BASE, 'hardware_wallet_1', 'hardware_wallet_2', 'hardware_wallet_4']) + assert.ok(s.ids.includes(id)); + assert.ok(!s.ids.includes('hardware_wallet_3')); + assert.equal(new Set(s.ids).size, s.ids.length); +}); +test('LNURL and address changes add their risk coverage', () => { + assert.ok(pick(['Bitkit/Views/Lnurl/LnurlAuth.swift']).ids.includes('lnurl_1')); + const s = pick(['Bitkit/Views/Send/UtxoSelection.swift']); + assert.ok(s.ids.includes('multi_address_3')); + assert.ok(s.ids.includes('multi_address_4')); +}); +test('fee changes add CPFP as well as RBF', () => + assert.ok(pick(['Bitkit/Views/Send/FeePicker.swift']).ids.includes('boost_1'))); +test('Pubky and recovery changes select their required executor', () => { + assert.equal(pick(['Bitkit/Views/Profile/Profile.swift']).staging, true); + assert.equal(pick(['Bitkit/Views/Backup/Restore.swift']).migration, true); +}); +for (const file of [ + 'Bitkit/Services/UnknownService.swift', + 'Package.resolved', + 'Bitkit/CompletelyNewFeature.swift', + '.github/workflows/e2e-tests.yml', + 'Bitkit/Assets.xcassets/new.png', +]) + test(`broad/unknown ${file} selects full coverage`, () => { + const s = pick([file]); + assert.equal(s.ids.length, 42); + assert.equal(s.staging, true); + assert.equal(s.migration, true); + }); +test('old and new rename paths both contribute risks', () => { + const s = pick(['Bitkit/Views/Hardware/Old.swift', 'Bitkit/Views/Lnurl/New.swift']); + assert.ok(s.ids.includes('hardware_wallet_4')); + assert.ok(s.ids.includes('lnurl_1')); +}); +test('nightly runs full local and specialized coverage with no changes', () => { + const s = pick([], { mode: 'nightly' }); + assert.equal(s.ids.length, 42); + assert.equal(s.staging, true); + assert.equal(s.migration, true); + assert.ok(s.ids.includes('onchain_3')); +}); +test('two shards partition exact cases without splitting spec fixtures', () => { + const s = pick([], { mode: 'full', shards: 2 }); + assert.equal(s.matrix.length, 2); + assert.deepEqual(s.matrix.flatMap((x) => x.ids).sort(), s.ids); + for (const file of new Set(cases.map((c) => c.file))) { + const ids = cases.filter((c) => c.file === file).map((c) => c.id); + assert.ok(s.matrix.filter((shard) => shard.ids.some((id) => ids.includes(id))).length <= 1); + } +}); +test('known gaps remain explicit instead of executed as passing coverage', () => { + const s = pick([], { mode: 'nightly' }); + assert.ok(s.gaps.some((g) => g.id === 'transfer_1')); + assert.ok(!s.ids.includes('transfer_1')); +}); diff --git a/test/unit/reporter-events.test.ts b/test/unit/reporter-events.test.ts index 289663aa..c80b1fa3 100644 --- a/test/unit/reporter-events.test.ts +++ b/test/unit/reporter-events.test.ts @@ -43,13 +43,13 @@ function fixture(t: test.TestContext) { }); const plan = createPlan( loadInventory(process.cwd()).cases, - executionIdentity('ios', 'regtest'), + executionIdentity('ios', 'local'), digest(fs.readFileSync('test/inventory.json', 'utf8')), - 'smoke' + 'local-money' ); process.env.E2E_PLAN = path.join(directory, 'plan.json'); process.env.PLATFORM = 'ios'; - process.env.BACKEND = 'regtest'; + process.env.BACKEND = 'local'; process.env.ATTEMPT = '1'; writeJson(process.env.E2E_PLAN, plan); const reporter = new InventoryReporter({ writeStream: { write: () => true } }); @@ -57,7 +57,7 @@ function fixture(t: test.TestContext) { cid: '0-0', capabilities: { platformName: 'iOS' }, config: {}, - specs: ['test/specs/smoke.e2e.ts'], + specs: ['test/specs/send.e2e.ts'], }); const receipt = () => JSON.parse( @@ -65,7 +65,7 @@ function fixture(t: test.TestContext) { ) as WorkerReport; const passAll = () => { for (const id of plan.expected) { - const e = { uid: id, cid: '0-0', title: `@${id} - test`, fullTitle: `smoke @${id} - test` }; + const e = { uid: id, cid: '0-0', title: `@${id} - test`, fullTitle: `fixture @${id} - test` }; reporter.emit('test:start', e); reporter.emit('test:pass', e); reporter.emit('test:end', e); @@ -107,10 +107,10 @@ test('hook failure is recorded even if runner failure count is zero', (t) => { test('interruption retains incremental case evidence without a successful finish', (t) => { const f = fixture(t); f.passAll(); - assert.equal(f.receipt().events.length, 3); + assert.equal(f.receipt().events.length, f.plan.expected.length); assert.equal(reconcile(f.plan, [f.receipt()]).status, 'incomplete'); }); -test('framework retries cannot silently turn one-attempt smoke green', (t) => { +test('framework retries cannot silently turn one-attempt fixture green', (t) => { const f = fixture(t); const id = f.plan.expected[0]; const event = { uid: id, cid: '0-0', title: `@${id} - retry` }; diff --git a/test/unit/reporting-cli.test.ts b/test/unit/reporting-cli.test.ts index aa4fd6f4..ae4800a8 100644 --- a/test/unit/reporting-cli.test.ts +++ b/test/unit/reporting-cli.test.ts @@ -13,7 +13,7 @@ test('real CLI rejects a retained successful plan under a replacement execution' const env = { ...process.env, PLATFORM: 'ios', - BACKEND: 'regtest', + BACKEND: 'local', E2E_RUN_ID: 'cli-original', E2E_APP_SHA: 'a'.repeat(40), E2E_ARTIFACT_DIGEST: 'b'.repeat(64), @@ -32,9 +32,9 @@ test('real CLI rejects a retained successful plan under a replacement execution' '--platform', 'ios', '--backend', - 'regtest', + 'local', '--pack', - 'smoke', + 'local-money', ]); assert.equal(preparation.status, 0, preparation.stderr); const plan = JSON.parse(fs.readFileSync(planPath, 'utf8')); diff --git a/test/unit/reporting.test.ts b/test/unit/reporting.test.ts index e11084ff..c50fd29c 100644 --- a/test/unit/reporting.test.ts +++ b/test/unit/reporting.test.ts @@ -15,7 +15,7 @@ import type { InventoryCase } from '../reporting/inventory.ts'; const assignment = { backend: 'regtest' as const, - pack: 'smoke', + pack: 'fixture', disposition: 'required' as const, fixtures: [], }; @@ -36,7 +36,7 @@ const identity: Identity = { variant: 'default', }; function plan() { - return createPlan(cases, identity, 'd'.repeat(64), 'smoke', 3); + return createPlan(cases, identity, 'd'.repeat(64), 'fixture', 3); } function report( events: WorkerReport['events'], @@ -68,7 +68,7 @@ test('same inputs on the following night require fresh receipts', () => { cases, { ...identity, runId: 'night-2026-11-04' }, 'd'.repeat(64), - 'smoke', + 'fixture', 3 ); assert.notEqual(next.id, plan().id); @@ -86,7 +86,7 @@ for (const change of [ cases, { ...identity, ...change } as Identity, 'd'.repeat(64), - 'smoke', + 'fixture', 3 ); assert.equal(reconcile(next, [report(passes)]).status, 'incomplete'); @@ -95,7 +95,7 @@ for (const change of [ test('unknown/empty packs and incomplete build identity cannot create a green plan', () => { assert.throws(() => createPlan(cases, identity, 'd'.repeat(64), 'typo'), /No expected cases/); assert.throws( - () => createPlan(cases, { ...identity, appSha: '' }, 'd'.repeat(64), 'smoke'), + () => createPlan(cases, { ...identity, appSha: '' }, 'd'.repeat(64), 'fixture'), /identity/ ); }); @@ -141,8 +141,8 @@ test('an old pass cannot mask a later failure', () => { 'incomplete' ); }); -test('single-attempt smoke rejects retry receipts', () => { - const single = createPlan(cases, identity, 'd'.repeat(64), 'smoke'); +test('single-attempt fixture rejects retry receipts', () => { + const single = createPlan(cases, identity, 'd'.repeat(64), 'fixture'); assert.equal( reconcile(single, [report(passes, { planId: single.id, attempt: 2 })]).status, 'incomplete' @@ -156,7 +156,7 @@ test('known coverage gaps remain visible even if a case happened to pass', () => reason: 'missing fixture validation', reference: '#264', }; - const p = createPlan(data, identity, 'd'.repeat(64), 'smoke'); + const p = createPlan(data, identity, 'd'.repeat(64), 'fixture'); const result = reconcile(p, [report(passes, { planId: p.id })]); assert.equal(result.status, 'incomplete'); assert.match(result.problems.join(' '), /Unresolved coverage gap/); diff --git a/test/unit/smoke-oracle.test.ts b/test/unit/smoke-oracle.test.ts deleted file mode 100644 index 3933c6d3..00000000 --- a/test/unit/smoke-oracle.test.ts +++ /dev/null @@ -1,154 +0,0 @@ -import assert from 'node:assert/strict'; -import net from 'node:net'; -import test from 'node:test'; -import { address, networks, Transaction } from 'bitcoinjs-lib'; -import { - bitcoinAmountToSats, - electrumRequest, - freshRegtestRecipient, - history, - scriptHash, - verifyPayment, - waitForPayment, -} from '../helpers/smoke-oracle.ts'; - -function transaction(recipient: string, sats = 10_000, funding = 'ab'.repeat(32)) { - const tx = new Transaction(); - tx.addInput(Buffer.from(funding, 'hex').reverse(), 0); - tx.addOutput(address.toOutputScript(recipient, networks.regtest), sats); - return tx; -} -async function server( - t: test.TestContext, - handle: (socket: net.Socket, request: { id: number; method: string; params: unknown[] }) => void -) { - const sockets = new Set(); - const server = net.createServer((socket) => { - sockets.add(socket); - socket.on('close', () => sockets.delete(socket)); - let data = ''; - socket.on('data', (chunk) => { - data += chunk; - if (data.includes('\n')) handle(socket, JSON.parse(data)); - }); - }); - await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); - t.after(async () => { - for (const s of sockets) s.destroy(); - await new Promise((resolve) => server.close(() => resolve())); - }); - return { - host: '127.0.0.1', - port: (server.address() as net.AddressInfo).port, - protocol: 'tcp' as const, - }; -} -test('fresh recipients are distinct valid regtest addresses', () => { - const one = freshRegtestRecipient(), - two = freshRegtestRecipient(); - assert.notEqual(one, two); - assert.match(one, /^bcrt1/); - assert.match(scriptHash(one), /^[a-f0-9]{64}$/); -}); -test('oracle verifies raw transaction identity, exact output and wallet funding input', () => { - const recipient = freshRegtestRecipient(), - tx = transaction(recipient); - assert.equal( - verifyPayment(tx.toHex(), tx.getId(), recipient, 10_000, 'ab'.repeat(32)).txid, - tx.getId() - ); - assert.throws(() => verifyPayment(tx.toHex(), 'cd'.repeat(32), recipient, 10_000), /identity/); - assert.throws( - () => verifyPayment(tx.toHex(), tx.getId(), freshRegtestRecipient(), 10_000), - /recipient\/amount/ - ); - assert.throws(() => verifyPayment(tx.toHex(), tx.getId(), recipient, 9_999), /recipient\/amount/); - assert.throws( - () => verifyPayment(tx.toHex(), tx.getId(), recipient, 10_000, 'cd'.repeat(32)), - /funding transaction/ - ); -}); -test('unrelated concurrent transactions cannot satisfy another case', () => { - const first = freshRegtestRecipient(), - second = freshRegtestRecipient(); - const firstTx = transaction(first), - secondTx = transaction(second, 10_000, 'cd'.repeat(32)); - assert.throws( - () => verifyPayment(secondTx.toHex(), secondTx.getId(), first, 10_000, 'ab'.repeat(32)), - /recipient/ - ); - assert.equal( - verifyPayment(firstTx.toHex(), firstTx.getId(), first, 10_000, 'ab'.repeat(32)).address, - first - ); -}); -test('split TCP responses and notifications are handled without accepting another request ID', async (t) => { - const endpoint = await server(t, (socket) => { - socket.write('{"id":2,"result":"wrong"}\n{"id":1,"res'); - socket.end('ult":"correct"}\n'); - }); - assert.equal(await electrumRequest(endpoint, 'server.version', []), 'correct'); -}); -test('a nonresponding backend is bounded and closes the connection', async (t) => { - const endpoint = await server(t, () => {}); - await assert.rejects(electrumRequest(endpoint, 'server.version', [], 30), /timed out/); -}); -test('closed connections and backend errors cannot become successful readiness', async (t) => { - const closed = await server(t, (socket) => socket.end()); - await assert.rejects(electrumRequest(closed, 'server.version', []), /closed before/); - const error = await server(t, (socket) => - socket.end('{"id":1,"error":{"code":-1,"message":"unavailable"}}\n') - ); - await assert.rejects(electrumRequest(error, 'server.version', []), /backend rejected/); -}); -test('malformed history fails rather than looking like an empty ready wallet', async (t) => { - const endpoint = await server(t, (socket) => socket.end('{"id":1,"result":null}\n')); - await assert.rejects(history(endpoint, freshRegtestRecipient()), /Malformed.*history/); -}); -test('end-to-end oracle reads history then independently decodes the payment', async (t) => { - const recipient = freshRegtestRecipient(), - tx = transaction(recipient); - const endpoint = await server(t, (socket, request) => - socket.end( - JSON.stringify({ - id: 1, - result: request.method.endsWith('get_history') - ? [{ tx_hash: tx.getId(), height: 1 }] - : tx.toHex(), - }) + '\n' - ) - ); - const result = await waitForPayment(endpoint, recipient, 10_000, { spendsTxid: 'ab'.repeat(32) }); - assert.equal(result.txid, tx.getId()); -}); - -test('BIP21 amounts compare exact satoshis without floating-point rounding', () => { - assert.equal(bitcoinAmountToSats('0.00000123'), 123); - assert.equal(bitcoinAmountToSats('0.00000001'), 1); - assert.equal(bitcoinAmountToSats('1.1'), 110_000_000); - assert.notEqual(bitcoinAmountToSats('0.00000124'), 123); - for (const value of ['1e-8', '-0.1', '0.000000001', 'NaN', '', '1.2x', '999999999999999']) { - assert.throws(() => bitcoinAmountToSats(value), /BIP21/); - } -}); -test('TLS endpoints send a TLS handshake instead of plaintext RPC', async (t) => { - let firstByte: number | undefined; - const listener = net.createServer((socket) => - socket.once('data', (data) => { - firstByte = data[0]; - socket.destroy(); - }) - ); - await new Promise((resolve) => listener.listen(0, '127.0.0.1', resolve)); - t.after(() => new Promise((resolve) => listener.close(() => resolve()))); - await assert.rejects( - electrumRequest( - { host: '127.0.0.1', port: (listener.address() as net.AddressInfo).port, protocol: 'tls' }, - 'server.version', - [], - 1000 - ), - /Smoke backend/ - ); - assert.equal(firstByte, 0x16, 'First record must be TLS handshake, not plaintext JSON'); -}); From a4996b56f8a73e4d9bbc2fb300a05000595bd6cd Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Tue, 3 Nov 2026 21:00:30 +0100 Subject: [PATCH 06/18] Serialize planned specs and pin migration fixture source --- .github/workflows/inventory.yml | 4 + .github/workflows/migration-wallet-setup.yml | 49 ++--- package.json | 2 +- scripts/ci-run.ts | 212 ++++++++++--------- scripts/verify-e2e-sha.sh | 6 + test/reporting/specs.ts | 10 + test/unit/pinned-fixture-source.test.cjs | 41 ++++ test/unit/wdio-scheduling.test.cjs | 44 ++++ tsconfig.json | 8 +- wdio.conf.ts | 5 +- 10 files changed, 249 insertions(+), 132 deletions(-) create mode 100755 scripts/verify-e2e-sha.sh create mode 100644 test/reporting/specs.ts create mode 100644 test/unit/pinned-fixture-source.test.cjs create mode 100644 test/unit/wdio-scheduling.test.cjs diff --git a/.github/workflows/inventory.yml b/.github/workflows/inventory.yml index 4df92bb7..42bbadec 100644 --- a/.github/workflows/inventory.yml +++ b/.github/workflows/inventory.yml @@ -6,6 +6,8 @@ on: - 'test/**' - 'scripts/e2e-inventory.ts' - 'scripts/ci-*' + - 'scripts/verify-e2e-sha.sh' + - '.github/workflows/migration-wallet-setup.yml' - 'wdio.conf.ts' - 'package*.json' - '.github/workflows/inventory.yml' @@ -15,6 +17,8 @@ on: - 'test/**' - 'scripts/e2e-inventory.ts' - 'scripts/ci-*' + - 'scripts/verify-e2e-sha.sh' + - '.github/workflows/migration-wallet-setup.yml' - 'wdio.conf.ts' - 'package*.json' - '.github/workflows/inventory.yml' diff --git a/.github/workflows/migration-wallet-setup.yml b/.github/workflows/migration-wallet-setup.yml index 065ac080..e9d784be 100644 --- a/.github/workflows/migration-wallet-setup.yml +++ b/.github/workflows/migration-wallet-setup.yml @@ -1,63 +1,69 @@ name: Migration Wallet Setup - on: workflow_call: inputs: e2e_branch: - description: 'Candidate branch of synonymdev/bitkit-e2e-tests. Re-validated immediately before checkout. Callers may pass an already-resolved ref; a missing default companion falls back to main.' + description: Candidate branch of synonymdev/bitkit-e2e-tests. Re-validated immediately before checkout. Callers may pass an already-resolved ref; a missing default companion falls back to main. required: true type: string e2e_branch_input: - description: 'Resolution mode (main | default-feature-branch | custom branch name). Empty inherits the caller workflow_dispatch input e2e_branch, then default-feature-branch. A custom name hard-fails when the branch is missing.' + description: Resolution mode (main | default-feature-branch | custom branch name). Empty inherits the caller workflow_dispatch input e2e_branch, then default-feature-branch. A custom name hard-fails when the branch is missing. required: false type: string - default: '' + default: "" rn_version: - description: 'Legacy RN app version to use for setup (e.g., v1.1.6)' + description: Legacy RN app version to use for setup (e.g., v1.1.6) required: false type: string - default: 'v1.1.6' + default: v1.1.6 setup_type: - description: 'Wallet setup type (standard | passphrase | sweep)' + description: Wallet setup type (standard | passphrase | sweep) required: true type: string scenario_name: - description: 'Migration scenario name for artifact naming' + description: Migration scenario name for artifact naming required: true type: string - + e2e_sha: + description: Optional immutable E2E commit. Checkout must match; never resolve or fall back as a branch. + type: string + default: "" env: TERM: xterm-256color FORCE_COLOR: 1 - jobs: prepare-wallets: runs-on: ubuntu-latest - steps: - # Re-resolve in this job. "Re-run failed jobs" does not re-run a - # successful resolve job in the caller, so inputs.e2e_branch can name - # a companion that was deleted after that job finished. + - name: Validate pinned commit + if: inputs.e2e_sha != '' + env: + E2E_SHA: ${{ inputs.e2e_sha }} + run: "[[ \"$E2E_SHA\" =~ ^[0-9a-f]{40}$ ]] || { echo 'Invalid pinned E2E commit' >&2; exit 1; }" - name: Resolve E2E tests ref id: e2e-ref uses: synonymdev/bitkit-e2e-tests/.github/actions/resolve-e2e-ref@main with: selected: ${{ inputs.e2e_branch }} e2e_branch_input: ${{ inputs.e2e_branch_input || github.event.inputs.e2e_branch || 'default-feature-branch' }} - + if: inputs.e2e_sha == '' - name: Clone E2E tests uses: actions/checkout@v4 with: repository: synonymdev/bitkit-e2e-tests path: bitkit-e2e-tests - ref: ${{ steps.e2e-ref.outputs.ref }} - + ref: ${{ inputs.e2e_sha || steps.e2e-ref.outputs.ref }} + - name: Verify pinned checkout + if: inputs.e2e_sha != '' + working-directory: bitkit-e2e-tests + env: + E2E_SHA: ${{ inputs.e2e_sha }} + run: bash scripts/verify-e2e-sha.sh checkout - name: Enable KVM run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm - - name: Download RN app for migration working-directory: bitkit-e2e-tests env: @@ -66,12 +72,10 @@ jobs: python3 scripts/download-migration-app.py android rn "$RN_VERSION" # WDIO initializes its session using the RN source during preparation. ln -sf bitkit_rn_regtest.apk aut/bitkit_e2e.apk - - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: 22 - - name: Cache npm cache uses: actions/cache@v3 with: @@ -79,11 +83,9 @@ jobs: key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} restore-keys: | ${{ runner.os }}-node- - - name: Install dependencies working-directory: bitkit-e2e-tests run: npm ci - - name: Prepare independent wallets for all consumer attempts uses: reactivecircus/android-emulator-runner@v2 with: @@ -97,7 +99,6 @@ jobs: env: BACKEND: regtest SETUP_TYPE: ${{ inputs.setup_type }} - - name: Upload preparation diagnostics if: always() uses: actions/upload-artifact@v4 @@ -106,7 +107,6 @@ jobs: path: | bitkit-e2e-tests/artifacts/attempt-*/ bitkit-e2e-tests/artifacts/migration-source.json - - name: Upload migration env file uses: actions/upload-artifact@v4 with: @@ -118,3 +118,4 @@ jobs: bitkit-e2e-tests/artifacts/wallet-attempt-3/ if-no-files-found: error retention-days: 1 + timeout-minutes: 90 diff --git a/package.json b/package.json index 30cf446b..9c07c7a1 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "scripts": { "inventory:check": "node --experimental-strip-types scripts/e2e-inventory.ts check", - "test:reporting": "node --experimental-strip-types --test test/unit/inventory.test.ts test/unit/reporting.test.ts test/unit/reporting-cli.test.ts test/unit/reporter-events.test.ts test/unit/ci-selection.test.cjs test/unit/ci-retry.test.ts", + "test:reporting": "node --import tsx --test test/unit/inventory.test.ts test/unit/reporting.test.ts test/unit/reporting-cli.test.ts test/unit/reporter-events.test.ts test/unit/ci-selection.test.cjs test/unit/ci-retry.test.ts test/unit/wdio-scheduling.test.cjs test/unit/pinned-fixture-source.test.cjs", "e2e:android": "PLATFORM=android wdio run ./wdio.conf.ts", "e2e:ios": "PLATFORM=ios wdio run ./wdio.conf.ts", "lint": "eslint './**/*.{ts,js}'", diff --git a/scripts/ci-run.ts b/scripts/ci-run.ts index bd0918d4..494bca1b 100644 --- a/scripts/ci-run.ts +++ b/scripts/ci-run.ts @@ -129,114 +129,118 @@ function summary(plan: Plan) { if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown); return result; } -try { - if (command === 'prepare') { - if (fs.existsSync(planFile)) throw Error('Plan exists: use a fresh run directory'); - const inventory = loadInventory(process.cwd()); - validateInventory(process.cwd(), inventory); - const platform = process.env.PLATFORM as Platform, - backend = (process.env.BACKEND || 'local') as Backend; - const ids: string[] = process.env.E2E_CASE_TAG - ? [process.env.E2E_CASE_TAG.replace(/^@/, '')] - : JSON.parse(process.env.E2E_CASE_IDS || '[]'); - if (!Array.isArray(ids) || !ids.length || new Set(ids).size !== ids.length) - throw Error('Empty or duplicate case selection'); - const cases = inventory.cases.filter((c) => ids.includes(c.id)); - if ( - cases.length !== ids.length || - cases.some( - (c) => - c.kind !== 'test' || - c.platforms[platform]?.disposition !== 'required' || - c.platforms[platform]?.backend !== backend - ) - ) - throw Error('Selection contains unknown, unavailable or wrong-backend cases'); - const metadata = JSON.parse(fs.readFileSync('aut/e2e-build.json', 'utf8')); - if ( - metadata.appSha !== process.env.E2E_APP_SHA || - metadata.backend !== backend || - metadata.platform !== platform - ) - throw Error('Downloaded build does not match selected app/platform/backend'); - const artifact = artifactHash( - path.join( - 'aut', - process.env.AUT_FILENAME || (platform === 'ios' ? 'bitkit.app' : 'bitkit_e2e.apk') +async function main() { + try { + if (command === 'prepare') { + if (fs.existsSync(planFile)) throw Error('Plan exists: use a fresh run directory'); + const inventory = loadInventory(process.cwd()); + validateInventory(process.cwd(), inventory); + const platform = process.env.PLATFORM as Platform, + backend = (process.env.BACKEND || 'local') as Backend; + const ids: string[] = process.env.E2E_CASE_TAG + ? [process.env.E2E_CASE_TAG.replace(/^@/, '')] + : JSON.parse(process.env.E2E_CASE_IDS || '[]'); + if (!Array.isArray(ids) || !ids.length || new Set(ids).size !== ids.length) + throw Error('Empty or duplicate case selection'); + const cases = inventory.cases.filter((c) => ids.includes(c.id)); + if ( + cases.length !== ids.length || + cases.some( + (c) => + c.kind !== 'test' || + c.platforms[platform]?.disposition !== 'required' || + c.platforms[platform]?.backend !== backend + ) ) - ); - process.env.E2E_ARTIFACT_DIGEST = artifact; - if (process.env.GITHUB_ENV) - fs.appendFileSync(process.env.GITHUB_ENV, `E2E_ARTIFACT_DIGEST=${artifact}\n`); - const plan = createPlan( - cases, - executionIdentity(platform, backend), - digest(fs.readFileSync('test/inventory.json', 'utf8')), - 'selected', - 3 - ); - writeJson(planFile, plan); - fs.writeFileSync( - path.join(directory, 'grep.txt'), - `(?:^| )@(?:${plan.expected.join('|')})(?=[ ,]|$)\n` - ); - // Only load selected spec files, avoiding unrelated fixture hooks and environment requirements. - writeJson(path.join(directory, 'specs.json'), [...new Set(cases.map((c) => c.file))]); - } else { - const plan: Plan = JSON.parse(fs.readFileSync(planFile, 'utf8')); - assertExecutionIdentity(plan); - if (command === 'run' || command === 'attempt') { - const currentAttempt = command === 'attempt' ? Number(process.env.ATTEMPT) : 1; + throw Error('Selection contains unknown, unavailable or wrong-backend cases'); + const metadata = JSON.parse(fs.readFileSync('aut/e2e-build.json', 'utf8')); if ( - !Number.isInteger(currentAttempt) || - currentAttempt < 1 || - currentAttempt > plan.maxAttempts + metadata.appSha !== process.env.E2E_APP_SHA || + metadata.backend !== backend || + metadata.platform !== platform ) - throw Error('Invalid attempt'); - if (receipts().length !== currentAttempt - 1) - throw Error('Unexpected attempt receipts; refusing to reuse another execution'); - if (currentAttempt > 1 && reconcileAttempts(plan, workers(), receipts()).status === 'passed') - throw Error('Previous attempt already passed'); - const specs: string[] = JSON.parse( - fs.readFileSync(path.join(directory, 'specs.json'), 'utf8') + throw Error('Downloaded build does not match selected app/platform/backend'); + const artifact = artifactHash( + path.join( + 'aut', + process.env.AUT_FILENAME || (platform === 'ios' ? 'bitkit.app' : 'bitkit_e2e.apk') + ) ); - const grep = fs.readFileSync(path.join(directory, 'grep.txt'), 'utf8').trim(); - const runner = plan.identity.platform === 'ios' ? './ci_run_ios.sh' : './ci_run_android.sh'; - for ( - let attempt = currentAttempt; - attempt <= (command === 'attempt' ? currentAttempt : plan.maxAttempts); - attempt++ - ) { - const receipt: AttemptReceipt = { attempt, startedAt: new Date().toISOString() }; - const file = path.join(directory, `attempt-${attempt}.json`); - writeJson(file, receipt); - const args = specs.flatMap((spec) => ['--spec', spec]).concat(['--mochaOpts.grep', grep]); - const child = await runProcess( - runner, - args, - { ...process.env, ATTEMPT: String(attempt), E2E_PLAN: planFile }, - path.join(directory, `attempt-${attempt}-runner.log`) - ); - receipt.finishedAt = new Date().toISOString(); - receipt.exitCode = child.status ?? 1; - writeJson(file, receipt); - const result = reconcile( - plan, - workers().filter((w) => w.attempt === attempt) - ); - writeJson(path.join(directory, `attempt-${attempt}-summary.json`), { - ...result, - runnerExit: receipt.exitCode, - }); - if (child.cancelled || (receipt.exitCode === 0 && result.status === 'passed')) break; - console.error( - `Attempt ${attempt} failed; evidence retained in ${directory} and artifacts/attempt-${attempt}` - ); - } - } else if (command !== 'report') throw Error('Usage: ci-run.ts prepare|run|attempt|report'); - if (summary(plan).status !== 'passed') process.exitCode = 1; + process.env.E2E_ARTIFACT_DIGEST = artifact; + if (process.env.GITHUB_ENV) + fs.appendFileSync(process.env.GITHUB_ENV, `E2E_ARTIFACT_DIGEST=${artifact}\n`); + const plan = createPlan( + cases, + executionIdentity(platform, backend), + digest(fs.readFileSync('test/inventory.json', 'utf8')), + 'selected', + 3 + ); + writeJson(planFile, plan); + fs.writeFileSync( + path.join(directory, 'grep.txt'), + `(?:^| )@(?:${plan.expected.join('|')})(?=[ ,]|$)\n` + ); + } else { + const plan: Plan = JSON.parse(fs.readFileSync(planFile, 'utf8')); + assertExecutionIdentity(plan); + if (command === 'run' || command === 'attempt') { + const currentAttempt = command === 'attempt' ? Number(process.env.ATTEMPT) : 1; + if ( + !Number.isInteger(currentAttempt) || + currentAttempt < 1 || + currentAttempt > plan.maxAttempts + ) + throw Error('Invalid attempt'); + if (receipts().length !== currentAttempt - 1) + throw Error('Unexpected attempt receipts; refusing to reuse another execution'); + if ( + currentAttempt > 1 && + reconcileAttempts(plan, workers(), receipts()).status === 'passed' + ) + throw Error('Previous attempt already passed'); + const grep = fs.readFileSync(path.join(directory, 'grep.txt'), 'utf8').trim(); + const runner = plan.identity.platform === 'ios' ? './ci_run_ios.sh' : './ci_run_android.sh'; + for ( + let attempt = currentAttempt; + attempt <= (command === 'attempt' ? currentAttempt : plan.maxAttempts); + attempt++ + ) { + const receipt: AttemptReceipt = { attempt, startedAt: new Date().toISOString() }; + const file = path.join(directory, `attempt-${attempt}.json`); + writeJson(file, receipt); + const args = ['--mochaOpts.grep', grep]; + const child = await runProcess( + runner, + args, + { ...process.env, ATTEMPT: String(attempt), E2E_PLAN: planFile }, + path.join(directory, `attempt-${attempt}-runner.log`) + ); + receipt.finishedAt = new Date().toISOString(); + receipt.exitCode = child.status ?? 1; + writeJson(file, receipt); + const result = reconcile( + plan, + workers().filter((w) => w.attempt === attempt) + ); + writeJson(path.join(directory, `attempt-${attempt}-summary.json`), { + ...result, + runnerExit: receipt.exitCode, + }); + if (child.cancelled || (receipt.exitCode === 0 && result.status === 'passed')) break; + console.error( + `Attempt ${attempt} failed; evidence retained in ${directory} and artifacts/attempt-${attempt}` + ); + } + } else if (command !== 'report') throw Error('Usage: ci-run.ts prepare|run|attempt|report'); + if (summary(plan).status !== 'passed') process.exitCode = 1; + } + } catch (error) { + console.error(error); + process.exitCode = 1; } -} catch (error) { +} +void main().catch((error) => { console.error(error); process.exitCode = 1; -} +}); diff --git a/scripts/verify-e2e-sha.sh b/scripts/verify-e2e-sha.sh new file mode 100755 index 00000000..f1fed849 --- /dev/null +++ b/scripts/verify-e2e-sha.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +[[ "${E2E_SHA:-}" =~ ^[0-9a-f]{40}$ ]] || { echo 'Invalid pinned E2E commit' >&2; exit 1; } +if [[ "${1:-}" == checkout ]]; then + [[ "$(git rev-parse HEAD)" == "$E2E_SHA" ]] || { echo 'Checkout differs from pinned E2E commit' >&2; exit 1; } +fi diff --git a/test/reporting/specs.ts b/test/reporting/specs.ts new file mode 100644 index 00000000..f68a029e --- /dev/null +++ b/test/reporting/specs.ts @@ -0,0 +1,10 @@ +import fs from 'node:fs'; +import { validatePlan, type Plan } from './report.ts'; +import { assertExecutionIdentity } from './identity.ts'; + +export function selectedSpecs(file: string): string[] { + const plan: Plan = JSON.parse(fs.readFileSync(file, 'utf8')); + validatePlan(plan); + assertExecutionIdentity(plan); + return [...new Set(plan.cases.filter((c) => plan.expected.includes(c.id)).map((c) => c.file))]; +} diff --git a/test/unit/pinned-fixture-source.test.cjs b/test/unit/pinned-fixture-source.test.cjs new file mode 100644 index 00000000..0b381e89 --- /dev/null +++ b/test/unit/pinned-fixture-source.test.cjs @@ -0,0 +1,41 @@ +const test = require('node:test'), + assert = require('node:assert/strict'), + fs = require('node:fs'), + os = require('node:os'), + path = require('node:path'), + { spawnSync, execFileSync } = require('node:child_process'), + Y = require('yaml'); +test('migration fixture bypasses branch resolution only for explicit commit pins', () => { + const w = Y.parse(fs.readFileSync('.github/workflows/migration-wallet-setup.yml', 'utf8')); + const steps = w.jobs['prepare-wallets'].steps; + assert.equal(steps.find((s) => s.id === 'e2e-ref').if, "inputs.e2e_sha == ''"); + assert.equal( + steps.find((s) => s.name === 'Clone E2E tests').with.ref, + '${{ inputs.e2e_sha || steps.e2e-ref.outputs.ref }}' + ); + const validation = steps.find((s) => s.name === 'Validate pinned commit'); + for (const pin of ['branch', 'a'.repeat(39), 'a'.repeat(40) + '; echo bad']) + assert.notEqual( + spawnSync('bash', ['-e', '-c', validation.run], { env: { ...process.env, E2E_SHA: pin } }) + .status, + 0 + ); + const head = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(); + assert.equal( + spawnSync('bash', ['-e', '-c', validation.run], { env: { ...process.env, E2E_SHA: head } }) + .status, + 0 + ); + assert.equal( + spawnSync('bash', ['scripts/verify-e2e-sha.sh', 'checkout'], { + env: { ...process.env, E2E_SHA: head }, + }).status, + 0 + ); + assert.notEqual( + spawnSync('bash', ['scripts/verify-e2e-sha.sh', 'checkout'], { + env: { ...process.env, E2E_SHA: 'f'.repeat(40) }, + }).status, + 0 + ); +}); diff --git a/test/unit/wdio-scheduling.test.cjs b/test/unit/wdio-scheduling.test.cjs new file mode 100644 index 00000000..f18e1567 --- /dev/null +++ b/test/unit/wdio-scheduling.test.cjs @@ -0,0 +1,44 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { fileURLToPath } = require('node:url'); +const { loadInventory } = require('../reporting/inventory.ts'); +const { executionIdentity } = require('../reporting/identity.ts'); +const { createPlan, writeJson } = require('../reporting/report.ts'); + +test('actual WDIO config schedules selected specs together on one simulator', async (t) => { + const before = { ...process.env }; + t.after(() => { + for (const k of Object.keys(process.env)) if (!(k in before)) delete process.env[k]; + Object.assign(process.env, before); + }); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'wdio-plan-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + Object.assign(process.env, { + PLATFORM: 'ios', + BACKEND: 'local', + E2E_RUN_ID: 'schedule-test', + E2E_APP_SHA: 'a'.repeat(40), + E2E_ARTIFACT_DIGEST: 'b'.repeat(64), + E2E_BACKEND_REVISION: 'local', + E2E_VARIANT: 'default', + }); + const ids = ['send_1', 'send_2', 'lightning_1', 'transfer_2', 'boost_2']; + const cases = loadInventory(process.cwd()).cases.filter((c) => ids.includes(c.id)); + const plan = createPlan(cases, executionIdentity('ios', 'local'), 'c'.repeat(64), 'selected', 3); + process.env.E2E_PLAN = path.join(root, 'plan.json'); + writeJson(process.env.E2E_PLAN, plan); + const { ConfigParser } = await import('@wdio/config/node'); + const parser = new ConfigParser(path.resolve('wdio.conf.ts')); + await parser.initialize(); + const specs = parser.getSpecs(); + assert.equal(specs.length, 1, 'Exactly one worker group per simulator'); + assert.ok(Array.isArray(specs[0])); + assert.deepEqual( + specs[0].map((f) => path.relative(process.cwd(), fileURLToPath(f))).sort(), + [...new Set(cases.map((c) => c.file))].sort() + ); + assert.equal(parser.getConfig().maxInstances, 1); +}); diff --git a/tsconfig.json b/tsconfig.json index 5d19c29f..c9151d8e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -22,5 +22,11 @@ "noFallthroughCasesInSwitch": true, "esModuleInterop": true }, - "include": ["test", "test/types/**/*.d.ts", "wdio.conf.ts", "scripts/e2e-inventory.ts"] + "include": [ + "test", + "test/types/**/*.d.ts", + "wdio.conf.ts", + "scripts/e2e-inventory.ts", + "scripts/ci-run.ts" + ] } diff --git a/wdio.conf.ts b/wdio.conf.ts index e8a69069..11e6ec32 100644 --- a/wdio.conf.ts +++ b/wdio.conf.ts @@ -3,6 +3,7 @@ import fs from 'node:fs'; import { grantIOSCameraPermission } from './test/helpers/setup'; import InventoryReporter from './test/reporting/reporter'; +import { selectedSpecs } from './test/reporting/specs'; const appiumPort = Number(process.env.APPIUM_PORT ?? '4723'); const isAndroid = process.env.PLATFORM === 'android'; @@ -48,7 +49,7 @@ export const config: WebdriverIO.Config = { // // Spec in [] will be run in a separate worker process. // Running all specs sequentially for now. - specs: [['./test/specs/**/*.ts']], + specs: process.env.E2E_PLAN ? [selectedSpecs(process.env.E2E_PLAN)] : [['./test/specs/**/*.ts']], // Patterns to exclude. exclude: [ // 'path/to/excluded/files' @@ -69,7 +70,7 @@ export const config: WebdriverIO.Config = { // and 30 processes will get spawned. The property handles how many capabilities // from the same test should run tests. // - maxInstances: 10, + maxInstances: process.env.E2E_PLAN ? 1 : 10, // // If you have trouble getting all important capabilities together, check out the // Sauce Labs platform configurator - a great tool to configure your capabilities: From 39f8964ee28c801874bae1cc7667505c9bb03065 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Tue, 3 Nov 2026 21:01:55 +0100 Subject: [PATCH 07/18] Use supported cache action for migration fixtures --- .github/workflows/migration-wallet-setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/migration-wallet-setup.yml b/.github/workflows/migration-wallet-setup.yml index e9d784be..74fb45ec 100644 --- a/.github/workflows/migration-wallet-setup.yml +++ b/.github/workflows/migration-wallet-setup.yml @@ -77,7 +77,7 @@ jobs: with: node-version: 22 - name: Cache npm cache - uses: actions/cache@v3 + uses: actions/cache@v6 with: path: ~/.npm key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} From ee9e8bed6147ce0cc6ccb446e7df598e9614ff31 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Tue, 3 Nov 2026 22:39:41 +0100 Subject: [PATCH 08/18] Move app path selection out of shared E2E repository --- AGENTS.md | 2 +- docs/inventory-reporting.md | 6 +- package.json | 2 +- scripts/ci-selection.cjs | 198 -------------------------------- test/unit/ci-selection.test.cjs | 77 ------------- 5 files changed, 6 insertions(+), 279 deletions(-) delete mode 100644 scripts/ci-selection.cjs delete mode 100644 test/unit/ci-selection.test.cjs diff --git a/AGENTS.md b/AGENTS.md index 87ad379a..d47c0745 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -171,7 +171,7 @@ Implication for feature work: ## iOS CI selection -The iOS consumer selects cases from `test/inventory.json` using `scripts/ci-selection.cjs`, not the historical `@ios_gate` and `@ios_nightly` tags. Its base cases are `send_1`, `send_2`, `lightning_1`, `transfer_2`, `boost_2`; automatic risk selection expands coverage. Nightly runs all applicable local/staging/migration cases. See [inventory and reporting](docs/inventory-reporting.md). +The iOS consumer owns its path policy in `bitkit-ios/.github/e2e-policy.json` and selector in `.github/scripts/e2e-selection.cjs`. It reads this repo's `test/inventory.json`; iOS paths and selection policy do not belong in this shared repo. Historical `@ios_gate` and `@ios_nightly` tags do not control that consumer. Its base cases are `send_1`, `send_2`, `lightning_1`, `transfer_2`, `boost_2`; automatic risk selection expands coverage. Nightly runs all applicable local/staging/migration cases. See [inventory and reporting](docs/inventory-reporting.md). Keep capability tags for Android's existing selectors and manual execution. Android workflows and selections are unchanged. Do not rename tags merely to match the iOS policy. diff --git a/docs/inventory-reporting.md b/docs/inventory-reporting.md index acd67c12..59f5de4a 100644 --- a/docs/inventory-reporting.md +++ b/docs/inventory-reporting.md @@ -10,9 +10,11 @@ Every iOS app-code PR starts with these existing cases, unchanged: - `send_1`, `send_2`, `lightning_1`, `transfer_2`, `boost_2`. -`scripts/ci-selection.cjs` adds cases by changed path. Hardware adds active hardware tests; LNURL adds its case; address/coin-selection adds multi-address, on-chain and boost tests; fees add boost/on-chain/send; onboarding/backup/recovery adds its tests and migration; security adds security and payment checks; Pubky/Paykit/profile/contact changes add the staging pack; receive/send/lightning/settings/widgets add the corresponding cases. Rules are additive and IDs are deduplicated. +The iOS repo owns `.github/e2e-policy.json`, its selector and policy tests. Its tested app checkout supplies the policy; this repository supplies only the case inventory and execution/reporting. Change iOS paths and their policy together in the same PR. See [iOS policy documentation](https://github.com/synonymdev/bitkit-ios/blob/master/Docs/e2e-ci.md). -Shared services, repositories, models, dependencies, build/CI configuration, USDT changes and unclassified code paths select all applicable local, staging and migration packs. Both sides of renamed files are considered. Documentation-only changes skip device jobs. The selection artifact records the tested app SHA, E2E SHA, cases, reasons, lanes and explicit coverage gaps. +Feature directories and explicit service/model/helper files add only their relevant existing cases. For example, widget services add widget cases, Paykit services add staging, and backup services add recovery/migration. Only named shared wallet/lifecycle dependencies and build/orchestration configuration select every lane. There is no blanket Services/Models/Managers/Utilities rule. + +Cheap iOS validation checks all tracked paths, stale policy entries and referenced case IDs. Unknown paths fail with an actionable policy-update error before paid builds. Old and new rename paths contribute coverage from the base and tested policies, so a refactor cannot silently drop coverage. Documentation/native-test/policy-only changes skip device jobs. Selection records cases, reasons, app/E2E/policy identity and explicit gaps. USDT-specific paths retain the Bitcoin base gate and report the lack of existing USDT E2E coverage; unrelated Bitcoin tests do not establish USDT correctness. PRs stacked on another branch retain the existing automatic exclusion. Manual dispatch with `pr` checks the PR's current merge against its actual base/head, including stacked PRs. The workflow source is the branch selected for dispatch. `selection=auto` uses the PR diff; `base` runs the five cases; `full` runs all applicable lanes. Automatic PRs cannot narrow selection with a manual input. diff --git a/package.json b/package.json index 9c07c7a1..75c38821 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "scripts": { "inventory:check": "node --experimental-strip-types scripts/e2e-inventory.ts check", - "test:reporting": "node --import tsx --test test/unit/inventory.test.ts test/unit/reporting.test.ts test/unit/reporting-cli.test.ts test/unit/reporter-events.test.ts test/unit/ci-selection.test.cjs test/unit/ci-retry.test.ts test/unit/wdio-scheduling.test.cjs test/unit/pinned-fixture-source.test.cjs", + "test:reporting": "node --import tsx --test test/unit/inventory.test.ts test/unit/reporting.test.ts test/unit/reporting-cli.test.ts test/unit/reporter-events.test.ts test/unit/ci-retry.test.ts test/unit/wdio-scheduling.test.cjs test/unit/pinned-fixture-source.test.cjs", "e2e:android": "PLATFORM=android wdio run ./wdio.conf.ts", "e2e:ios": "PLATFORM=ios wdio run ./wdio.conf.ts", "lint": "eslint './**/*.{ts,js}'", diff --git a/scripts/ci-selection.cjs b/scripts/ci-selection.cjs deleted file mode 100644 index b6cf5a2d..00000000 --- a/scripts/ci-selection.cjs +++ /dev/null @@ -1,198 +0,0 @@ -const fs = require('node:fs'); -const path = require('node:path'); -const { execFileSync } = require('node:child_process'); - -const BASE = ['send_1', 'send_2', 'lightning_1', 'transfer_2', 'boost_2']; -const RULES = [ - { name: 'hardware', match: /hardware|trezor|jade/i, ids: /^hardware_wallet_/ }, - { name: 'lnurl', match: /lnurl/i, ids: /^lnurl_/ }, - { - name: 'addresses', - match: /multi.?address|address.?type|address.?viewer|utxo|coin.?selection/i, - ids: /^(multi_address_|onchain_|boost_)/, - }, - { - name: 'fees', - match: /boost|cpfp|rbf|fee|transaction.?speed/i, - ids: /^(boost_|onchain_|send_)/, - }, - { - name: 'onboarding', - match: /onboarding|mnemonic|seedqr|passphrase|backup|restore|recovery|migration/i, - ids: /^(onboarding_|backup_|security_|migration_(?!setup)|settings_07)/, - migration: true, - }, - { - name: 'security', - match: /security|privacy|pin(code|pad)?|biometric|keychain|authcheck/i, - ids: /^(security_|settings_06|settings_07|send_)/, - }, - { - name: 'pubky-paykit', - match: /pubky|paykit|contact|profile|subscription|allowance|payment.?request|marketplace|shop/i, - ids: /^(pubky_profile_|paykit_)/, - staging: true, - }, - { - name: 'receive', - match: /receive|invoice|scanner|qrscann/i, - ids: /^(receive_|numberpad_|send_|onchain_|lnurl_)/, - }, - { - name: 'send', - match: /send|quickpay|amount|numberpad/i, - ids: /^(send_|numberpad_|onchain_|boost_|lnurl_)/, - }, - { - name: 'lightning', - match: /lightning|channel|transfer|connection/i, - ids: /^(lightning_|transfer_|multi_address_|send_|lnurl_)/, - }, - { name: 'settings', match: /settings|electrum|gossip|currency|denomination/i, ids: /^settings_/ }, - { name: 'widgets', match: /widget/i, ids: /^widgets_/ }, -]; -// Shared services, dependencies, routing and unknown paths expand rather than guess. -const BROAD = - /(^|\/)(services?|repositories|managers|data|models|helpers|navigation|utilities|extensions)(\/|$)|appscene|appviewmodel|mainnav|mainactivity|lightningrepo|\.github\/|\.gradle|gradle\/|Package\.(swift|resolved)|project\.pbxproj|\.xcconfig|\.toml$|\.properties$|usdt|bitkit.?core|ldk/i; -const DOCUMENTATION = - /^(docs?\/|journeys\/|changelog\.d\/)|(^|\/)(README|CHANGELOG|AGENTS|CLAUDE)\.md$|\.md$/i; - -function select(cases, { platform, mode = 'auto', files = [], shards = 1, release = false }) { - if (!['ios', 'android'].includes(platform)) throw Error('Invalid platform'); - if (!['auto', 'base', 'full', 'nightly'].includes(mode)) throw Error('Invalid selection mode'); - if (![1, 2].includes(shards)) throw Error('Only one or two shards are supported'); - if (!Array.isArray(files) || files.some((f) => typeof f !== 'string' || !f || f.includes('\0'))) - throw Error('Invalid changed paths'); - const runnable = cases.filter( - (c) => c.kind === 'test' && c.platforms[platform].disposition === 'required' - ); - for (const id of BASE) - if (!runnable.some((c) => c.id === id && c.platforms[platform].backend === 'local')) - throw Error(`Missing base case: ${id}`); - const local = runnable.filter((c) => c.platforms[platform].backend === 'local'); - const selected = new Set(BASE); - const reasons = []; - let broad = ['full', 'nightly'].includes(mode); - let staging = broad, - migration = broad || release; - const changed = files.filter((f) => !DOCUMENTATION.test(f)); - const code = mode !== 'auto' || changed.length > 0 || release; - if (mode === 'auto') - for (const file of changed) { - const rules = RULES.filter((r) => r.match.test(file)); - if (BROAD.test(file) || rules.length === 0) { - broad = true; - reasons.push({ file, rule: 'broad-or-unclassified' }); - } else - for (const rule of rules) { - for (const c of runnable) if (rule.ids.test(c.id)) selected.add(c.id); - staging ||= !!rule.staging; - migration ||= !!rule.migration; - reasons.push({ file, rule: rule.name }); - } - } - if (broad) { - for (const c of local) selected.add(c.id); - staging = true; - migration = true; - } - const ids = code - ? local - .filter((c) => selected.has(c.id)) - .map((c) => c.id) - .sort() - : []; - // Keep a spec's cases together: before hooks can own a wallet shared by sibling cases. - const bins = Array.from({ length: shards }, () => []); - const specs = [...new Set(local.filter((c) => ids.includes(c.id)).map((c) => c.file))].sort(); - for (const file of specs) { - const group = local.filter((c) => c.file === file && ids.includes(c.id)).map((c) => c.id); - bins.sort((a, b) => a.length - b.length); - bins[0].push(...group); - } - const matrix = bins - .filter((b) => b.length) - .map((ids, i) => ({ name: `local-${i + 1}`, ids: ids.sort() })); - return { - schemaVersion: 1, - platform, - mode, - code, - base: BASE, - ids, - matrix, - staging: code && staging, - migration: code && migration, - reasons, - gaps: cases - .filter((c) => c.platforms[platform].disposition === 'gap') - .map((c) => ({ id: c.id, ...c.platforms[platform] })), - }; -} - -if (require.main === module) { - const app = path.resolve(process.env.E2E_APP_ROOT || '.'); - const git = (...args) => execFileSync('git', ['-C', app, ...args], { encoding: 'utf8' }).trim(); - const tested = git('rev-parse', 'HEAD'); - const event = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT_PATH, 'utf8')); - const pr = event.pull_request; - const manualPr = process.env.E2E_MANUAL_PR; - let base = pr?.base?.sha; - if (manualPr) { - if (!/^\d+$/.test(manualPr)) throw Error('Invalid PR number'); - const current = JSON.parse( - execFileSync('gh', ['api', `repos/${process.env.GITHUB_REPOSITORY}/pulls/${manualPr}`], { - encoding: 'utf8', - }) - ); - base = current.base.sha; - if (git('show', '-s', '--format=%P', 'HEAD') !== `${base} ${current.head.sha}`) - throw Error('Manual PR merge does not match current base and head'); - } else if (pr && git('show', '-s', '--format=%P', 'HEAD') !== `${base} ${pr.head.sha}`) - throw Error('PR merge does not match event base and head'); - const files = base - ? execFileSync('git', ['-C', app, 'diff', '--no-renames', '--name-only', '-z', base, tested], { - encoding: 'utf8', - }) - .split('\0') - .filter(Boolean) - : []; - const mode = - process.env.GITHUB_EVENT_NAME === 'schedule' - ? 'nightly' - : process.env.GITHUB_EVENT_NAME === 'pull_request' - ? 'auto' - : process.env.E2E_SELECTION || 'full'; - if (mode === 'auto' && !base) throw Error('Automatic manual selection requires a PR number'); - const shards = - process.env.GITHUB_EVENT_NAME === 'workflow_dispatch' ? Number(process.env.E2E_SHARDS || 1) : 1; - const inventory = JSON.parse( - fs.readFileSync(path.join(__dirname, '../test/inventory.json'), 'utf8') - ); - const result = select(inventory.cases, { - platform: process.env.PLATFORM, - mode, - files, - shards, - release: /^release-/.test(pr?.head?.ref || '') || /^release-/.test(pr?.base?.ref || ''), - }); - result.appSha = tested; - result.e2eSha = execFileSync('git', ['rev-parse', 'HEAD'], { - cwd: path.join(__dirname, '..'), - encoding: 'utf8', - }).trim(); - fs.writeFileSync('selection.json', JSON.stringify(result, null, 2) + '\n'); - if (process.env.GITHUB_OUTPUT) - for (const key of ['code', 'matrix', 'staging', 'migration', 'appSha', 'e2eSha']) - fs.appendFileSync( - process.env.GITHUB_OUTPUT, - `${key}=${typeof result[key] === 'object' ? JSON.stringify(result[key]) : result[key]}\n` - ); - if (process.env.GITHUB_STEP_SUMMARY) - fs.appendFileSync( - process.env.GITHUB_STEP_SUMMARY, - `## E2E selection\n\nApp: ${tested}; tests: ${result.e2eSha}\n\nLocal cases (${result.ids.length}): ${result.ids.join(', ') || 'none'}\n\nStaging: ${result.staging}; migration: ${result.migration}; shards: ${result.matrix.length}\n\n${result.reasons.map((r) => `- ${r.file}: ${r.rule}`).join('\n')}\n\nKnown gaps (not counted as passes): ${result.gaps.map((g) => g.id).join(', ')}\n` - ); - console.log(JSON.stringify(result, null, 2)); -} -module.exports = { BASE, RULES, select }; diff --git a/test/unit/ci-selection.test.cjs b/test/unit/ci-selection.test.cjs deleted file mode 100644 index 500932d1..00000000 --- a/test/unit/ci-selection.test.cjs +++ /dev/null @@ -1,77 +0,0 @@ -const test = require('node:test'), - assert = require('node:assert/strict'); -const { select, BASE } = require('../../scripts/ci-selection.cjs'); -const cases = require('../inventory.json').cases; -const pick = (files, extra = {}) => - select(cases, { platform: 'ios', mode: 'auto', files, ...extra }); -test('base is exactly the agreed five existing cases', () => - assert.deepEqual([...pick(['Bitkit/Views/Send/SendAmount.swift']).base].sort(), [ - 'boost_2', - 'lightning_1', - 'send_1', - 'send_2', - 'transfer_2', - ])); -test('documentation-only changes avoid device jobs', () => { - const s = pick(['Docs/guide.md']); - assert.equal(s.code, false); - assert.deepEqual(s.matrix, []); -}); -test('hardware changes add all active hardware cases once', () => { - const s = pick(['Bitkit/Views/Hardware/HardwareSend.swift']); - for (const id of [...BASE, 'hardware_wallet_1', 'hardware_wallet_2', 'hardware_wallet_4']) - assert.ok(s.ids.includes(id)); - assert.ok(!s.ids.includes('hardware_wallet_3')); - assert.equal(new Set(s.ids).size, s.ids.length); -}); -test('LNURL and address changes add their risk coverage', () => { - assert.ok(pick(['Bitkit/Views/Lnurl/LnurlAuth.swift']).ids.includes('lnurl_1')); - const s = pick(['Bitkit/Views/Send/UtxoSelection.swift']); - assert.ok(s.ids.includes('multi_address_3')); - assert.ok(s.ids.includes('multi_address_4')); -}); -test('fee changes add CPFP as well as RBF', () => - assert.ok(pick(['Bitkit/Views/Send/FeePicker.swift']).ids.includes('boost_1'))); -test('Pubky and recovery changes select their required executor', () => { - assert.equal(pick(['Bitkit/Views/Profile/Profile.swift']).staging, true); - assert.equal(pick(['Bitkit/Views/Backup/Restore.swift']).migration, true); -}); -for (const file of [ - 'Bitkit/Services/UnknownService.swift', - 'Package.resolved', - 'Bitkit/CompletelyNewFeature.swift', - '.github/workflows/e2e-tests.yml', - 'Bitkit/Assets.xcassets/new.png', -]) - test(`broad/unknown ${file} selects full coverage`, () => { - const s = pick([file]); - assert.equal(s.ids.length, 42); - assert.equal(s.staging, true); - assert.equal(s.migration, true); - }); -test('old and new rename paths both contribute risks', () => { - const s = pick(['Bitkit/Views/Hardware/Old.swift', 'Bitkit/Views/Lnurl/New.swift']); - assert.ok(s.ids.includes('hardware_wallet_4')); - assert.ok(s.ids.includes('lnurl_1')); -}); -test('nightly runs full local and specialized coverage with no changes', () => { - const s = pick([], { mode: 'nightly' }); - assert.equal(s.ids.length, 42); - assert.equal(s.staging, true); - assert.equal(s.migration, true); - assert.ok(s.ids.includes('onchain_3')); -}); -test('two shards partition exact cases without splitting spec fixtures', () => { - const s = pick([], { mode: 'full', shards: 2 }); - assert.equal(s.matrix.length, 2); - assert.deepEqual(s.matrix.flatMap((x) => x.ids).sort(), s.ids); - for (const file of new Set(cases.map((c) => c.file))) { - const ids = cases.filter((c) => c.file === file).map((c) => c.id); - assert.ok(s.matrix.filter((shard) => shard.ids.some((id) => ids.includes(id))).length <= 1); - } -}); -test('known gaps remain explicit instead of executed as passing coverage', () => { - const s = pick([], { mode: 'nightly' }); - assert.ok(s.gaps.some((g) => g.id === 'transfer_1')); - assert.ok(!s.ids.includes('transfer_1')); -}); From 68687142f25ad1d06a5f6694957b3c232cd134fb Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Tue, 3 Nov 2026 23:26:47 +0100 Subject: [PATCH 09/18] test: keep legacy sweep fixture on wallet home --- test/specs/migration.e2e.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/specs/migration.e2e.ts b/test/specs/migration.e2e.ts index 07dd6780..cc67b5c9 100644 --- a/test/specs/migration.e2e.ts +++ b/test/specs/migration.e2e.ts @@ -845,7 +845,9 @@ async function sendRnOnchain( await mineBlocks(1); await electrumClient?.waitForSync(); await sleep(1000); - await dismissSheetRN(); + // Close already dismissed SendSuccess. Another Android Back from Home + // backgrounds the RN app, leaving the sweep fixture unable to read its balance. + await elementById('TotalBalance').waitForDisplayed(); console.info(`→ Sent ${sats} sats`); } From d2aea6b64a29b31d05a673b45a35389b424a4206 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Tue, 3 Nov 2026 23:43:13 +0100 Subject: [PATCH 10/18] ci: seal bounded execution budgets into test plans --- docs/inventory-reporting.md | 4 +++- scripts/ci-run.ts | 26 +++++++++++++++++++----- test/reporting/report.ts | 14 ++++++++++--- test/unit/ci-retry.test.ts | 40 ++++++++++++++++++++++++++++++------- 4 files changed, 68 insertions(+), 16 deletions(-) diff --git a/docs/inventory-reporting.md b/docs/inventory-reporting.md index 59f5de4a..492e0a27 100644 --- a/docs/inventory-reporting.md +++ b/docs/inventory-reporting.md @@ -40,7 +40,7 @@ A recovered failure is reported as a recovery, with the first failed/missing cas ## Cost controls and manual validation -Automatic runs use one local shard. Only manual dispatch can request two, partitioned by spec so a spec's fixture lifecycle stays together. Namespace supplies a separate Mac and backend instance per shard; every shard destroys its backend in cleanup. The local app is built once and reused. Staging and migration share one remote-regtest app build in the combined run. Builds have a 40-minute limit; execution jobs 135 minutes; individual runner attempts 40 minutes. Artifacts are retained for seven days and superseded runs cancel. +Automatic runs use one local shard. Only manual dispatch can request two, partitioned by spec so a spec's fixture lifecycle stays together. Namespace supplies a separate Mac and backend instance per shard; every shard destroys its backend in cleanup. The local app is built once and reused. Staging and migration share one remote-regtest app build in the combined run. Builds have a 40-minute limit; execution limits depend on the selected pack as described below. Artifacts are retained for seven days and superseded runs cancel. Compare `selection=full, shards=1` and `selection=full, shards=2` on the same app and E2E revisions. Inspect elapsed/build/setup/test time, first-pass success and billed Mac plus backend usage. Two shards are an experiment, not an assumed cost saving or a new automatic default. @@ -55,3 +55,5 @@ npx tsc --noEmit ``` The CLI workflow tests use controlled runner fixtures; they are not evidence that hosted Appium journeys passed. Actual hosted measurements and device results must name their revisions and environments. + +Expanded local selections (more than five cases per shard) allow 90 minutes per attempt, up to three attempts, within a 285-minute job and five-hour backend expiry. Base-sized local selections retain 40-minute attempts and a 135-minute job; staging/migration retain their existing limits. The iOS selector owns these budgets and the shared runner seals its attempt limit into the run plan. Limits are ceilings, not reservations: success stops retries and cleanup destroys the backend immediately. diff --git a/scripts/ci-run.ts b/scripts/ci-run.ts index 494bca1b..3e7277ae 100644 --- a/scripts/ci-run.ts +++ b/scripts/ci-run.ts @@ -11,6 +11,7 @@ import { import { executionIdentity, assertExecutionIdentity } from '../test/reporting/identity.ts'; import { createPlan, + validatePlan, digest, writeJson, reconcile, @@ -21,7 +22,13 @@ import { type AttemptReceipt, } from '../test/reporting/report.ts'; -async function runProcess(runner: string, args: string[], env: NodeJS.ProcessEnv, log: string) { +async function runProcess( + runner: string, + args: string[], + env: NodeJS.ProcessEnv, + log: string, + timeoutMinutes: number +) { const output = fs.createWriteStream(log); const child = spawn(runner, args, { env, detached: true, stdio: ['ignore', 'pipe', 'pipe'] }); let cancelled = false; @@ -49,9 +56,12 @@ async function runProcess(runner: string, args: string[], env: NodeJS.ProcessEnv const timer = setTimeout( () => { timedOut = true; + console.error( + `Runner attempt exceeded ${timeoutMinutes} minutes; terminating its process group` + ); stop(); }, - 40 * 60 * 1000 + timeoutMinutes * 60 * 1000 ); child.stdout?.on('data', (data) => { process.stdout.write(data); @@ -116,7 +126,7 @@ function summary(plan: Plan) { writeJson(path.join(directory, 'summary.json'), result); const markdown = markdownReport(result) + - `\nAttempts: ${result.attemptsUsed}/${result.maxAttempts}.\n\n` + + `\nAttempts: ${result.attemptsUsed}/${result.maxAttempts}. Limit per attempt: ${plan.attemptTimeoutMinutes ?? 40} minutes.\n\n` + result.attempts .map( (a) => @@ -174,7 +184,8 @@ async function main() { executionIdentity(platform, backend), digest(fs.readFileSync('test/inventory.json', 'utf8')), 'selected', - 3 + 3, + Number(process.env.E2E_ATTEMPT_TIMEOUT_MINUTES ?? '40') ); writeJson(planFile, plan); fs.writeFileSync( @@ -183,6 +194,7 @@ async function main() { ); } else { const plan: Plan = JSON.parse(fs.readFileSync(planFile, 'utf8')); + validatePlan(plan); assertExecutionIdentity(plan); if (command === 'run' || command === 'attempt') { const currentAttempt = command === 'attempt' ? Number(process.env.ATTEMPT) : 1; @@ -210,11 +222,15 @@ async function main() { const file = path.join(directory, `attempt-${attempt}.json`); writeJson(file, receipt); const args = ['--mochaOpts.grep', grep]; + console.info( + `Attempt ${attempt}/${plan.maxAttempts}: limit ${plan.attemptTimeoutMinutes ?? 40} minutes` + ); const child = await runProcess( runner, args, { ...process.env, ATTEMPT: String(attempt), E2E_PLAN: planFile }, - path.join(directory, `attempt-${attempt}-runner.log`) + path.join(directory, `attempt-${attempt}-runner.log`), + plan.attemptTimeoutMinutes ?? 40 ); receipt.finishedAt = new Date().toISOString(); receipt.exitCode = child.status ?? 1; diff --git a/test/reporting/report.ts b/test/reporting/report.ts index 2c43487f..64dd5486 100644 --- a/test/reporting/report.ts +++ b/test/reporting/report.ts @@ -20,6 +20,7 @@ export type Plan = { inventoryDigest: string; pack: string; maxAttempts: number; + attemptTimeoutMinutes?: number; cases: InventoryCase[]; expected: string[]; }; @@ -56,7 +57,8 @@ export function createPlan( identity: Identity, inventoryDigest: string, pack: string, - maxAttempts = 1 + maxAttempts = 1, + attemptTimeoutMinutes?: number ): Plan { if ( !['ios', 'android'].includes(identity.platform) || @@ -68,7 +70,11 @@ export function createPlan( !/^[a-f0-9]{64}$/.test(inventoryDigest) || !Number.isInteger(maxAttempts) || maxAttempts < 1 || - maxAttempts > 3 + maxAttempts > 3 || + (attemptTimeoutMinutes !== undefined && + (!Number.isInteger(attemptTimeoutMinutes) || + attemptTimeoutMinutes < 1 || + attemptTimeoutMinutes > 90)) ) { throw new Error('Invalid or incomplete run identity/attempt policy'); } @@ -92,6 +98,7 @@ export function createPlan( inventoryDigest, pack, maxAttempts, + ...(attemptTimeoutMinutes === undefined ? {} : { attemptTimeoutMinutes }), cases: selected, expected, }; @@ -104,7 +111,8 @@ export function validatePlan(plan: Plan): void { plan.identity, plan.inventoryDigest, plan.pack, - plan.maxAttempts + plan.maxAttempts, + plan.attemptTimeoutMinutes ); if ( rebuilt.id !== plan.id || diff --git a/test/unit/ci-retry.test.ts b/test/unit/ci-retry.test.ts index 4efeb895..4f27ee64 100644 --- a/test/unit/ci-retry.test.ts +++ b/test/unit/ci-retry.test.ts @@ -5,6 +5,7 @@ import path from 'node:path'; import { spawnSync } from 'node:child_process'; import { createPlan, + validatePlan, reconcileAttempts, type Identity, type WorkerReport, @@ -93,7 +94,7 @@ for (const [name, reports, attempts] of [ assert.equal(reconcileAttempts(plan, reports, attempts).status, 'incomplete') ); -for (const failures of [0, 1, 2, 3]) +for (const failures of [0, 1, 2, 3, 'timeout'] as const) test(`actual CI wrapper stops correctly after ${failures} initial failures`, (t) => { fs.mkdirSync('artifacts', { recursive: true }); const root = fs.mkdtempSync(path.resolve('artifacts/retry-fixture-')); @@ -120,12 +121,13 @@ for (const failures of [0, 1, 2, 3]) ); fs.writeFileSync( path.join(root, 'stub.cjs'), - `const fs=require('fs');const p=JSON.parse(fs.readFileSync(process.env.E2E_PLAN,'utf8'));const a=Number(process.env.ATTEMPT);const failed=a<=${failures};fs.mkdirSync('artifacts/reporting/workers',{recursive:true});fs.mkdirSync('artifacts/attempt-'+a,{recursive:true});fs.writeFileSync('artifacts/attempt-'+a+'/evidence.txt',failed?'first failure':'pass');fs.writeFileSync('artifacts/reporting/workers/attempt-'+a+'.json',JSON.stringify({schemaVersion:1,planId:p.id,attempt:a,cid:'0-0',startedAt:new Date().toISOString(),finishedAt:new Date().toISOString(),failures:failed?1:0,errors:[],events:p.expected.map(id=>({id,status:failed?'failed':'passed',durationMs:1}))}));process.exitCode=failed?1:0;` + `const fs=require('fs');const p=JSON.parse(fs.readFileSync(process.env.E2E_PLAN,'utf8'));const a=Number(process.env.ATTEMPT);const failed=a<=${typeof failures === 'number' ? failures : 0};fs.mkdirSync('artifacts/reporting/workers',{recursive:true});fs.mkdirSync('artifacts/attempt-'+a,{recursive:true});fs.writeFileSync('artifacts/attempt-'+a+'/evidence.txt',failed?'first failure':'pass');fs.writeFileSync('artifacts/reporting/workers/attempt-'+a+'.json',JSON.stringify({schemaVersion:1,planId:p.id,attempt:a,cid:'0-0',startedAt:new Date().toISOString(),finishedAt:new Date().toISOString(),failures:failed?1:0,errors:[],events:p.expected.map(id=>({id,status:failed?'failed':'passed',durationMs:1}))}));process.exitCode=failed?1:0;${failures === 'timeout' ? 'setInterval(()=>{},1000);' : ''}` ); const env = { ...process.env, PLATFORM: 'ios', BACKEND: 'local', + E2E_ATTEMPT_TIMEOUT_MINUTES: '90', E2E_APP_SHA: identity.appSha, E2E_RUN_ID: 'wrapper-fixture', E2E_BACKEND_REVISION: 'local', @@ -134,10 +136,21 @@ for (const failures of [0, 1, 2, 3]) GITHUB_ENV: path.join(root, 'env'), GITHUB_STEP_SUMMARY: path.join(root, 'summary.md'), }; + const clock = path.join(root, 'fast-clock.cjs'); + // Exercise the real timeout/kill/retry path without waiting 90 minutes. + fs.writeFileSync( + clock, + `const real=global.setTimeout;global.setTimeout=(fn,ms,...args)=>real(fn,ms===5400000?2000:ms,...args);` + ); const invoke = (cmd: string) => spawnSync( process.execPath, - ['--experimental-strip-types', path.resolve('scripts/ci-run.ts'), cmd], + [ + ...(failures === 'timeout' ? ['--require', clock] : []), + '--experimental-strip-types', + path.resolve('scripts/ci-run.ts'), + cmd, + ], { cwd: root, env, encoding: 'utf8' } ); const prep = invoke('prepare'); @@ -152,15 +165,28 @@ for (const failures of [0, 1, 2, 3]) .map((line) => line.split('=')) ) ); + const prepared = JSON.parse(fs.readFileSync(path.join(root, env.E2E_PLAN), 'utf8')); + assert.equal(prepared.attemptTimeoutMinutes, 90); + env.E2E_ATTEMPT_TIMEOUT_MINUTES = '1'; // Run uses the sealed plan, not a mutable environment. const run = invoke('run'); - assert.equal(run.status, failures === 3 ? 1 : 0, run.stderr); + assert.equal(run.status, failures === 3 || failures === 'timeout' ? 1 : 0, run.stderr); + if (failures === 'timeout') assert.match(run.stderr, /exceeded 90 minutes/); const s = JSON.parse( fs.readFileSync(path.join(root, 'artifacts/reporting/summary.json'), 'utf8') ); - assert.equal(s.attemptsUsed, Math.min(failures + 1, 3)); - assert.equal(s.firstFailure, failures > 0); + assert.equal(s.attemptsUsed, failures === 'timeout' ? 3 : Math.min(failures + 1, 3)); + assert.equal(s.firstFailure, failures === 'timeout' || failures > 0); assert.equal( fs.readFileSync(path.join(root, 'artifacts/attempt-1/evidence.txt'), 'utf8'), - failures ? 'first failure' : 'pass' + typeof failures === 'number' && failures ? 'first failure' : 'pass' ); }); + +test('attempt budget is bounded, sealed and backwards compatible', () => { + validatePlan(plan); // Existing plans retain their default 40-minute budget. + const p = createPlan(cases, identity, 'd'.repeat(64), 'selected', 3, 90); + validatePlan(p); + assert.throws(() => validatePlan({ ...p, attemptTimeoutMinutes: 40 }), /Run plan changed/); + for (const n of [0, -1, 91, Infinity, NaN, 1.5]) + assert.throws(() => createPlan(cases, identity, 'd'.repeat(64), 'selected', 3, n), /Invalid/); +}); From fce4fa10317fb595efeadd48a9af8dd8763545d0 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Tue, 3 Nov 2026 23:52:32 +0100 Subject: [PATCH 11/18] docs: record full staging execution budget --- docs/inventory-reporting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/inventory-reporting.md b/docs/inventory-reporting.md index 492e0a27..b98ba911 100644 --- a/docs/inventory-reporting.md +++ b/docs/inventory-reporting.md @@ -56,4 +56,4 @@ npx tsc --noEmit The CLI workflow tests use controlled runner fixtures; they are not evidence that hosted Appium journeys passed. Actual hosted measurements and device results must name their revisions and environments. -Expanded local selections (more than five cases per shard) allow 90 minutes per attempt, up to three attempts, within a 285-minute job and five-hour backend expiry. Base-sized local selections retain 40-minute attempts and a 135-minute job; staging/migration retain their existing limits. The iOS selector owns these budgets and the shared runner seals its attempt limit into the run plan. Limits are ceilings, not reservations: success stops retries and cleanup destroys the backend immediately. +Expanded local selections (more than five cases per shard) allow 90 minutes per attempt, up to three attempts, within a 285-minute job and five-hour backend expiry. Base-sized local selections retain 40-minute attempts and a 135-minute job; the six-case staging pack also uses 90-minute attempts within 285 minutes. Individual migration cases retain 40-minute attempts and 135-minute jobs. The iOS selector owns these budgets and the shared runner seals its attempt limit into the run plan. Limits are ceilings, not reservations: success stops retries and cleanup destroys the backend immediately. From e49cfab3c25b8bfe18c59ae471405d0b6d45bdfc Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Wed, 4 Nov 2026 01:04:22 +0100 Subject: [PATCH 12/18] ci: validate RN migration inputs and bound preparation logs --- .github/workflows/migration-wallet-setup.yml | 5 ++++ docs/inventory-reporting.md | 4 +++- test/specs/migration.e2e.ts | 24 +++++++++++++++++++- test/unit/pinned-fixture-source.test.cjs | 7 ++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/.github/workflows/migration-wallet-setup.yml b/.github/workflows/migration-wallet-setup.yml index 74fb45ec..2f6af7f3 100644 --- a/.github/workflows/migration-wallet-setup.yml +++ b/.github/workflows/migration-wallet-setup.yml @@ -28,6 +28,10 @@ on: description: Optional immutable E2E commit. Checkout must match; never resolve or fall back as a branch. type: string default: "" + diagnostics_retention_days: + description: Preparation-log retention in days; zero preserves the repository default. + type: number + default: 0 env: TERM: xterm-256color FORCE_COLOR: 1 @@ -104,6 +108,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: migration-setup-logs_${{ inputs.scenario_name }}_${{ github.run_attempt }} + retention-days: ${{ inputs.diagnostics_retention_days }} path: | bitkit-e2e-tests/artifacts/attempt-*/ bitkit-e2e-tests/artifacts/migration-source.json diff --git a/docs/inventory-reporting.md b/docs/inventory-reporting.md index b98ba911..f17d7dbf 100644 --- a/docs/inventory-reporting.md +++ b/docs/inventory-reporting.md @@ -40,7 +40,7 @@ A recovered failure is reported as a recovery, with the first failed/missing cas ## Cost controls and manual validation -Automatic runs use one local shard. Only manual dispatch can request two, partitioned by spec so a spec's fixture lifecycle stays together. Namespace supplies a separate Mac and backend instance per shard; every shard destroys its backend in cleanup. The local app is built once and reused. Staging and migration share one remote-regtest app build in the combined run. Builds have a 40-minute limit; execution limits depend on the selected pack as described below. Artifacts are retained for seven days and superseded runs cancel. +Automatic runs use one local shard. Only manual dispatch can request two, partitioned by spec so a spec's fixture lifecycle stays together. Namespace supplies a separate Mac and backend instance per shard; every shard destroys its backend in cleanup. The local app is built once and reused. Staging and migration share one remote-regtest app build in the combined run. Builds have a 40-minute limit; execution limits depend on the selected pack as described below. iOS build and diagnostic artifacts, including migration preparation logs, are retained for seven days; prepared wallet environments expire after one day. The shared preparation workflow preserves repository-default diagnostic retention for consumers that omit its optional override. Superseded runs cancel. Compare `selection=full, shards=1` and `selection=full, shards=2` on the same app and E2E revisions. Inspect elapsed/build/setup/test time, first-pass success and billed Mac plus backend usage. Two shards are an experiment, not an assumed cost saving or a new automatic default. @@ -57,3 +57,5 @@ npx tsc --noEmit The CLI workflow tests use controlled runner fixtures; they are not evidence that hosted Appium journeys passed. Actual hosted measurements and device results must name their revisions and environments. Expanded local selections (more than five cases per shard) allow 90 minutes per attempt, up to three attempts, within a 285-minute job and five-hour backend expiry. Base-sized local selections retain 40-minute attempts and a 135-minute job; the six-case staging pack also uses 90-minute attempts within 285 minutes. Individual migration cases retain 40-minute attempts and 135-minute jobs. The iOS selector owns these budgets and the shared runner seals its attempt limit into the run plan. Limits are ceilings, not reservations: success stops retries and cleanup destroys the backend immediately. + +RN passphrase migration verifies the exact restored passphrase and the funded RN source balance before installing the native target. Native verification dismisses the optional USDT introduction before asserting balances, history, tags and persistence after relaunch. A malformed fixture or blocked screen remains a failure; neither is reported as a successful migration. diff --git a/test/specs/migration.e2e.ts b/test/specs/migration.e2e.ts index cc67b5c9..9286bbcc 100644 --- a/test/specs/migration.e2e.ts +++ b/test/specs/migration.e2e.ts @@ -22,6 +22,7 @@ import { tap, typeText, waitForSetupWalletScreenFinish, + waitForWalletHome, } from '../helpers/actions'; import { expectMigrationBalances, @@ -356,6 +357,15 @@ async function setupLegacyWallet( await restoreRnWallet(IOS_RN_MNEMONIC); } + // Prove the source wallet is the funded fixture before installing the target. + // A dropped character in RN's passphrase field otherwise looks like lost + // funds after migration, even though the wrong wallet was restored first. + await browser.waitUntil(async () => (await getRnTotalBalance()) === IOS_RN_BALANCE, { + timeout: 180_000, + interval: 1_000, + timeoutMsg: 'Restored RN wallet balance does not match the prepared migration fixture', + }); + console.info('=== iOS: RN wallet restored ==='); return { mnemonic: IOS_RN_MNEMONIC, balance: IOS_RN_BALANCE }; } @@ -685,7 +695,17 @@ async function restoreRnWallet( // Passphrase if provided if (passphrase) { await tap('AdvancedButton'); - await typeText('PassphraseInput', passphrase); + if (driver.isIOS) { + // RN can drop characters from XCUITest's simulated typing. Paste the + // complete value, then verify it before deriving/restoring the wallet. + await pasteIOSText('PassphraseInput', passphrase); + const entered = await elementById('PassphraseInput').getAttribute('value'); + if (entered !== passphrase) { + throw new Error('RN passphrase input does not match the migration fixture'); + } + } else { + await typeText('PassphraseInput', passphrase); + } await confirmInputOnKeyboard(); } @@ -1106,6 +1126,8 @@ async function verifyMigration( ): Promise { console.info('=== Verifying migration ==='); + if (driver.isIOS) await waitForWalletHome({ timeout: 180_000 }); + await expectMigrationBalances({ total: expectedBalance, spending: 0, diff --git a/test/unit/pinned-fixture-source.test.cjs b/test/unit/pinned-fixture-source.test.cjs index 0b381e89..c526e70d 100644 --- a/test/unit/pinned-fixture-source.test.cjs +++ b/test/unit/pinned-fixture-source.test.cjs @@ -8,6 +8,13 @@ const test = require('node:test'), test('migration fixture bypasses branch resolution only for explicit commit pins', () => { const w = Y.parse(fs.readFileSync('.github/workflows/migration-wallet-setup.yml', 'utf8')); const steps = w.jobs['prepare-wallets'].steps; + assert.equal(w.on.workflow_call.inputs.diagnostics_retention_days.type, 'number'); + assert.equal(w.on.workflow_call.inputs.diagnostics_retention_days.default, 0); + assert.equal( + steps.find((s) => s.name === 'Upload preparation diagnostics').with['retention-days'], + '${{ inputs.diagnostics_retention_days }}' + ); + assert.equal(steps.find((s) => s.name === 'Upload migration env file').with['retention-days'], 1); assert.equal(steps.find((s) => s.id === 'e2e-ref').if, "inputs.e2e_sha == ''"); assert.equal( steps.find((s) => s.name === 'Clone E2E tests').with.ref, From 6f4492f146d464e08cba4699925a369ba629ea88 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Wed, 4 Nov 2026 09:09:38 +0100 Subject: [PATCH 13/18] ci: allow full local packs to finish and handle early QuickPay --- docs/inventory-reporting.md | 2 +- test/reporting/report.ts | 2 +- test/specs/transfer.e2e.ts | 17 +++++++++++++++-- test/unit/ci-retry.test.ts | 4 ++-- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/docs/inventory-reporting.md b/docs/inventory-reporting.md index f17d7dbf..3834b7ab 100644 --- a/docs/inventory-reporting.md +++ b/docs/inventory-reporting.md @@ -56,6 +56,6 @@ npx tsc --noEmit The CLI workflow tests use controlled runner fixtures; they are not evidence that hosted Appium journeys passed. Actual hosted measurements and device results must name their revisions and environments. -Expanded local selections (more than five cases per shard) allow 90 minutes per attempt, up to three attempts, within a 285-minute job and five-hour backend expiry. Base-sized local selections retain 40-minute attempts and a 135-minute job; the six-case staging pack also uses 90-minute attempts within 285 minutes. Individual migration cases retain 40-minute attempts and 135-minute jobs. The iOS selector owns these budgets and the shared runner seals its attempt limit into the run plan. Limits are ceilings, not reservations: success stops retries and cleanup destroys the backend immediately. +Local shards with 6–30 cases allow 90 minutes per attempt within a 285-minute job and five-hour backend expiry. Larger shards allow 110 minutes per attempt within a 345-minute job and six-hour backend expiry. Both sizes allow three attempts. Base-sized local selections retain 40-minute attempts and a 135-minute job; the six-case staging pack also uses 90-minute attempts within 285 minutes. Individual migration cases retain 40-minute attempts and 135-minute jobs. The iOS selector owns these budgets and the shared runner seals its attempt limit into the run plan. Limits are ceilings, not reservations: success stops retries and cleanup destroys the backend immediately. RN passphrase migration verifies the exact restored passphrase and the funded RN source balance before installing the native target. Native verification dismisses the optional USDT introduction before asserting balances, history, tags and persistence after relaunch. A malformed fixture or blocked screen remains a failure; neither is reported as a successful migration. diff --git a/test/reporting/report.ts b/test/reporting/report.ts index 64dd5486..20bc5cc8 100644 --- a/test/reporting/report.ts +++ b/test/reporting/report.ts @@ -74,7 +74,7 @@ export function createPlan( (attemptTimeoutMinutes !== undefined && (!Number.isInteger(attemptTimeoutMinutes) || attemptTimeoutMinutes < 1 || - attemptTimeoutMinutes > 90)) + attemptTimeoutMinutes > 110)) ) { throw new Error('Invalid or incomplete run identity/attempt policy'); } diff --git a/test/specs/transfer.e2e.ts b/test/specs/transfer.e2e.ts index 5f2e6293..f798d05e 100644 --- a/test/specs/transfer.e2e.ts +++ b/test/specs/transfer.e2e.ts @@ -475,7 +475,20 @@ describe('@transfer - Transfer', () => { await sleep(2500); // check activity - await elementById('ActivityShort-0').waitForDisplayed(); + let dismissedEarlyQuickPay = false; + if (driver.isIOS) { + // The first Lightning balance change can present QuickPay before the + // activity rows are accessible. It is shown once per wallet. + await browser.waitUntil( + async () => { + if (await tryDismissQuickPayIntroIfVisible()) dismissedEarlyQuickPay = true; + return isDisplayed('ActivityShort-0'); + }, + { timeout: 30_000, timeoutMsg: 'Channel-opening activity row did not appear' } + ); + } else { + await elementById('ActivityShort-0').waitForDisplayed(); + } await expectTextWithin('ActivityShort-0', 'Transfer'); await elementById('ActivityShort-1').waitForDisplayed(); await expectTextWithin('ActivityShort-1', 'Received'); @@ -488,7 +501,7 @@ describe('@transfer - Transfer', () => { await sleep(1000); await dismissBackgroundPaymentsTimedSheet({ triggerTimedSheet: true }); - await dismissQuickPayIntro({ triggerTimedSheet: true }); + if (!dismissedEarlyQuickPay) await dismissQuickPayIntro({ triggerTimedSheet: true }); await expectNoTextWithin('ActivitySpending', '0'); await waitForActiveChannel(lnd, ldkNodeId); diff --git a/test/unit/ci-retry.test.ts b/test/unit/ci-retry.test.ts index 4f27ee64..3883a518 100644 --- a/test/unit/ci-retry.test.ts +++ b/test/unit/ci-retry.test.ts @@ -184,9 +184,9 @@ for (const failures of [0, 1, 2, 3, 'timeout'] as const) test('attempt budget is bounded, sealed and backwards compatible', () => { validatePlan(plan); // Existing plans retain their default 40-minute budget. - const p = createPlan(cases, identity, 'd'.repeat(64), 'selected', 3, 90); + const p = createPlan(cases, identity, 'd'.repeat(64), 'selected', 3, 110); validatePlan(p); assert.throws(() => validatePlan({ ...p, attemptTimeoutMinutes: 40 }), /Run plan changed/); - for (const n of [0, -1, 91, Infinity, NaN, 1.5]) + for (const n of [0, -1, 111, Infinity, NaN, 1.5]) assert.throws(() => createPlan(cases, identity, 'd'.repeat(64), 'selected', 3, n), /Invalid/); }); From f91c7f4a3ebbd190b5e242712f0d8e888caa8126 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Wed, 4 Nov 2026 09:12:43 +0100 Subject: [PATCH 14/18] test: handle QuickPay throughout transfer verification --- test/specs/transfer.e2e.ts | 44 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/test/specs/transfer.e2e.ts b/test/specs/transfer.e2e.ts index f798d05e..c8341b5e 100644 --- a/test/specs/transfer.e2e.ts +++ b/test/specs/transfer.e2e.ts @@ -469,29 +469,29 @@ describe('@transfer - Transfer', () => { // check transfer card // await elementById('Suggestion-lightning_setting_up').waitForDisplayed(); - const totalBalance = await elementByIdWithin('TotalBalance-primary', 'MoneyText'); - const totalAmtAfterChannelOpen = await totalBalance.getText(); - await expect(totalBalance).not.toHaveText('100 000'); - await sleep(2500); - - // check activity - let dismissedEarlyQuickPay = false; - if (driver.isIOS) { - // The first Lightning balance change can present QuickPay before the - // activity rows are accessible. It is shown once per wallet. - await browser.waitUntil( - async () => { - if (await tryDismissQuickPayIntroIfVisible()) dismissedEarlyQuickPay = true; - return isDisplayed('ActivityShort-0'); - }, - { timeout: 30_000, timeoutMsg: 'Channel-opening activity row did not appear' } - ); - } else { + // QuickPay is shown once per wallet and can cover Home anywhere between + // channel opening and the last activity assertion. + let dismissedEarlyQuickPay = driver.isIOS && (await tryDismissQuickPayIntroIfVisible()); + const readOpeningActivity = async () => { + const totalBalance = await elementByIdWithin('TotalBalance-primary', 'MoneyText'); + await expect(totalBalance).not.toHaveText('100 000'); + const totalAmtAfterChannelOpen = await totalBalance.getText(); + await sleep(2500); await elementById('ActivityShort-0').waitForDisplayed(); - } - await expectTextWithin('ActivityShort-0', 'Transfer'); - await elementById('ActivityShort-1').waitForDisplayed(); - await expectTextWithin('ActivityShort-1', 'Received'); + await expectTextWithin('ActivityShort-0', 'Transfer'); + await elementById('ActivityShort-1').waitForDisplayed(); + await expectTextWithin('ActivityShort-1', 'Received'); + return { totalBalance, totalAmtAfterChannelOpen }; + }; + const { totalBalance, totalAmtAfterChannelOpen } = await readOpeningActivity().catch( + async (error) => { + if (!driver.isIOS || dismissedEarlyQuickPay || !(await tryDismissQuickPayIntroIfVisible())) { + throw error; + } + dismissedEarlyQuickPay = true; + return readOpeningActivity(); + } + ); await swipeFullScreen('down'); await swipeFullScreen('down'); From ca9e73187ad3c9c37d629da1c93d2318bc7d31a3 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Wed, 4 Nov 2026 10:14:00 +0100 Subject: [PATCH 15/18] docs: distinguish PR E2E from nightly and release migrations --- docs/inventory-reporting.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/inventory-reporting.md b/docs/inventory-reporting.md index 3834b7ab..00e2ed44 100644 --- a/docs/inventory-reporting.md +++ b/docs/inventory-reporting.md @@ -12,11 +12,11 @@ Every iOS app-code PR starts with these existing cases, unchanged: The iOS repo owns `.github/e2e-policy.json`, its selector and policy tests. Its tested app checkout supplies the policy; this repository supplies only the case inventory and execution/reporting. Change iOS paths and their policy together in the same PR. See [iOS policy documentation](https://github.com/synonymdev/bitkit-ios/blob/master/Docs/e2e-ci.md). -Feature directories and explicit service/model/helper files add only their relevant existing cases. For example, widget services add widget cases, Paykit services add staging, and backup services add recovery/migration. Only named shared wallet/lifecycle dependencies and build/orchestration configuration select every lane. There is no blanket Services/Models/Managers/Utilities rule. +Feature directories and explicit service/model/helper files add only their relevant existing cases. For example, widget services add widget cases, Paykit services add staging, and backup services add recovery coverage. Only named shared wallet/lifecycle dependencies and build/orchestration configuration select every PR-eligible local and staging case. There is no blanket Services/Models/Managers/Utilities rule. Cheap iOS validation checks all tracked paths, stale policy entries and referenced case IDs. Unknown paths fail with an actionable policy-update error before paid builds. Old and new rename paths contribute coverage from the base and tested policies, so a refactor cannot silently drop coverage. Documentation/native-test/policy-only changes skip device jobs. Selection records cases, reasons, app/E2E/policy identity and explicit gaps. USDT-specific paths retain the Bitcoin base gate and report the lack of existing USDT E2E coverage; unrelated Bitcoin tests do not establish USDT correctness. -PRs stacked on another branch retain the existing automatic exclusion. Manual dispatch with `pr` checks the PR's current merge against its actual base/head, including stacked PRs. The workflow source is the branch selected for dispatch. `selection=auto` uses the PR diff; `base` runs the five cases; `full` runs all applicable lanes. Automatic PRs cannot narrow selection with a manual input. +PRs stacked on another branch retain the existing automatic exclusion. Manual dispatch with `pr` checks the PR's current merge against its actual base/head, including stacked PRs. The workflow source is the branch selected for dispatch. `selection=auto` uses the PR diff; `base` runs the five cases; `full` runs all applicable local and staging cases. Migration runs nightly and after each push to an iOS `release-*` branch. An explicit branch dispatch with `selection=nightly` reproduces the nightly for validation. Automatic PRs cannot narrow selection with a manual input. ## Nightly From ee7a4299d82c9e6f66363fb6e8ef34569a817c22 Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Wed, 4 Nov 2026 10:25:26 +0100 Subject: [PATCH 16/18] test: verify the accessible legacy iOS wallet balance --- test/specs/migration.e2e.ts | 38 ++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/test/specs/migration.e2e.ts b/test/specs/migration.e2e.ts index 9286bbcc..0b63034f 100644 --- a/test/specs/migration.e2e.ts +++ b/test/specs/migration.e2e.ts @@ -360,11 +360,21 @@ async function setupLegacyWallet( // Prove the source wallet is the funded fixture before installing the target. // A dropped character in RN's passphrase field otherwise looks like lost // funds after migration, even though the wrong wallet was restored first. - await browser.waitUntil(async () => (await getRnTotalBalance()) === IOS_RN_BALANCE, { - timeout: 180_000, - interval: 1_000, - timeoutMsg: 'Restored RN wallet balance does not match the prepared migration fixture', - }); + // The iOS accessibility tree exposes the amount on the touchable parent. + // The nested MoneyText used by Android is hidden on RN iOS v1.1.6. + const rnBalance = elementById('TotalBalance'); + await rnBalance.waitForDisplayed(); + await browser.waitUntil( + async () => { + const label = await rnBalance.getAttribute('label'); + return label.replace(/\s/g, '') === String(IOS_RN_BALANCE); + }, + { + timeout: 180_000, + interval: 1_000, + timeoutMsg: 'Restored RN wallet balance does not match the prepared migration fixture', + } + ); console.info('=== iOS: RN wallet restored ==='); return { mnemonic: IOS_RN_MNEMONIC, balance: IOS_RN_BALANCE }; @@ -675,8 +685,22 @@ async function restoreRnWallet( // Terms of service await elementById('Continue').waitForDisplayed(); - await tap('Check1'); - await tap('Check2'); + if (driver.isIOS) { + const next = elementById('Continue'); + // Legacy RN labels Continue as enabled only when both boxes are selected. + // Alternating toggles visits all four combinations if an initial tap was missed. + for (const toggle of ['Check1', 'Check2', 'Check1', 'Check2', 'Check1', 'Check2']) { + if ((await next.getAttribute('label')) === 'enabled') break; + await tap(toggle); + } + await browser.waitUntil(async () => (await next.getAttribute('label')) === 'enabled', { + timeout: 5_000, + timeoutMsg: 'Both legacy RN consent boxes must be selected', + }); + } else { + await tap('Check1'); + await tap('Check2'); + } await tap('Continue'); // Skip intro and go to restore From b094f31ee83708750e52d859cec458c76c847a8b Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Wed, 4 Nov 2026 12:47:02 +0100 Subject: [PATCH 17/18] docs: describe independent iOS migration runs --- docs/inventory-reporting.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/inventory-reporting.md b/docs/inventory-reporting.md index 00e2ed44..b029750f 100644 --- a/docs/inventory-reporting.md +++ b/docs/inventory-reporting.md @@ -2,7 +2,7 @@ Issue: https://github.com/synonymdev/bitkit-e2e-tests/issues/264 -The iOS `e2e-tests.yml` entry point owns PR selection, manual validation and the nightly schedule. Existing staging and migration workflows are reusable execution lanes. There is no shadow gate. Android workflow triggers, test selection, backend assignments and legacy retry behavior remain unchanged. +The iOS `e2e-tests.yml` entry point owns PR selection, manual validation and the nightly schedule. Staging is reusable. The independent `e2e_migration.yml` workflow owns its schedule, release pushes, app build and result. There is no shadow gate. Android workflow triggers, test selection, backend assignments and legacy retry behavior remain unchanged. ## PR selection @@ -16,17 +16,17 @@ Feature directories and explicit service/model/helper files add only their relev Cheap iOS validation checks all tracked paths, stale policy entries and referenced case IDs. Unknown paths fail with an actionable policy-update error before paid builds. Old and new rename paths contribute coverage from the base and tested policies, so a refactor cannot silently drop coverage. Documentation/native-test/policy-only changes skip device jobs. Selection records cases, reasons, app/E2E/policy identity and explicit gaps. USDT-specific paths retain the Bitcoin base gate and report the lack of existing USDT E2E coverage; unrelated Bitcoin tests do not establish USDT correctness. -PRs stacked on another branch retain the existing automatic exclusion. Manual dispatch with `pr` checks the PR's current merge against its actual base/head, including stacked PRs. The workflow source is the branch selected for dispatch. `selection=auto` uses the PR diff; `base` runs the five cases; `full` runs all applicable local and staging cases. Migration runs nightly and after each push to an iOS `release-*` branch. An explicit branch dispatch with `selection=nightly` reproduces the nightly for validation. Automatic PRs cannot narrow selection with a manual input. +PRs stacked on another branch retain the existing automatic exclusion. Manual dispatch with `pr` checks the PR's current merge against its actual base/head, including stacked PRs. The workflow source is the branch selected for dispatch. `selection=auto` uses the PR diff; `base` runs the five cases; `full` runs all applicable local and staging cases. Migration runs nightly and after each push to an iOS `release-*` branch. Manual E2E dispatches use local and staging cases. Automatic PRs cannot narrow selection with a manual input. ## Nightly -The main iOS workflow runs nightly at 03:00 UTC, without a changed-revision shortcut: +Two separate iOS workflows run each night at 03:00 UTC: | Lane | Active existing cases | | --- | --- | | Local | 42: all core UI, local money, on-chain, multi-address and active hardware cases | | Staging | 6: `pubky_profile_1` through `pubky_profile_5`, `paykit_1` | -| Migration | 6: native restore/upgrade, RN restore/upgrade, passphrase and legacy-address migration | +| Independent migration workflow | 6: native restore/upgrade, RN restore/upgrade, passphrase and legacy-address migration | The 54 cases use the appropriate local or remote-regtest build and fixtures. Migration setup entries are fixtures, not extra regression passes. Five existing skipped cases and six unsupported iOS mainnet cases remain explicit gaps; the interactive Lightning payment utility remains manual. This does not invent missing USDT, allowance or subscription automation. @@ -36,11 +36,11 @@ The 54 cases use the appropriate local or remote-regtest build and fixtures. Mig A run permits up to three attempts and stops after the first runner exit-zero with exact complete case coverage. Each attempt retains worker events, process outcome and timing. The local/staging wrapper owns retries; migration keeps its existing per-attempt prepared-wallet setup and invokes the same wrapper one attempt at a time. An inventory-enabled attempt reruns the complete selected pack, avoiding dependence on the machine's legacy `/tmp/lock`. Android's non-inventory execution retains its existing retry locks. -A recovered failure is reported as a recovery, with the first failed/missing case results and earlier errors retained. Missing reports, setup/teardown errors, incomplete workers, unexpected skips, stale identities or a nonzero final runner cannot produce a pass. The parent `e2e-status` requires every selected local/staging/migration job and build to succeed. +A recovered failure is reported as a recovery, with the first failed/missing case results and earlier errors retained. Missing reports, setup/teardown errors, incomplete workers, unexpected skips, stale identities or a nonzero final runner cannot produce a pass. The parent `e2e-status` requires every selected local or staging job and build to succeed. The migration workflow reports its own result. ## Cost controls and manual validation -Automatic runs use one local shard. Only manual dispatch can request two, partitioned by spec so a spec's fixture lifecycle stays together. Namespace supplies a separate Mac and backend instance per shard; every shard destroys its backend in cleanup. The local app is built once and reused. Staging and migration share one remote-regtest app build in the combined run. Builds have a 40-minute limit; execution limits depend on the selected pack as described below. iOS build and diagnostic artifacts, including migration preparation logs, are retained for seven days; prepared wallet environments expire after one day. The shared preparation workflow preserves repository-default diagnostic retention for consumers that omit its optional override. Superseded runs cancel. +Automatic runs use one local shard. Only manual dispatch can request two, partitioned by spec so a spec's fixture lifecycle stays together. Namespace supplies a separate Mac and backend instance per shard; every shard destroys its backend in cleanup. The local app is built once and reused. Staging and the independent migration workflow each use their own network build. Builds have a 40-minute limit; execution limits depend on the selected pack as described below. iOS build and diagnostic artifacts, including migration preparation logs, are retained for seven days; prepared wallet environments expire after one day. The shared preparation workflow preserves repository-default diagnostic retention for consumers that omit its optional override. Superseded runs cancel. Compare `selection=full, shards=1` and `selection=full, shards=2` on the same app and E2E revisions. Inspect elapsed/build/setup/test time, first-pass success and billed Mac plus backend usage. Two shards are an experiment, not an assumed cost saving or a new automatic default. From b691df2e5d601868566b0456f60494f5d24ab77f Mon Sep 17 00:00:00 2001 From: Piotr Stachyra Date: Wed, 4 Nov 2026 16:09:08 +0100 Subject: [PATCH 18/18] ci: restore visible failed-only iOS retries --- AGENTS.md | 4 +- docs/inventory-reporting.md | 4 +- scripts/ci-run.ts | 86 +++++++++++------------ test/helpers/suite.ts | 15 ++-- test/reporting/report.ts | 43 +++++++----- test/unit/ci-retry.test.ts | 134 ++++++++++++++++++++++++++++++++---- test/unit/reporting.test.ts | 22 +++++- 7 files changed, 221 insertions(+), 87 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d47c0745..3d8ea509 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -171,8 +171,8 @@ Implication for feature work: ## iOS CI selection -The iOS consumer owns its path policy in `bitkit-ios/.github/e2e-policy.json` and selector in `.github/scripts/e2e-selection.cjs`. It reads this repo's `test/inventory.json`; iOS paths and selection policy do not belong in this shared repo. Historical `@ios_gate` and `@ios_nightly` tags do not control that consumer. Its base cases are `send_1`, `send_2`, `lightning_1`, `transfer_2`, `boost_2`; automatic risk selection expands coverage. Nightly runs all applicable local/staging/migration cases. See [inventory and reporting](docs/inventory-reporting.md). +The iOS consumer owns its path policy in `bitkit-ios/.github/e2e-policy.json` and selector in `.github/scripts/e2e-selection.cjs`. It reads this repo's `test/inventory.json`; iOS paths and selection policy do not belong in this shared repo. Historical `@ios_gate` and `@ios_nightly` tags do not control that consumer. Its base cases are `send_1`, `send_2`, `lightning_1`, `transfer_2`, `boost_2`; automatic risk selection expands coverage. Nightly E2E runs all applicable local/staging cases; the independent migration workflow runs all six migrations nightly and on release branches. See [inventory and reporting](docs/inventory-reporting.md). Keep capability tags for Android's existing selectors and manual execution. Android workflows and selections are unchanged. Do not rename tags merely to match the iOS policy. -Inventory-enabled iOS execution uses `scripts/ci-run.ts`: up to three attempts, complete selected pack per attempt, stop after exact successful coverage, first-failure evidence retained. This bypasses legacy `/tmp/lock`; Android non-inventory retries retain their existing behavior. Plans and receipts must identify the app, test source, artifact, backend and run. Never reuse a previous night's pass as current execution. +Inventory-enabled iOS execution uses `scripts/ci-run.ts attempt` from three visible workflow steps. `ciIt()` skips cases that passed earlier in the same run, so only failed or uncompleted cases repeat. Completion locks are scoped to the run directory and rebuilt from verified worker results before each attempt. Reports combine current-run passes and retain first-failure evidence. Android non-inventory retries retain their existing `/tmp/lock` behavior. Plans and receipts must identify the app, test source, artifact, backend and run. Never reuse a previous night's pass as current execution. diff --git a/docs/inventory-reporting.md b/docs/inventory-reporting.md index b029750f..b87357ae 100644 --- a/docs/inventory-reporting.md +++ b/docs/inventory-reporting.md @@ -34,7 +34,9 @@ The 54 cases use the appropriate local or remote-regtest build and fixtures. Mig `scripts/ci-run.ts` prepares a plan tied to the actual E2E checkout, selected app SHA, downloaded artifact digest, platform, backend, variant and run identity. Selection rejects unknown, skipped or wrong-backend cases. Only selected spec files load. -A run permits up to three attempts and stops after the first runner exit-zero with exact complete case coverage. Each attempt retains worker events, process outcome and timing. The local/staging wrapper owns retries; migration keeps its existing per-attempt prepared-wallet setup and invokes the same wrapper one attempt at a time. An inventory-enabled attempt reruns the complete selected pack, avoiding dependence on the machine's legacy `/tmp/lock`. Android's non-inventory execution retains its existing retry locks. +Local and staging jobs expose `Run E2E Tests 1`, `2`, and `3` as separate workflow steps. Each invokes `scripts/ci-run.ts attempt` once; later steps run only after failure. The established `ciIt()` pass locks skip cases already passed, so only failed or uncompleted cases repeat. Inventory-enabled locks live beside the current run plan and are rebuilt from verified worker results before each attempt; stale machine locks and passes from failed hooks or interrupted workers cannot skip work. Android keeps its existing `/tmp/lock` behavior. Migration remains independent and keeps its per-attempt prepared-wallet setup. + +The final report combines selected cases passed across these attempts. It requires the last runner to exit successfully and every selected case to have a verified current-run pass. All three failed attempts produce a failed job despite the attempts using `continue-on-error`. Each attempt retains worker events, process outcome and timing. A recovered failure is reported as a recovery, with the first failed/missing case results and earlier errors retained. Missing reports, setup/teardown errors, incomplete workers, unexpected skips, stale identities or a nonzero final runner cannot produce a pass. The parent `e2e-status` requires every selected local or staging job and build to succeed. The migration workflow reports its own result. diff --git a/scripts/ci-run.ts b/scripts/ci-run.ts index 3e7277ae..0cb98ac7 100644 --- a/scripts/ci-run.ts +++ b/scripts/ci-run.ts @@ -9,6 +9,7 @@ import { type Backend, } from '../test/reporting/inventory.ts'; import { executionIdentity, assertExecutionIdentity } from '../test/reporting/identity.ts'; +import { lockFile } from '../test/helpers/suite.ts'; import { createPlan, validatePlan, @@ -196,59 +197,58 @@ async function main() { const plan: Plan = JSON.parse(fs.readFileSync(planFile, 'utf8')); validatePlan(plan); assertExecutionIdentity(plan); - if (command === 'run' || command === 'attempt') { - const currentAttempt = command === 'attempt' ? Number(process.env.ATTEMPT) : 1; - if ( - !Number.isInteger(currentAttempt) || - currentAttempt < 1 || - currentAttempt > plan.maxAttempts - ) + if (command === 'attempt') { + const attempt = Number(process.env.ATTEMPT); + if (!Number.isInteger(attempt) || attempt < 1 || attempt > plan.maxAttempts) throw Error('Invalid attempt'); - if (receipts().length !== currentAttempt - 1) + const previousReceipts = receipts(); + const previousWorkers = workers(); + if (previousReceipts.length !== attempt - 1) throw Error('Unexpected attempt receipts; refusing to reuse another execution'); if ( - currentAttempt > 1 && - reconcileAttempts(plan, workers(), receipts()).status === 'passed' + attempt > 1 && + reconcileAttempts(plan, previousWorkers, previousReceipts).status === 'passed' ) throw Error('Previous attempt already passed'); + + // Keep ciIt's established pass-lock behavior, isolated to this execution. + // Rebuild locks from verified results so an interrupted body/failed hook or + // a previous run can never supply a pass to the next attempt. + const completed = path.join(directory, 'completed'); + fs.rmSync(completed, { recursive: true, force: true }); + fs.mkdirSync(completed, { recursive: true }); + for (const result of reconcile(plan, previousWorkers).results) { + if (result.status === 'passed') + fs.writeFileSync(lockFile(`@${result.id}`, planFile), '1'); + } const grep = fs.readFileSync(path.join(directory, 'grep.txt'), 'utf8').trim(); const runner = plan.identity.platform === 'ios' ? './ci_run_ios.sh' : './ci_run_android.sh'; - for ( - let attempt = currentAttempt; - attempt <= (command === 'attempt' ? currentAttempt : plan.maxAttempts); - attempt++ - ) { - const receipt: AttemptReceipt = { attempt, startedAt: new Date().toISOString() }; - const file = path.join(directory, `attempt-${attempt}.json`); - writeJson(file, receipt); - const args = ['--mochaOpts.grep', grep]; - console.info( - `Attempt ${attempt}/${plan.maxAttempts}: limit ${plan.attemptTimeoutMinutes ?? 40} minutes` - ); - const child = await runProcess( - runner, - args, - { ...process.env, ATTEMPT: String(attempt), E2E_PLAN: planFile }, - path.join(directory, `attempt-${attempt}-runner.log`), - plan.attemptTimeoutMinutes ?? 40 - ); - receipt.finishedAt = new Date().toISOString(); - receipt.exitCode = child.status ?? 1; - writeJson(file, receipt); - const result = reconcile( - plan, - workers().filter((w) => w.attempt === attempt) - ); - writeJson(path.join(directory, `attempt-${attempt}-summary.json`), { - ...result, - runnerExit: receipt.exitCode, - }); - if (child.cancelled || (receipt.exitCode === 0 && result.status === 'passed')) break; + const receipt: AttemptReceipt = { attempt, startedAt: new Date().toISOString() }; + const file = path.join(directory, `attempt-${attempt}.json`); + writeJson(file, receipt); + console.info( + `Attempt ${attempt}/${plan.maxAttempts}: limit ${plan.attemptTimeoutMinutes ?? 40} minutes` + ); + const child = await runProcess( + runner, + ['--mochaOpts.grep', grep], + { ...process.env, ATTEMPT: String(attempt), E2E_PLAN: planFile }, + path.join(directory, `attempt-${attempt}-runner.log`), + plan.attemptTimeoutMinutes ?? 40 + ); + receipt.finishedAt = new Date().toISOString(); + receipt.exitCode = child.status; + writeJson(file, receipt); + const result = reconcile(plan, workers()); + writeJson(path.join(directory, `attempt-${attempt}-summary.json`), { + ...result, + runnerExit: receipt.exitCode, + }); + if (receipt.exitCode !== 0 || result.status !== 'passed') console.error( `Attempt ${attempt} failed; evidence retained in ${directory} and artifacts/attempt-${attempt}` ); - } - } else if (command !== 'report') throw Error('Usage: ci-run.ts prepare|run|attempt|report'); + } else if (command !== 'report') throw Error('Usage: ci-run.ts prepare|attempt|report'); if (summary(plan).status !== 'passed') process.exitCode = 1; } } catch (error) { diff --git a/test/helpers/suite.ts b/test/helpers/suite.ts index 94a99921..bd585a54 100644 --- a/test/helpers/suite.ts +++ b/test/helpers/suite.ts @@ -42,19 +42,22 @@ ciIt.only = (name: string, fn?: () => Promise) => { }; export function checkComplete(name: string): boolean { - // Inventory runs use run-bound receipts, never shared or previous-night locks. - if (process.env.E2E_PLAN) return false; return ON_CI && fs.existsSync(lockFile(name)); } export function markComplete(name: string) { - if (process.env.E2E_PLAN) return; if (!ON_CI) return; - fs.mkdirSync(LOCK_PATH, { recursive: true }); - fs.writeFileSync(lockFile(name), '1'); + const file = lockFile(name); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, '1'); } -function lockFile(key: string) { +export function lockFile(key: string, planFile = process.env.E2E_PLAN) { + if (planFile) { + const id = /^@([a-z][a-z0-9_]*)(?=[\s,]|$)/.exec(key)?.[1]; + if (!id) throw new Error(`Missing case ID for retry lock: ${key}`); + return path.join(path.dirname(planFile), 'completed', `lock-${id}`); + } // replace anything that's *not* a letter, number, dot, underscore, or dash // with an underscore. This avoids weird characters in filenames. const safe = key.replace(/[^a-zA-Z0-9._-]/g, '_'); diff --git a/test/reporting/report.ts b/test/reporting/report.ts index 20bc5cc8..72a2a63e 100644 --- a/test/reporting/report.ts +++ b/test/reporting/report.ts @@ -126,7 +126,7 @@ export function reconcile(plan: Plan, reports: WorkerReport[]) { validatePlan(plan); const problems: string[] = []; const expected = new Set(plan.expected); - const observations = new Map(); + const observations = new Map(); const workers = new Set(); const executions = new Set(); let firstFailure = false; @@ -152,7 +152,8 @@ export function reconcile(plan: Plan, reports: WorkerReport[]) { const worker = `${report.attempt}/${report.cid}`; if (workers.has(worker)) problems.push(`Duplicate worker report: ${worker}`); workers.add(worker); - if (!report.finishedAt) problems.push(`Interrupted worker: ${worker}`); + if (!report.finishedAt && report.attempt === finalAttempt) + problems.push(`Interrupted worker: ${worker}`); firstFailure ||= report.failures > 0 || report.errors.length > 0; if (report.attempt === finalAttempt && (report.failures || report.errors.length)) { problems.push(`Final attempt failed: ${worker} (${report.errors.join('; ')})`); @@ -171,7 +172,14 @@ export function reconcile(plan: Plan, reports: WorkerReport[]) { if (executions.has(execution)) problems.push(`Duplicate case execution: ${execution}`); executions.add(execution); const history = observations.get(event.id) ?? []; - history.push({ attempt: report.attempt, event }); + history.push({ + attempt: report.attempt, + event, + verified: + !!report.finishedAt && + report.errors.length === 0 && + report.failures <= report.events.filter((e) => e.status === 'failed').length, + }); observations.set(event.id, history); } } @@ -179,7 +187,7 @@ export function reconcile(plan: Plan, reports: WorkerReport[]) { const history = (observations.get(id) ?? []).sort((a, b) => a.attempt - b.attempt); let status = 'missing'; let previouslyPassed = false; - for (const { attempt, event } of history) { + for (const { attempt, event, verified } of history) { firstFailure ||= event.status === 'failed'; if (event.status === 'skipped') { if (!previouslyPassed) { @@ -187,8 +195,8 @@ export function reconcile(plan: Plan, reports: WorkerReport[]) { status = 'unexpected-skip'; } } else { - status = event.status; - previouslyPassed = event.status === 'passed'; + status = event.status === 'passed' && !verified ? 'unverified' : event.status; + previouslyPassed = status === 'passed'; } } if (status !== 'passed') problems.push(`${id}: ${status}`); @@ -196,7 +204,7 @@ export function reconcile(plan: Plan, reports: WorkerReport[]) { id, status, firstAttempt: history.find((h) => h.attempt === 1)?.event.status ?? 'missing', - history, + history: history.map(({ attempt, event }) => ({ attempt, event })), }; }); const dispositions = plan.cases.filter( @@ -262,9 +270,13 @@ export function reconcileAttempts(plan: Plan, reports: WorkerReport[], attempts: problems.push('Invalid, missing or out-of-order attempt receipt'); const coverage = reconcile( plan, - reports.filter((r) => r.attempt === attempt.attempt) + reports.filter((r) => r.attempt <= attempt.attempt) ); - const passed = attempt.exitCode === 0 && !!attempt.finishedAt && coverage.status === 'passed'; + const passed = + attempt.exitCode === 0 && + !!attempt.finishedAt && + reports.some((r) => r.attempt === attempt.attempt) && + coverage.status === 'passed'; return { ...attempt, passed, problems: coverage.problems, coverage }; }); if (!perAttempt.length) problems.push('Runner never started'); @@ -274,7 +286,7 @@ export function reconcileAttempts(plan: Plan, reports: WorkerReport[], attempts: problems.push('Worker report without matching attempt receipt'); const last = perAttempt.at(-1); if (!last?.passed) - problems.push('Final attempt did not complete successfully with exact coverage'); + problems.push('Final attempt did not complete successfully with all selected cases passed'); const fallback = reconcile(plan, []); const result = last?.coverage ?? fallback; return { @@ -285,14 +297,7 @@ export function reconcileAttempts(plan: Plan, reports: WorkerReport[], attempts: maxAttempts: plan.maxAttempts, attempts: perAttempt.map(({ coverage, ...receipt }) => receipt), problems: [...problems, ...(last?.passed ? [] : result.problems)], - results: result.results.map((r) => ({ - ...r, - firstAttempt: - perAttempt[0]?.coverage.results.find((first) => first.id === r.id)?.status ?? 'missing', - history: perAttempt.flatMap( - (a) => a.coverage.results.find((c) => c.id === r.id)?.history ?? [] - ), - })), - durationMs: perAttempt.reduce((sum, a) => sum + a.coverage.durationMs, 0), + results: result.results, + durationMs: result.durationMs, }; } diff --git a/test/unit/ci-retry.test.ts b/test/unit/ci-retry.test.ts index 3883a518..150365f3 100644 --- a/test/unit/ci-retry.test.ts +++ b/test/unit/ci-retry.test.ts @@ -94,8 +94,48 @@ for (const [name, reports, attempts] of [ assert.equal(reconcileAttempts(plan, reports, attempts).status, 'incomplete') ); -for (const failures of [0, 1, 2, 3, 'timeout'] as const) - test(`actual CI wrapper stops correctly after ${failures} initial failures`, (t) => { +test('failed-only retries accumulate passes and preserve the original failure', () => { + const first = worker(1, true); + first.events[0].status = 'passed'; + const second = worker(2, true); + second.events[0].status = 'skipped'; + const third = worker(3); + third.events[0].status = 'skipped'; + const s = reconcileAttempts( + plan, + [first, second, third], + [receipt(1, 1), receipt(2, 1), receipt(3)] + ); + assert.equal(s.status, 'passed'); + assert.equal(s.firstFailure, true); + assert.deepEqual( + s.results[0].history.map((h) => h.event.status), + ['passed', 'skipped', 'skipped'] + ); + assert.deepEqual( + s.results[1].history.map((h) => h.event.status), + ['failed', 'failed', 'passed'] + ); + assert.equal(s.durationMs, 60); +}); + +test('passes from a failed hook or interrupted worker cannot justify a skipped retry', () => { + for (const first of [ + { ...worker(1), errors: ['Hook failed: afterEach'] }, + { ...worker(1), finishedAt: undefined }, + { ...worker(1), failures: 1 }, + ]) { + const retry = worker(2); + retry.events[0].status = 'skipped'; + assert.equal( + reconcileAttempts(plan, [first, retry], [receipt(1, 1), receipt(2)]).status, + 'incomplete' + ); + } +}); + +for (const failures of [0, 1, 2, 3, 'timeout', 'startup', 'hook'] as const) + test(`visible CI attempts retry only failures: ${failures}`, (t) => { fs.mkdirSync('artifacts', { recursive: true }); const root = fs.mkdtempSync(path.resolve('artifacts/retry-fixture-')); t.after(() => fs.rmSync(root, { recursive: true, force: true })); @@ -116,15 +156,50 @@ for (const failures of [0, 1, 2, 3, 'timeout'] as const) ); fs.writeFileSync( path.join(root, 'ci_run_ios.sh'), - '#!/usr/bin/env bash\nexec node stub.cjs\n', + '#!/usr/bin/env bash\nexec node --experimental-strip-types stub.cjs\n', { mode: 0o755 } ); fs.writeFileSync( path.join(root, 'stub.cjs'), - `const fs=require('fs');const p=JSON.parse(fs.readFileSync(process.env.E2E_PLAN,'utf8'));const a=Number(process.env.ATTEMPT);const failed=a<=${typeof failures === 'number' ? failures : 0};fs.mkdirSync('artifacts/reporting/workers',{recursive:true});fs.mkdirSync('artifacts/attempt-'+a,{recursive:true});fs.writeFileSync('artifacts/attempt-'+a+'/evidence.txt',failed?'first failure':'pass');fs.writeFileSync('artifacts/reporting/workers/attempt-'+a+'.json',JSON.stringify({schemaVersion:1,planId:p.id,attempt:a,cid:'0-0',startedAt:new Date().toISOString(),finishedAt:new Date().toISOString(),failures:failed?1:0,errors:[],events:p.expected.map(id=>({id,status:failed?'failed':'passed',durationMs:1}))}));process.exitCode=failed?1:0;${failures === 'timeout' ? 'setInterval(()=>{},1000);' : ''}` + String.raw` +const fs = require('fs'); +const Mocha = require(${JSON.stringify(path.resolve('node_modules/mocha'))}); +const {ciIt} = require(${JSON.stringify(path.resolve('test/helpers/suite.ts'))}); +const p = JSON.parse(fs.readFileSync(process.env.E2E_PLAN, 'utf8')); +const a = Number(process.env.ATTEMPT); +const mode = ${JSON.stringify(failures)}; +fs.mkdirSync('artifacts/attempt-' + a, {recursive: true}); +fs.writeFileSync('artifacts/attempt-' + a + '/evidence.txt', 'attempt ' + a); +if (mode === 'startup' && a === 1) process.exit(1); +const mocha = new Mocha({reporter: 'dot'}); +mocha.suite.emit('pre-require', global, 'fixture', mocha); +for (const id of p.expected) { + ciIt('@' + id + ' - fixture', async () => { + fs.appendFileSync('bodies.jsonl', JSON.stringify({attempt: a, id}) + '\n'); + if (id === 'send_2' && typeof mode === 'number' && a <= mode) throw Error('controlled case failure'); + }); +} +if (mode === 'hook') afterEach(() => {if (a === 1) throw Error('controlled hook failure')}); +const report = {schemaVersion: 1, planId: p.id, attempt: a, cid: '0-0', startedAt: new Date().toISOString(), failures: 0, errors: [], events: []}; +const runner = mocha.run(count => { + report.finishedAt = new Date().toISOString(); + report.failures = count; + fs.mkdirSync('artifacts/reporting/workers', {recursive: true}); + fs.writeFileSync('artifacts/reporting/workers/attempt-' + a + '.json', JSON.stringify(report)); + process.exitCode = count ? 1 : 0; + if (mode === 'timeout') setInterval(() => {}, 1000); +}); +for (const [event, status] of [['pass', 'passed'], ['fail', 'failed'], ['pending', 'skipped']]) { + runner.on(event, item => { + if (item.type === 'hook') {report.errors.push('Hook failed: ' + item.title); return} + report.events.push({id: item.title.match(/^@(\w+)/)[1], status, durationMs: 1}); + }); +} +` ); const env = { ...process.env, + CI: '1', PLATFORM: 'ios', BACKEND: 'local', E2E_ATTEMPT_TIMEOUT_MINUTES: '90', @@ -135,12 +210,12 @@ for (const failures of [0, 1, 2, 3, 'timeout'] as const) E2E_PLAN: 'artifacts/reporting/plan.json', GITHUB_ENV: path.join(root, 'env'), GITHUB_STEP_SUMMARY: path.join(root, 'summary.md'), + ATTEMPT: '1', }; const clock = path.join(root, 'fast-clock.cjs'); - // Exercise the real timeout/kill/retry path without waiting 90 minutes. fs.writeFileSync( clock, - `const real=global.setTimeout;global.setTimeout=(fn,ms,...args)=>real(fn,ms===5400000?2000:ms,...args);` + `const real=global.setTimeout;global.setTimeout=(fn,ms,...args)=>real(fn,ms===5400000?1000:ms,...args);` ); const invoke = (cmd: string) => spawnSync( @@ -167,19 +242,52 @@ for (const failures of [0, 1, 2, 3, 'timeout'] as const) ); const prepared = JSON.parse(fs.readFileSync(path.join(root, env.E2E_PLAN), 'utf8')); assert.equal(prepared.attemptTimeoutMinutes, 90); - env.E2E_ATTEMPT_TIMEOUT_MINUTES = '1'; // Run uses the sealed plan, not a mutable environment. - const run = invoke('run'); - assert.equal(run.status, failures === 3 || failures === 'timeout' ? 1 : 0, run.stderr); - if (failures === 'timeout') assert.match(run.stderr, /exceeded 90 minutes/); + env.E2E_ATTEMPT_TIMEOUT_MINUTES = '1'; // Attempt uses the sealed plan. + // A leftover lock must not suppress current execution, even under this plan path. + fs.mkdirSync(path.join(root, 'artifacts/reporting/completed'), { recursive: true }); + fs.writeFileSync(path.join(root, 'artifacts/reporting/completed/lock-send_1'), '1'); + let last; + for (let attempt = 1; attempt <= 3; attempt++) { + env.ATTEMPT = String(attempt); + last = invoke('attempt'); + if (last.status === 0) break; // The workflow's visible-step condition. + } + assert.equal(last?.status, failures === 3 || failures === 'timeout' ? 1 : 0, last?.stderr); + if (failures === 'timeout') assert.match(last!.stderr, /exceeded 90 minutes/); + const final = invoke('report'); + assert.equal(final.status, last!.status, final.stderr); const s = JSON.parse( fs.readFileSync(path.join(root, 'artifacts/reporting/summary.json'), 'utf8') ); - assert.equal(s.attemptsUsed, failures === 'timeout' ? 3 : Math.min(failures + 1, 3)); - assert.equal(s.firstFailure, failures === 'timeout' || failures > 0); + const attempts = + typeof failures === 'number' ? Math.min(failures + 1, 3) : failures === 'timeout' ? 3 : 2; + assert.equal(s.attemptsUsed, attempts); + assert.equal(s.firstFailure, failures !== 0); + const bodies = fs + .readFileSync(path.join(root, 'bodies.jsonl'), 'utf8') + .trim() + .split('\n') + .map((line) => JSON.parse(line)); + assert.equal(bodies.filter((b) => b.id === 'send_1').length, failures === 'hook' ? 2 : 1); + assert.equal( + bodies.filter((b) => b.id === 'send_2').length, + failures === 'timeout' || failures === 'startup' || failures === 'hook' ? 1 : attempts + ); assert.equal( fs.readFileSync(path.join(root, 'artifacts/attempt-1/evidence.txt'), 'utf8'), - typeof failures === 'number' && failures ? 'first failure' : 'pass' + 'attempt 1' ); + if (failures === 1 || failures === 2) { + assert.deepEqual( + s.results[0].history.map((h: any) => h.event.status), + ['passed', ...Array(attempts - 1).fill('skipped')] + ); + assert.equal(s.results[1].firstAttempt, 'failed'); + assert.match( + fs.readFileSync(path.join(root, 'artifacts/reporting/attempt-1-runner.log'), 'utf8'), + /controlled case failure/ + ); + } }); test('attempt budget is bounded, sealed and backwards compatible', () => { diff --git a/test/unit/reporting.test.ts b/test/unit/reporting.test.ts index c50fd29c..70cf8e5a 100644 --- a/test/unit/reporting.test.ts +++ b/test/unit/reporting.test.ts @@ -161,7 +161,7 @@ test('known coverage gaps remain visible even if a case happened to pass', () => assert.equal(result.status, 'incomplete'); assert.match(result.problems.join(' '), /Unresolved coverage gap/); }); -test('inventory-enabled runs bypass legacy completion locks without changing legacy mode', (t) => { +test('inventory retries reuse only current-plan locks; legacy paths stay unchanged', (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'report-lock-test-')); t.after(() => fs.rmSync(root, { recursive: true, force: true })); const helper = path.resolve('test/helpers/suite.ts'); @@ -170,9 +170,25 @@ test('inventory-enabled runs bypass legacy completion locks without changing leg [ '--experimental-strip-types', '-e', - `const fs=require('node:fs'); fs.existsSync=()=>true; fs.mkdirSync=()=>{throw Error('must not touch legacy locks')}; fs.writeFileSync=()=>{throw Error('must not touch legacy locks')}; const s=require(${JSON.stringify(helper)}); if(s.checkComplete('case'))throw Error('stale pass'); s.markComplete('case'); delete process.env.E2E_PLAN; if(!s.checkComplete('case'))throw Error('legacy behavior changed'); console.log('ok');`, + ` + const assert = require('node:assert/strict'); + const s = require(${JSON.stringify(helper)}); + const name = '@send_1 - sends coins'; + assert.equal(s.checkComplete(name), false); + s.markComplete(name); + assert.equal(s.checkComplete(name), true); + assert.equal(s.checkComplete('@send_2 - another case'), false); + process.env.E2E_PLAN = ${JSON.stringify(path.join(root, 'next-run/plan.json'))}; + assert.equal(s.checkComplete(name), false); + delete process.env.E2E_PLAN; + assert.equal(s.lockFile(name), '/tmp/lock/lock-_send_1_-_sends_coins'); + console.log('ok'); + `, ], - { env: { ...process.env, CI: '1', E2E_PLAN: path.join(root, 'plan.json') }, encoding: 'utf8' } + { + env: { ...process.env, CI: '1', E2E_PLAN: path.join(root, 'run/plan.json') }, + encoding: 'utf8', + } ); assert.match(output, /ok/); });