diff --git a/.github/workflows/inventory.yml b/.github/workflows/inventory.yml new file mode 100644 index 00000000..42bbadec --- /dev/null +++ b/.github/workflows/inventory.yml @@ -0,0 +1,46 @@ +name: E2E inventory and reporting + +on: + pull_request: + paths: + - 'test/**' + - 'scripts/e2e-inventory.ts' + - 'scripts/ci-*' + - 'scripts/verify-e2e-sha.sh' + - '.github/workflows/migration-wallet-setup.yml' + - 'wdio.conf.ts' + - 'package*.json' + - '.github/workflows/inventory.yml' + push: + branches: [main] + paths: + - 'test/**' + - 'scripts/e2e-inventory.ts' + - 'scripts/ci-*' + - 'scripts/verify-e2e-sha.sh' + - '.github/workflows/migration-wallet-setup.yml' + - 'wdio.conf.ts' + - 'package*.json' + - '.github/workflows/inventory.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: inventory-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + inventory: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: '22' + cache: npm + - run: npm ci --ignore-scripts + - run: npm run inventory:check + - run: npm run test:reporting diff --git a/.github/workflows/migration-wallet-setup.yml b/.github/workflows/migration-wallet-setup.yml index 065ac080..2f6af7f3 100644 --- a/.github/workflows/migration-wallet-setup.yml +++ b/.github/workflows/migration-wallet-setup.yml @@ -1,63 +1,73 @@ name: Migration Wallet Setup - on: workflow_call: inputs: e2e_branch: - description: 'Candidate branch of synonymdev/bitkit-e2e-tests. Re-validated immediately before checkout. Callers may pass an already-resolved ref; a missing default companion falls back to main.' + description: Candidate branch of synonymdev/bitkit-e2e-tests. Re-validated immediately before checkout. Callers may pass an already-resolved ref; a missing default companion falls back to main. required: true type: string e2e_branch_input: - description: 'Resolution mode (main | default-feature-branch | custom branch name). Empty inherits the caller workflow_dispatch input e2e_branch, then default-feature-branch. A custom name hard-fails when the branch is missing.' + description: Resolution mode (main | default-feature-branch | custom branch name). Empty inherits the caller workflow_dispatch input e2e_branch, then default-feature-branch. A custom name hard-fails when the branch is missing. required: false type: string - default: '' + default: "" rn_version: - description: 'Legacy RN app version to use for setup (e.g., v1.1.6)' + description: Legacy RN app version to use for setup (e.g., v1.1.6) required: false type: string - default: 'v1.1.6' + default: v1.1.6 setup_type: - description: 'Wallet setup type (standard | passphrase | sweep)' + description: Wallet setup type (standard | passphrase | sweep) required: true type: string scenario_name: - description: 'Migration scenario name for artifact naming' + description: Migration scenario name for artifact naming required: true type: string - + e2e_sha: + description: Optional immutable E2E commit. Checkout must match; never resolve or fall back as a branch. + type: string + default: "" + diagnostics_retention_days: + description: Preparation-log retention in days; zero preserves the repository default. + type: number + default: 0 env: TERM: xterm-256color FORCE_COLOR: 1 - jobs: prepare-wallets: runs-on: ubuntu-latest - steps: - # Re-resolve in this job. "Re-run failed jobs" does not re-run a - # successful resolve job in the caller, so inputs.e2e_branch can name - # a companion that was deleted after that job finished. + - name: Validate pinned commit + if: inputs.e2e_sha != '' + env: + E2E_SHA: ${{ inputs.e2e_sha }} + run: "[[ \"$E2E_SHA\" =~ ^[0-9a-f]{40}$ ]] || { echo 'Invalid pinned E2E commit' >&2; exit 1; }" - name: Resolve E2E tests ref id: e2e-ref uses: synonymdev/bitkit-e2e-tests/.github/actions/resolve-e2e-ref@main with: selected: ${{ inputs.e2e_branch }} e2e_branch_input: ${{ inputs.e2e_branch_input || github.event.inputs.e2e_branch || 'default-feature-branch' }} - + if: inputs.e2e_sha == '' - name: Clone E2E tests uses: actions/checkout@v4 with: repository: synonymdev/bitkit-e2e-tests path: bitkit-e2e-tests - ref: ${{ steps.e2e-ref.outputs.ref }} - + ref: ${{ inputs.e2e_sha || steps.e2e-ref.outputs.ref }} + - name: Verify pinned checkout + if: inputs.e2e_sha != '' + working-directory: bitkit-e2e-tests + env: + E2E_SHA: ${{ inputs.e2e_sha }} + run: bash scripts/verify-e2e-sha.sh checkout - name: Enable KVM run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm - - name: Download RN app for migration working-directory: bitkit-e2e-tests env: @@ -66,24 +76,20 @@ jobs: python3 scripts/download-migration-app.py android rn "$RN_VERSION" # WDIO initializes its session using the RN source during preparation. ln -sf bitkit_rn_regtest.apk aut/bitkit_e2e.apk - - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: 22 - - name: Cache npm cache - uses: actions/cache@v3 + uses: actions/cache@v6 with: path: ~/.npm key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} restore-keys: | ${{ runner.os }}-node- - - name: Install dependencies working-directory: bitkit-e2e-tests run: npm ci - - name: Prepare independent wallets for all consumer attempts uses: reactivecircus/android-emulator-runner@v2 with: @@ -97,16 +103,15 @@ jobs: env: BACKEND: regtest SETUP_TYPE: ${{ inputs.setup_type }} - - name: Upload preparation diagnostics if: always() uses: actions/upload-artifact@v4 with: name: migration-setup-logs_${{ inputs.scenario_name }}_${{ github.run_attempt }} + retention-days: ${{ inputs.diagnostics_retention_days }} path: | bitkit-e2e-tests/artifacts/attempt-*/ bitkit-e2e-tests/artifacts/migration-source.json - - name: Upload migration env file uses: actions/upload-artifact@v4 with: @@ -118,3 +123,4 @@ jobs: bitkit-e2e-tests/artifacts/wallet-attempt-3/ if-no-files-found: error retention-days: 1 + timeout-minutes: 90 diff --git a/AGENTS.md b/AGENTS.md index c4cf190c..3d8ea509 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -169,26 +169,10 @@ Implication for feature work: - Keep Android/iOS platform differences behind helpers in `test/helpers/`. - Prefer extracting shared flows into `test/helpers/` over copying logic between specs. Keep helpers small and reuse existing ones before adding new test code. -## Local vs staging tags +## iOS CI selection -The app merge gate (`e2e.yml` / `e2e-tests.yml`) runs `BACKEND=local` (docker Electrum/LND). Staging (`e2e-staging.yml`) and migration (`e2e_migration.yml`) run `BACKEND=regtest` against stag0. +The iOS consumer owns its path policy in `bitkit-ios/.github/e2e-policy.json` and selector in `.github/scripts/e2e-selection.cjs`. It reads this repo's `test/inventory.json`; iOS paths and selection policy do not belong in this shared repo. Historical `@ios_gate` and `@ios_nightly` tags do not control that consumer. Its base cases are `send_1`, `send_2`, `lightning_1`, `transfer_2`, `boost_2`; automatic risk selection expands coverage. Nightly E2E runs all applicable local/staging cases; the independent migration workflow runs all six migrations nightly and on release branches. See [inventory and reporting](docs/inventory-reporting.md). -Capability tags (`@settings`, `@boost`, `@send`, …) stay on titles so Android local can keep grepping them. iOS queue tags are disjoint from `@staging` and from `*_staging` (mocha grep is a regex substring). +Keep capability tags for Android's existing selectors and manual execution. Android workflows and selections are unchanged. Do not rename tags merely to match the iOS policy. -| Tag | Android local | iOS Mini (`e2e-tests.yml`) | Staging (`e2e-staging.yml`) | -|---|---|---|---| -| `@ios_gate` | ignored | merge gate grep | no | -| `@ios_nightly` | ignored (still run via `@settings` etc.) | no | extra iOS shard (`ios_nightly_ui`), GitHub mac, not Mini | -| `@transfer_1`, `@transfer_max` | no | no | `@transfer_staging` (keep `@transfer_1` / `@transfer_max` / `@transfer`). `@transfer_1` is not in app `e2e-staging.yml` yet — add the transfer shard once it is 3× green. | -| `@multi_address_2` | no | no | `@multi_address_staging` only | -| `@pubky` / `@paykit` / `@pubky_profile` | no | no | `@pubky_staging` (keep capability tags for other greps) | -| `@boost`, `@multi_address_3` | yes | `@ios_gate` (needs exclusive miner) | no | -| `@send_2`, `@send_3`, `@lightning`, `@lnurl`, `@transfer_2`, `@multi_address_4` | yes | `@ios_gate` (local LND) | no | -| `@hardware_wallet` | no (staging only) | `@ios_gate` | Android staging greps `@hardware_wallet`; do **not** add `@hardware_wallet_staging`. iOS Mini greps `@ios_gate` on this describe. No Trezor on GitHub mac. | -| `@onboarding`, `@backup`, `@receive`, `@onchain_1`, `@onchain_2`, `@send_1`, `@multi_address_1`, `@settings`, `@numberpad`, `@widgets`, `@security` | yes | `@ios_nightly` | iOS nightly shard | -| `@onchain_3` | yes (`@onchain`) | no | no | -| `@migration_*` | migration workflow (nightly, dispatch, `release-*` PRs) | same | no | - -Put `@ios_gate` / `@ios_nightly` on `describe` when the whole file is one iOS queue; on `ciIt` when siblings split (`@send_1` vs `@send_2`/`_3`, `@multi_address_1` vs `_3`/`_4`). Local-LND specs (`@send_3`, `@multi_address_4`) stay `@ios_gate` — they cannot run on the regtest nightly shard. Mini keeps boost/RBF, local-LND money path, and hardware wallet. - -Poke staging with `gh workflow run e2e-staging.yml` on the app repo (does not queue the iOS Mini). Optional Slack post to `#bitkit-staging-nightly` via dispatch `post_to_slack`. +Inventory-enabled iOS execution uses `scripts/ci-run.ts attempt` from three visible workflow steps. `ciIt()` skips cases that passed earlier in the same run, so only failed or uncompleted cases repeat. Completion locks are scoped to the run directory and rebuilt from verified worker results before each attempt. Reports combine current-run passes and retain first-failure evidence. Android non-inventory retries retain their existing `/tmp/lock` behavior. Plans and receipts must identify the app, test source, artifact, backend and run. Never reuse a previous night's pass as current execution. diff --git a/docs/inventory-reporting.md b/docs/inventory-reporting.md new file mode 100644 index 00000000..b87357ae --- /dev/null +++ b/docs/inventory-reporting.md @@ -0,0 +1,63 @@ +# iOS E2E selection and reporting + +Issue: https://github.com/synonymdev/bitkit-e2e-tests/issues/264 + +The iOS `e2e-tests.yml` entry point owns PR selection, manual validation and the nightly schedule. Staging is reusable. The independent `e2e_migration.yml` workflow owns its schedule, release pushes, app build and result. There is no shadow gate. Android workflow triggers, test selection, backend assignments and legacy retry behavior remain unchanged. + +## PR selection + +Every iOS app-code PR starts with these existing cases, unchanged: + +- `send_1`, `send_2`, `lightning_1`, `transfer_2`, `boost_2`. + +The iOS repo owns `.github/e2e-policy.json`, its selector and policy tests. Its tested app checkout supplies the policy; this repository supplies only the case inventory and execution/reporting. Change iOS paths and their policy together in the same PR. See [iOS policy documentation](https://github.com/synonymdev/bitkit-ios/blob/master/Docs/e2e-ci.md). + +Feature directories and explicit service/model/helper files add only their relevant existing cases. For example, widget services add widget cases, Paykit services add staging, and backup services add recovery coverage. Only named shared wallet/lifecycle dependencies and build/orchestration configuration select every PR-eligible local and staging case. There is no blanket Services/Models/Managers/Utilities rule. + +Cheap iOS validation checks all tracked paths, stale policy entries and referenced case IDs. Unknown paths fail with an actionable policy-update error before paid builds. Old and new rename paths contribute coverage from the base and tested policies, so a refactor cannot silently drop coverage. Documentation/native-test/policy-only changes skip device jobs. Selection records cases, reasons, app/E2E/policy identity and explicit gaps. USDT-specific paths retain the Bitcoin base gate and report the lack of existing USDT E2E coverage; unrelated Bitcoin tests do not establish USDT correctness. + +PRs stacked on another branch retain the existing automatic exclusion. Manual dispatch with `pr` checks the PR's current merge against its actual base/head, including stacked PRs. The workflow source is the branch selected for dispatch. `selection=auto` uses the PR diff; `base` runs the five cases; `full` runs all applicable local and staging cases. Migration runs nightly and after each push to an iOS `release-*` branch. Manual E2E dispatches use local and staging cases. Automatic PRs cannot narrow selection with a manual input. + +## Nightly + +Two separate iOS workflows run each night at 03:00 UTC: + +| Lane | Active existing cases | +| --- | --- | +| Local | 42: all core UI, local money, on-chain, multi-address and active hardware cases | +| Staging | 6: `pubky_profile_1` through `pubky_profile_5`, `paykit_1` | +| Independent migration workflow | 6: native restore/upgrade, RN restore/upgrade, passphrase and legacy-address migration | + +The 54 cases use the appropriate local or remote-regtest build and fixtures. Migration setup entries are fixtures, not extra regression passes. Five existing skipped cases and six unsupported iOS mainnet cases remain explicit gaps; the interactive Lightning payment utility remains manual. This does not invent missing USDT, allowance or subscription automation. + +## Attempts and evidence + +`scripts/ci-run.ts` prepares a plan tied to the actual E2E checkout, selected app SHA, downloaded artifact digest, platform, backend, variant and run identity. Selection rejects unknown, skipped or wrong-backend cases. Only selected spec files load. + +Local and staging jobs expose `Run E2E Tests 1`, `2`, and `3` as separate workflow steps. Each invokes `scripts/ci-run.ts attempt` once; later steps run only after failure. The established `ciIt()` pass locks skip cases already passed, so only failed or uncompleted cases repeat. Inventory-enabled locks live beside the current run plan and are rebuilt from verified worker results before each attempt; stale machine locks and passes from failed hooks or interrupted workers cannot skip work. Android keeps its existing `/tmp/lock` behavior. Migration remains independent and keeps its per-attempt prepared-wallet setup. + +The final report combines selected cases passed across these attempts. It requires the last runner to exit successfully and every selected case to have a verified current-run pass. All three failed attempts produce a failed job despite the attempts using `continue-on-error`. Each attempt retains worker events, process outcome and timing. + +A recovered failure is reported as a recovery, with the first failed/missing case results and earlier errors retained. Missing reports, setup/teardown errors, incomplete workers, unexpected skips, stale identities or a nonzero final runner cannot produce a pass. The parent `e2e-status` requires every selected local or staging job and build to succeed. The migration workflow reports its own result. + +## Cost controls and manual validation + +Automatic runs use one local shard. Only manual dispatch can request two, partitioned by spec so a spec's fixture lifecycle stays together. Namespace supplies a separate Mac and backend instance per shard; every shard destroys its backend in cleanup. The local app is built once and reused. Staging and the independent migration workflow each use their own network build. Builds have a 40-minute limit; execution limits depend on the selected pack as described below. iOS build and diagnostic artifacts, including migration preparation logs, are retained for seven days; prepared wallet environments expire after one day. The shared preparation workflow preserves repository-default diagnostic retention for consumers that omit its optional override. Superseded runs cancel. + +Compare `selection=full, shards=1` and `selection=full, shards=2` on the same app and E2E revisions. Inspect elapsed/build/setup/test time, first-pass success and billed Mac plus backend usage. Two shards are an experiment, not an assumed cost saving or a new automatic default. + +Merge the E2E producer before the iOS consumer. Before merging, dispatch the existing iOS workflow from the development branch and explicitly select the E2E development branch. Validate base/full/automatic selection, failures/recovery and the optional shard experiment. The default branch's gate stays intact until the consumer is merged. No parallel shadow workflow is installed. + +## Local checks + +```sh +npm run inventory:check +npm run test:reporting +npx tsc --noEmit +``` + +The CLI workflow tests use controlled runner fixtures; they are not evidence that hosted Appium journeys passed. Actual hosted measurements and device results must name their revisions and environments. + +Local shards with 6–30 cases allow 90 minutes per attempt within a 285-minute job and five-hour backend expiry. Larger shards allow 110 minutes per attempt within a 345-minute job and six-hour backend expiry. Both sizes allow three attempts. Base-sized local selections retain 40-minute attempts and a 135-minute job; the six-case staging pack also uses 90-minute attempts within 285 minutes. Individual migration cases retain 40-minute attempts and 135-minute jobs. The iOS selector owns these budgets and the shared runner seals its attempt limit into the run plan. Limits are ceilings, not reservations: success stops retries and cleanup destroys the backend immediately. + +RN passphrase migration verifies the exact restored passphrase and the funded RN source balance before installing the native target. Native verification dismisses the optional USDT introduction before asserting balances, history, tags and persistence after relaunch. A malformed fixture or blocked screen remains a failure; neither is reported as a successful migration. diff --git a/package-lock.json b/package-lock.json index 310d12c4..cc8aca94 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@radar/lnrpc": "^0.11.1-beta.1", "bip21": "^3.0.0", "bitcoin-json-rpc": "^1.4.1", + "bitcoinjs-lib": "6.1.7", "lnurl": "0.26.2", "rn-electrum-client": "^0.0.22" }, @@ -23,6 +24,7 @@ "@wdio/cli": "^9.18.2", "@wdio/local-runner": "^9.18.1", "@wdio/mocha-framework": "^9.18.0", + "@wdio/reporter": "9.18.0", "@wdio/spec-reporter": "^9.18.0", "appium": "3.0.0", "appium-uiautomator2-driver": "5.0.0", diff --git a/package.json b/package.json index c13ddf4e..75c38821 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,8 @@ "name": "bitkit-e2e-tests", "version": "1.0.0", "scripts": { + "inventory:check": "node --experimental-strip-types scripts/e2e-inventory.ts check", + "test:reporting": "node --import tsx --test test/unit/inventory.test.ts test/unit/reporting.test.ts test/unit/reporting-cli.test.ts test/unit/reporter-events.test.ts test/unit/ci-retry.test.ts test/unit/wdio-scheduling.test.cjs test/unit/pinned-fixture-source.test.cjs", "e2e:android": "PLATFORM=android wdio run ./wdio.conf.ts", "e2e:ios": "PLATFORM=ios wdio run ./wdio.conf.ts", "lint": "eslint './**/*.{ts,js}'", @@ -18,6 +20,7 @@ "@wdio/cli": "^9.18.2", "@wdio/local-runner": "^9.18.1", "@wdio/mocha-framework": "^9.18.0", + "@wdio/reporter": "9.18.0", "@wdio/spec-reporter": "^9.18.0", "appium": "3.0.0", "appium-uiautomator2-driver": "5.0.0", @@ -32,6 +35,7 @@ "@radar/lnrpc": "^0.11.1-beta.1", "bip21": "^3.0.0", "bitcoin-json-rpc": "^1.4.1", + "bitcoinjs-lib": "6.1.7", "lnurl": "0.26.2", "rn-electrum-client": "^0.0.22" }, diff --git a/scripts/ci-run.ts b/scripts/ci-run.ts new file mode 100644 index 00000000..0cb98ac7 --- /dev/null +++ b/scripts/ci-run.ts @@ -0,0 +1,262 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { + loadInventory, + validateInventory, + type Platform, + type Backend, +} from '../test/reporting/inventory.ts'; +import { executionIdentity, assertExecutionIdentity } from '../test/reporting/identity.ts'; +import { lockFile } from '../test/helpers/suite.ts'; +import { + createPlan, + validatePlan, + digest, + writeJson, + reconcile, + reconcileAttempts, + markdownReport, + type Plan, + type WorkerReport, + type AttemptReceipt, +} from '../test/reporting/report.ts'; + +async function runProcess( + runner: string, + args: string[], + env: NodeJS.ProcessEnv, + log: string, + timeoutMinutes: number +) { + const output = fs.createWriteStream(log); + const child = spawn(runner, args, { env, detached: true, stdio: ['ignore', 'pipe', 'pipe'] }); + let cancelled = false; + let timedOut = false; + let forceKill: ReturnType | undefined; + const kill = (signal: NodeJS.Signals) => { + if (child.pid) { + try { + process.kill(-child.pid, signal); + } catch (error: any) { + if (error.code !== 'ESRCH') throw error; + } + } + }; + const stop = () => { + kill('SIGTERM'); + forceKill = setTimeout(() => kill('SIGKILL'), 5000); + }; + const interrupt = () => { + cancelled = true; + stop(); + }; + process.once('SIGTERM', interrupt); + process.once('SIGINT', interrupt); + const timer = setTimeout( + () => { + timedOut = true; + console.error( + `Runner attempt exceeded ${timeoutMinutes} minutes; terminating its process group` + ); + stop(); + }, + timeoutMinutes * 60 * 1000 + ); + child.stdout?.on('data', (data) => { + process.stdout.write(data); + output.write(data); + }); + child.stderr?.on('data', (data) => { + process.stderr.write(data); + output.write(data); + }); + child.on('error', (error) => output.write(String(error) + '\n')); + const status = await new Promise((resolve) => + child.on('close', (code) => resolve(code ?? 1)) + ); + clearTimeout(timer); + if (forceKill) clearTimeout(forceKill); + // Also terminate descendants that outlived the runner itself. + kill('SIGKILL'); + process.removeListener('SIGTERM', interrupt); + process.removeListener('SIGINT', interrupt); + await new Promise((resolve) => output.end(resolve)); + return { status: cancelled || timedOut ? 1 : status, cancelled }; +} + +function artifactHash(directory: string): string { + const hash = crypto.createHash('sha256'); + function visit(relative: string) { + const filename = path.join(directory, relative); + const stat = fs.lstatSync(filename); + hash.update(relative + '\0'); + if (stat.isSymbolicLink()) hash.update('link\0' + fs.readlinkSync(filename)); + else if (stat.isDirectory()) + for (const name of fs.readdirSync(filename).sort()) visit(path.join(relative, name)); + else hash.update(fs.readFileSync(filename)); + } + visit(''); + return hash.digest('hex'); +} +const command = process.argv[2]; +const planFile = process.env.E2E_PLAN || 'artifacts/reporting/plan.json'; +const directory = path.dirname(planFile); +function workers(): WorkerReport[] { + const dir = path.join(directory, 'workers'); + return fs.existsSync(dir) + ? fs + .readdirSync(dir) + .filter((f) => f.endsWith('.json')) + .sort() + .map((f) => JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8'))) + : []; +} +function receipts(): AttemptReceipt[] { + return fs.existsSync(directory) + ? fs + .readdirSync(directory) + .filter((f) => /^attempt-\d+\.json$/.test(f)) + .sort() + .map((f) => JSON.parse(fs.readFileSync(path.join(directory, f), 'utf8'))) + : []; +} +function summary(plan: Plan) { + const result = reconcileAttempts(plan, workers(), receipts()); + writeJson(path.join(directory, 'summary.json'), result); + const markdown = + markdownReport(result) + + `\nAttempts: ${result.attemptsUsed}/${result.maxAttempts}. Limit per attempt: ${plan.attemptTimeoutMinutes ?? 40} minutes.\n\n` + + result.attempts + .map( + (a) => + `- Attempt ${a.attempt}: ${a.passed ? 'passed' : 'failed/incomplete'}; runner exit ${a.exitCode ?? 'missing'}${a.problems.length ? '; ' + a.problems.join('; ') : ''}` + ) + .join('\n') + + '\n'; + fs.writeFileSync(path.join(directory, 'summary.md'), markdown); + console.info(markdown); + if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown); + return result; +} +async function main() { + try { + if (command === 'prepare') { + if (fs.existsSync(planFile)) throw Error('Plan exists: use a fresh run directory'); + const inventory = loadInventory(process.cwd()); + validateInventory(process.cwd(), inventory); + const platform = process.env.PLATFORM as Platform, + backend = (process.env.BACKEND || 'local') as Backend; + const ids: string[] = process.env.E2E_CASE_TAG + ? [process.env.E2E_CASE_TAG.replace(/^@/, '')] + : JSON.parse(process.env.E2E_CASE_IDS || '[]'); + if (!Array.isArray(ids) || !ids.length || new Set(ids).size !== ids.length) + throw Error('Empty or duplicate case selection'); + const cases = inventory.cases.filter((c) => ids.includes(c.id)); + if ( + cases.length !== ids.length || + cases.some( + (c) => + c.kind !== 'test' || + c.platforms[platform]?.disposition !== 'required' || + c.platforms[platform]?.backend !== backend + ) + ) + throw Error('Selection contains unknown, unavailable or wrong-backend cases'); + const metadata = JSON.parse(fs.readFileSync('aut/e2e-build.json', 'utf8')); + if ( + metadata.appSha !== process.env.E2E_APP_SHA || + metadata.backend !== backend || + metadata.platform !== platform + ) + throw Error('Downloaded build does not match selected app/platform/backend'); + const artifact = artifactHash( + path.join( + 'aut', + process.env.AUT_FILENAME || (platform === 'ios' ? 'bitkit.app' : 'bitkit_e2e.apk') + ) + ); + process.env.E2E_ARTIFACT_DIGEST = artifact; + if (process.env.GITHUB_ENV) + fs.appendFileSync(process.env.GITHUB_ENV, `E2E_ARTIFACT_DIGEST=${artifact}\n`); + const plan = createPlan( + cases, + executionIdentity(platform, backend), + digest(fs.readFileSync('test/inventory.json', 'utf8')), + 'selected', + 3, + Number(process.env.E2E_ATTEMPT_TIMEOUT_MINUTES ?? '40') + ); + writeJson(planFile, plan); + fs.writeFileSync( + path.join(directory, 'grep.txt'), + `(?:^| )@(?:${plan.expected.join('|')})(?=[ ,]|$)\n` + ); + } else { + const plan: Plan = JSON.parse(fs.readFileSync(planFile, 'utf8')); + validatePlan(plan); + assertExecutionIdentity(plan); + if (command === 'attempt') { + const attempt = Number(process.env.ATTEMPT); + if (!Number.isInteger(attempt) || attempt < 1 || attempt > plan.maxAttempts) + throw Error('Invalid attempt'); + const previousReceipts = receipts(); + const previousWorkers = workers(); + if (previousReceipts.length !== attempt - 1) + throw Error('Unexpected attempt receipts; refusing to reuse another execution'); + if ( + attempt > 1 && + reconcileAttempts(plan, previousWorkers, previousReceipts).status === 'passed' + ) + throw Error('Previous attempt already passed'); + + // Keep ciIt's established pass-lock behavior, isolated to this execution. + // Rebuild locks from verified results so an interrupted body/failed hook or + // a previous run can never supply a pass to the next attempt. + const completed = path.join(directory, 'completed'); + fs.rmSync(completed, { recursive: true, force: true }); + fs.mkdirSync(completed, { recursive: true }); + for (const result of reconcile(plan, previousWorkers).results) { + if (result.status === 'passed') + fs.writeFileSync(lockFile(`@${result.id}`, planFile), '1'); + } + const grep = fs.readFileSync(path.join(directory, 'grep.txt'), 'utf8').trim(); + const runner = plan.identity.platform === 'ios' ? './ci_run_ios.sh' : './ci_run_android.sh'; + const receipt: AttemptReceipt = { attempt, startedAt: new Date().toISOString() }; + const file = path.join(directory, `attempt-${attempt}.json`); + writeJson(file, receipt); + console.info( + `Attempt ${attempt}/${plan.maxAttempts}: limit ${plan.attemptTimeoutMinutes ?? 40} minutes` + ); + const child = await runProcess( + runner, + ['--mochaOpts.grep', grep], + { ...process.env, ATTEMPT: String(attempt), E2E_PLAN: planFile }, + path.join(directory, `attempt-${attempt}-runner.log`), + plan.attemptTimeoutMinutes ?? 40 + ); + receipt.finishedAt = new Date().toISOString(); + receipt.exitCode = child.status; + writeJson(file, receipt); + const result = reconcile(plan, workers()); + writeJson(path.join(directory, `attempt-${attempt}-summary.json`), { + ...result, + runnerExit: receipt.exitCode, + }); + if (receipt.exitCode !== 0 || result.status !== 'passed') + console.error( + `Attempt ${attempt} failed; evidence retained in ${directory} and artifacts/attempt-${attempt}` + ); + } else if (command !== 'report') throw Error('Usage: ci-run.ts prepare|attempt|report'); + if (summary(plan).status !== 'passed') process.exitCode = 1; + } + } catch (error) { + console.error(error); + process.exitCode = 1; + } +} +void main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/scripts/e2e-inventory.ts b/scripts/e2e-inventory.ts new file mode 100644 index 00000000..65ed4b67 --- /dev/null +++ b/scripts/e2e-inventory.ts @@ -0,0 +1,79 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { assertExecutionIdentity, executionIdentity } from '../test/reporting/identity.ts'; +import { parseArgs } from 'node:util'; +import { + loadInventory, + validateInventory, + type Backend, + type Platform, +} from '../test/reporting/inventory.ts'; +import { + createPlan, + digest, + markdownReport, + reconcile, + writeJson, + type Plan, + type WorkerReport, +} from '../test/reporting/report.ts'; + +const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + platform: { type: 'string' }, + backend: { type: 'string' }, + pack: { type: 'string' }, + plan: { type: 'string', default: 'artifacts/reporting/plan.json' }, + }, +}); +const root = process.cwd(); +const command = positionals[0]; +try { + if (command === 'check' || command === 'plan') { + const inventory = loadInventory(root); + const discovered = validateInventory(root, inventory); + console.info(`Inventory accounts for ${discovered.length} cases on both platforms.`); + if (command === 'plan') { + if (fs.existsSync(values.plan!)) + throw new Error('Plan already exists; use a fresh run directory'); + const plan = createPlan( + inventory.cases, + executionIdentity(values.platform as Platform, values.backend as Backend), + digest(fs.readFileSync('test/inventory.json', 'utf8')), + values.pack!, + Number(process.env.E2E_MAX_ATTEMPTS || '1') + ); + writeJson(values.plan!, plan); + const grep = `(?:^| )@(?:${plan.expected.join('|')})(?=[ ,]|$)`; + fs.writeFileSync(path.join(path.dirname(values.plan!), 'grep.txt'), grep + '\n'); + console.info(`Planned ${plan.expected.length} cases; plan ${plan.id}`); + } + } else if (command === 'report') { + const plan: Plan = JSON.parse(fs.readFileSync(values.plan!, 'utf8')); + assertExecutionIdentity(plan); + const directory = path.join(path.dirname(values.plan!), 'workers'); + const reports: WorkerReport[] = fs.existsSync(directory) + ? fs + .readdirSync(directory) + .filter((file) => file.endsWith('.json')) + .sort() + .map((file) => JSON.parse(fs.readFileSync(path.join(directory, file), 'utf8'))) + : []; + const result = reconcile(plan, reports); + writeJson(path.join(path.dirname(values.plan!), 'summary.json'), result); + const markdown = markdownReport(result); + fs.writeFileSync(path.join(path.dirname(values.plan!), 'summary.md'), markdown); + if (process.env.GITHUB_STEP_SUMMARY) + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown); + console.info(markdown); + if (result.status !== 'passed') process.exitCode = 1; + } else { + throw new Error( + 'Usage: e2e-inventory.ts check | plan --platform ios|android --backend local|regtest|mainnet --pack NAME [--plan FILE] | report [--plan FILE]' + ); + } +} catch (error) { + console.error(String(error)); + process.exitCode = 1; +} diff --git a/scripts/verify-e2e-sha.sh b/scripts/verify-e2e-sha.sh new file mode 100755 index 00000000..f1fed849 --- /dev/null +++ b/scripts/verify-e2e-sha.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +[[ "${E2E_SHA:-}" =~ ^[0-9a-f]{40}$ ]] || { echo 'Invalid pinned E2E commit' >&2; exit 1; } +if [[ "${1:-}" == checkout ]]; then + [[ "$(git rev-parse HEAD)" == "$E2E_SHA" ]] || { echo 'Checkout differs from pinned E2E commit' >&2; exit 1; } +fi diff --git a/test/helpers/suite.ts b/test/helpers/suite.ts index e4436191..bd585a54 100644 --- a/test/helpers/suite.ts +++ b/test/helpers/suite.ts @@ -47,11 +47,17 @@ export function checkComplete(name: string): boolean { export function markComplete(name: string) { if (!ON_CI) return; - fs.mkdirSync(LOCK_PATH, { recursive: true }); - fs.writeFileSync(lockFile(name), '1'); + const file = lockFile(name); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, '1'); } -function lockFile(key: string) { +export function lockFile(key: string, planFile = process.env.E2E_PLAN) { + if (planFile) { + const id = /^@([a-z][a-z0-9_]*)(?=[\s,]|$)/.exec(key)?.[1]; + if (!id) throw new Error(`Missing case ID for retry lock: ${key}`); + return path.join(path.dirname(planFile), 'completed', `lock-${id}`); + } // replace anything that's *not* a letter, number, dot, underscore, or dash // with an underscore. This avoids weird characters in filenames. const safe = key.replace(/[^a-zA-Z0-9._-]/g, '_'); diff --git a/test/inventory.json b/test/inventory.json new file mode 100644 index 00000000..5d5c5c46 --- /dev/null +++ b/test/inventory.json @@ -0,0 +1,1577 @@ +{ + "schemaVersion": 1, + "cases": [ + { + "id": "backup_1", + "file": "test/specs/backup.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "boost_1", + "file": "test/specs/boost.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "boost_2", + "file": "test/specs/boost.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "hardware_wallet_1", + "file": "test/specs/hardware-wallet.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "hardware", + "disposition": "required", + "fixtures": ["fresh-wallet", "trezor-emulator", "trezor-bridge-build"] + }, + "android": { + "backend": "regtest", + "pack": "hardware", + "disposition": "required", + "fixtures": ["fresh-wallet", "trezor-emulator", "trezor-bridge-build"] + } + } + }, + { + "id": "hardware_wallet_2", + "file": "test/specs/hardware-wallet.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "hardware", + "disposition": "required", + "fixtures": ["fresh-wallet", "trezor-emulator", "trezor-bridge-build"] + }, + "android": { + "backend": "regtest", + "pack": "hardware", + "disposition": "required", + "fixtures": ["fresh-wallet", "trezor-emulator", "trezor-bridge-build"] + } + } + }, + { + "id": "hardware_wallet_3", + "file": "test/specs/hardware-wallet.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "hardware", + "disposition": "gap", + "fixtures": ["fresh-wallet", "trezor-emulator", "trezor-bridge-build"], + "reason": "Currently skipped after Blocktank channel-order retirement; retire or replace its assertions explicitly, rather than counting a skip as a pass.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "regtest", + "pack": "hardware", + "disposition": "gap", + "fixtures": ["fresh-wallet", "trezor-emulator", "trezor-bridge-build"], + "reason": "Currently skipped after Blocktank channel-order retirement; retire or replace its assertions explicitly, rather than counting a skip as a pass.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + } + } + }, + { + "id": "hardware_wallet_4", + "file": "test/specs/hardware-wallet.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "hardware", + "disposition": "required", + "fixtures": ["fresh-wallet", "trezor-emulator", "trezor-bridge-build"] + }, + "android": { + "backend": "regtest", + "pack": "hardware", + "disposition": "required", + "fixtures": ["fresh-wallet", "trezor-emulator", "trezor-bridge-build"] + } + } + }, + { + "id": "lightning_1", + "file": "test/specs/lightning.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "lnurl_1", + "file": "test/specs/lnurl.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "blocktank_sunset_1", + "file": "test/specs/mainnet/blocktank-sunset.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "gap", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "geocheck-enabled", + "BLOCKTANK_SUNSET_UI" + ], + "reason": "The current spec rejects iOS; equivalent applicable iOS behavior needs its own validated fixture/spec.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "required", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "geocheck-enabled", + "BLOCKTANK_SUNSET_UI" + ] + } + } + }, + { + "id": "blocktank_sunset_2", + "file": "test/specs/mainnet/blocktank-sunset.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "gap", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "geocheck-enabled", + "BLOCKTANK_SUNSET_UI" + ], + "reason": "The current spec rejects iOS; equivalent applicable iOS behavior needs its own validated fixture/spec.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "required", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "geocheck-enabled", + "BLOCKTANK_SUNSET_UI" + ] + } + } + }, + { + "id": "blocktank_sunset_3", + "file": "test/specs/mainnet/blocktank-sunset.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "gap", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "geocheck-enabled", + "BLOCKTANK_SUNSET_UI" + ], + "reason": "The current spec rejects iOS; equivalent applicable iOS behavior needs its own validated fixture/spec.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "required", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "geocheck-enabled", + "BLOCKTANK_SUNSET_UI" + ] + } + } + }, + { + "id": "strike_1", + "file": "test/specs/mainnet/ln.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "gap", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "funded-lightning-wallet", + "private-receiver-or-probe-targets" + ], + "reason": "iOS mainnet fixture, devtools and private nightly job support has not been validated.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "required", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "funded-lightning-wallet", + "private-receiver-or-probe-targets" + ] + } + } + }, + { + "id": "wos_1", + "file": "test/specs/mainnet/ln.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "gap", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "funded-lightning-wallet", + "private-receiver-or-probe-targets" + ], + "reason": "iOS mainnet fixture, devtools and private nightly job support has not been validated.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "required", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "funded-lightning-wallet", + "private-receiver-or-probe-targets" + ] + } + } + }, + { + "id": "probe_mainnet_1", + "file": "test/specs/mainnet/probe.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "gap", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "funded-lightning-wallet", + "private-receiver-or-probe-targets" + ], + "reason": "iOS mainnet fixture, devtools and private nightly job support has not been validated.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "mainnet", + "pack": "mainnet", + "disposition": "required", + "fixtures": [ + "mainnet-build", + "private-nightly-configuration", + "funded-lightning-wallet", + "private-receiver-or-probe-targets" + ] + } + } + }, + { + "id": "migration_native_restore", + "file": "test/specs/migration.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "previous-native-artifact", + "target-native-artifact", + "isolated-funded-wallet" + ] + }, + "android": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "previous-native-artifact", + "target-native-artifact", + "isolated-funded-wallet" + ] + } + } + }, + { + "id": "migration_native_upgrade", + "file": "test/specs/migration.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "previous-native-artifact", + "target-native-artifact", + "isolated-funded-wallet" + ] + }, + "android": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "previous-native-artifact", + "target-native-artifact", + "isolated-funded-wallet" + ] + } + } + }, + { + "id": "migration_setup_standard", + "file": "test/specs/migration.e2e.ts", + "kind": "fixture", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration-fixture", + "disposition": "inapplicable", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_SETUP_WALLET" + ], + "reason": "Fixture preparation runs on Android; iOS migration consumes its result, not this Android-only setup case.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "regtest", + "pack": "migration-fixture", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_SETUP_WALLET" + ] + } + } + }, + { + "id": "migration_setup_passphrase", + "file": "test/specs/migration.e2e.ts", + "kind": "fixture", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration-fixture", + "disposition": "inapplicable", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_SETUP_WALLET" + ], + "reason": "Fixture preparation runs on Android; iOS migration consumes its result, not this Android-only setup case.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "regtest", + "pack": "migration-fixture", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_SETUP_WALLET" + ] + } + } + }, + { + "id": "migration_setup_sweep", + "file": "test/specs/migration.e2e.ts", + "kind": "fixture", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration-fixture", + "disposition": "inapplicable", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_SETUP_WALLET" + ], + "reason": "Fixture preparation runs on Android; iOS migration consumes its result, not this Android-only setup case.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "regtest", + "pack": "migration-fixture", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_SETUP_WALLET" + ] + } + } + }, + { + "id": "migration_rn_restore", + "file": "test/specs/migration.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "android-prepared-RN_MNEMONIC-and-RN_BALANCE" + ] + }, + "android": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": ["rn-artifact", "target-native-artifact", "isolated-funded-wallet"] + } + } + }, + { + "id": "migration_rn_upgrade", + "file": "test/specs/migration.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "android-prepared-RN_MNEMONIC-and-RN_BALANCE" + ] + }, + "android": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": ["rn-artifact", "target-native-artifact", "isolated-funded-wallet"] + } + } + }, + { + "id": "migration_3", + "file": "test/specs/migration.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_EXTENDED=true", + "android-prepared-RN_MNEMONIC-and-RN_BALANCE" + ] + }, + "android": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_EXTENDED=true" + ] + } + } + }, + { + "id": "migration_4", + "file": "test/specs/migration.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_EXTENDED=true", + "android-prepared-RN_MNEMONIC-and-RN_BALANCE" + ] + }, + "android": { + "backend": "regtest", + "pack": "migration", + "disposition": "required", + "fixtures": [ + "rn-artifact", + "target-native-artifact", + "isolated-funded-wallet", + "MIGRATION_EXTENDED=true" + ] + } + } + }, + { + "id": "multi_address_1", + "file": "test/specs/multiaddress.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "multi_address_2", + "file": "test/specs/multiaddress.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "gap", + "fixtures": ["fresh-wallet", "blocktank-channel-orders"], + "reason": "Currently skipped after Blocktank channel-order retirement; retire or replace its assertions explicitly, rather than counting a skip as a pass.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "gap", + "fixtures": ["fresh-wallet", "blocktank-channel-orders"], + "reason": "Currently skipped after Blocktank channel-order retirement; retire or replace its assertions explicitly, rather than counting a skip as a pass.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + } + } + }, + { + "id": "multi_address_3", + "file": "test/specs/multiaddress.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "multi_address_4", + "file": "test/specs/multiaddress.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "numberpad_1", + "file": "test/specs/numberpad.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "numberpad_2", + "file": "test/specs/numberpad.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "numberpad_3", + "file": "test/specs/numberpad.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "numberpad_4", + "file": "test/specs/numberpad.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "onboarding_1", + "file": "test/specs/onboarding.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "onboarding_2", + "file": "test/specs/onboarding.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "onchain_1", + "file": "test/specs/onchain.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "onchain_2", + "file": "test/specs/onchain.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "onchain_3", + "file": "test/specs/onchain.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-local", + "disposition": "required", + "fixtures": ["fresh-wallet", "bitcoin-electrum"] + }, + "android": { + "backend": "local", + "pack": "core-local", + "disposition": "required", + "fixtures": ["fresh-wallet", "bitcoin-electrum"] + } + } + }, + { + "id": "paykit_1", + "file": "test/specs/paykit.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + } + } + }, + { + "id": "pubky_profile_1", + "file": "test/specs/pubky-profile.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + } + } + }, + { + "id": "pubky_profile_2", + "file": "test/specs/pubky-profile.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + } + } + }, + { + "id": "pubky_profile_3", + "file": "test/specs/pubky-profile.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + } + } + }, + { + "id": "pubky_profile_4", + "file": "test/specs/pubky-profile.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + } + } + }, + { + "id": "pubky_profile_5", + "file": "test/specs/pubky-profile.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "required", + "fixtures": [ + "fresh-wallet", + "staging-homegate", + "contact-payment-fixture", + "profile-media" + ] + } + } + }, + { + "id": "utility_receive_ln_1", + "file": "test/specs/receive-ln-payments.e2e.ts", + "kind": "utility", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "manual", + "disposition": "manual", + "fixtures": ["already-prepared-wallet"], + "reason": "Interactive payment utility; does not provision its own wallet and accepts partial payment success. Not an autonomous regression case.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "regtest", + "pack": "manual", + "disposition": "manual", + "fixtures": ["already-prepared-wallet"], + "reason": "Interactive payment utility; does not provision its own wallet and accepts partial payment success. Not an autonomous regression case.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + } + } + }, + { + "id": "receive_1", + "file": "test/specs/receive.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "security_1", + "file": "test/specs/security.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "send_1", + "file": "test/specs/send.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "send_2", + "file": "test/specs/send.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "send_3", + "file": "test/specs/send.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "settings_01", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_02", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_03", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_04", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_05", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_06", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_07", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_08", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "gap", + "fixtures": ["fresh-wallet"], + "reason": "Currently skipped with an address-switching-unavailable comment; applicability needs reassessment before restoring or retiring this case.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "gap", + "fixtures": ["fresh-wallet"], + "reason": "Currently skipped with an address-switching-unavailable comment; applicability needs reassessment before restoring or retiring this case.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + } + } + }, + { + "id": "settings_09", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_10", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-local", + "disposition": "required", + "fixtures": ["fresh-wallet", "bitcoin-electrum"] + }, + "android": { + "backend": "local", + "pack": "core-local", + "disposition": "required", + "fixtures": ["fresh-wallet", "bitcoin-electrum"] + } + } + }, + { + "id": "settings_11", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_12", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_13", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "settings_14", + "file": "test/specs/settings.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "transfer_1", + "file": "test/specs/transfer.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "gap", + "fixtures": ["fresh-wallet", "blocktank-channel-orders"], + "reason": "Currently skipped after Blocktank channel-order retirement; retire or replace its assertions explicitly, rather than counting a skip as a pass.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "gap", + "fixtures": ["fresh-wallet", "blocktank-channel-orders"], + "reason": "Currently skipped after Blocktank channel-order retirement; retire or replace its assertions explicitly, rather than counting a skip as a pass.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + } + } + }, + { + "id": "transfer_max", + "file": "test/specs/transfer.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "regtest", + "pack": "staging", + "disposition": "gap", + "fixtures": ["fresh-wallet", "blocktank-channel-orders"], + "reason": "Currently skipped after Blocktank channel-order retirement; retire or replace its assertions explicitly, rather than counting a skip as a pass.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + }, + "android": { + "backend": "regtest", + "pack": "staging", + "disposition": "gap", + "fixtures": ["fresh-wallet", "blocktank-channel-orders"], + "reason": "Currently skipped after Blocktank channel-order retirement; retire or replace its assertions explicitly, rather than counting a skip as a pass.", + "reference": "https://github.com/synonymdev/bitkit-e2e-tests/issues/264", + "owner": "bitkit-e2e-tests maintainers" + } + } + }, + { + "id": "transfer_2", + "file": "test/specs/transfer.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + }, + "android": { + "backend": "local", + "pack": "local-money", + "disposition": "required", + "fixtures": ["fresh-wallet", "exclusive-bitcoin-miner", "lnd", "lnurl-server"] + } + } + }, + { + "id": "widgets_1", + "file": "test/specs/widgets.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "widgets_2", + "file": "test/specs/widgets.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + }, + { + "id": "widgets_3", + "file": "test/specs/widgets.e2e.ts", + "kind": "test", + "platforms": { + "ios": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + }, + "android": { + "backend": "local", + "pack": "core-ui", + "disposition": "required", + "fixtures": ["fresh-wallet"] + } + } + } + ] +} diff --git a/test/reporting/identity.ts b/test/reporting/identity.ts new file mode 100644 index 00000000..64ebd7a1 --- /dev/null +++ b/test/reporting/identity.ts @@ -0,0 +1,31 @@ +import { execFileSync } from 'node:child_process'; +import type { Identity, Plan } from './report.ts'; + +/** Current execution inputs supplied by the builder plus the actual checked-out E2E revision. */ +export function executionIdentity( + platform: Identity['platform'], + backend: Identity['backend'] +): Identity { + return { + runId: process.env.E2E_RUN_ID!, + platform, + backend, + appSha: process.env.E2E_APP_SHA!, + e2eSha: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), + artifactDigest: process.env.E2E_ARTIFACT_DIGEST!, + backendRevision: process.env.E2E_BACKEND_REVISION!, + variant: process.env.E2E_VARIANT || 'default', + }; +} + +export function assertExecutionIdentity(plan: Plan): void { + const current = executionIdentity( + process.env.PLATFORM as Identity['platform'], + (process.env.BACKEND || 'local') as Identity['backend'] + ); + for (const key of Object.keys(current) as (keyof Identity)[]) { + if (current[key] !== plan.identity[key]) { + throw new Error(`Run plan does not match executing ${key}`); + } + } +} diff --git a/test/reporting/inventory.ts b/test/reporting/inventory.ts new file mode 100644 index 00000000..a2b9efb5 --- /dev/null +++ b/test/reporting/inventory.ts @@ -0,0 +1,190 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import ts from 'typescript'; + +export type Platform = 'ios' | 'android'; +export type Backend = 'local' | 'regtest' | 'mainnet'; +export type Disposition = 'required' | 'gap' | 'inapplicable' | 'manual'; +export type Assignment = { + backend: Backend; + pack: string; + disposition: Disposition; + fixtures: string[]; + reason?: string; + reference?: string; +}; +export type InventoryCase = { + id: string; + file: string; + kind: 'test' | 'fixture' | 'utility'; + platforms: Record; +}; +export type Inventory = { schemaVersion: 1; cases: InventoryCase[] }; +export type DiscoveredCase = { id: string; file: string; skipped: boolean }; + +export function caseId(title: string): string { + const match = /^@([a-z][a-z0-9_]*)(?=[\s,]|$)/.exec(title); + if (!match) throw new Error(`Test title needs a stable leading case ID: ${title}`); + return match[1]; +} + +function uncast(node: ts.Expression): ts.Expression { + return ts.isAsExpression(node) || ts.isParenthesizedExpression(node) + ? uncast(node.expression) + : node; +} + +// Only resolve finite source literals; never import specs or execute their hooks during discovery. +function finiteValues(expr: ts.Expression, owner: ts.Node, source: ts.SourceFile): string[] { + expr = uncast(expr); + if (ts.isStringLiteral(expr)) return [expr.text]; + if (ts.isIdentifier(expr)) { + for (let node: ts.Node | undefined = owner.parent; node; node = node.parent) { + if (!ts.isForOfStatement(node) || !ts.isVariableDeclarationList(node.initializer)) continue; + if (node.initializer.declarations[0].name.getText(source) !== expr.text) continue; + const values = uncast(node.expression); + if (ts.isArrayLiteralExpression(values)) { + return values.elements.flatMap((value) => + finiteValues(value as ts.Expression, node, source) + ); + } + } + } + if (ts.isPropertyAccessExpression(expr) && ts.isIdentifier(expr.expression)) { + const objectName = expr.expression.text; + const propertyName = expr.name.text; + for (let node: ts.Node | undefined = owner.parent; node; node = node.parent) { + if (!ts.isFunctionDeclaration(node) || !node.name) continue; + const index = node.parameters.findIndex((p) => p.name.getText(source) === objectName); + if (index < 0) continue; + const values: string[] = []; + const functionName = node.name.text; + function visit(call: ts.Node) { + if (ts.isCallExpression(call) && call.expression.getText(source) === functionName) { + const arg = call.arguments[index]; + if (!arg || !ts.isObjectLiteralExpression(arg)) { + throw new Error(`Non-literal case factory argument: ${call.getText(source)}`); + } + const property = arg.properties.find( + (p) => ts.isPropertyAssignment(p) && p.name.getText(source) === propertyName + ); + if (!property || !ts.isPropertyAssignment(property)) { + throw new Error(`Missing ${propertyName} in case factory ${functionName}`); + } + values.push(...finiteValues(property.initializer, call, source)); + } + ts.forEachChild(call, visit); + } + visit(source); + if (values.length) return values; + } + } + throw new Error(`Cannot discover dynamic case ID: ${expr.getText(source)}`); +} + +function titleIds(title: ts.Expression, call: ts.Node, source: ts.SourceFile): string[] { + if (ts.isStringLiteralLike(title)) return [caseId(title.text)]; + if (!ts.isTemplateExpression(title)) { + throw new Error(`Cannot discover test title: ${title.getText(source)}`); + } + let prefixes = [title.head.text]; + for (const span of title.templateSpans) { + if (prefixes.every((s) => /^@[a-z][a-z0-9_]*[\s,]/.test(s))) break; + const values = finiteValues(span.expression, call, source); + prefixes = prefixes.flatMap((s) => values.map((value) => s + value + span.literal.text)); + } + return prefixes.map(caseId); +} + +function filesIn(directory: string): string[] { + return fs.readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const file = path.join(directory, entry.name); + return entry.isDirectory() ? filesIn(file) : entry.name.endsWith('.ts') ? [file] : []; + }); +} + +export function discoverCases(root: string): DiscoveredCase[] { + const cases: DiscoveredCase[] = []; + for (const absolute of filesIn(path.join(root, 'test/specs')).sort()) { + const file = path.relative(root, absolute).split(path.sep).join('/'); + const source = ts.createSourceFile( + file, + fs.readFileSync(absolute, 'utf8'), + ts.ScriptTarget.Latest, + true + ); + function visit(node: ts.Node) { + if (ts.isCallExpression(node)) { + const name = node.expression.getText(source); + if (/^(?:ciIt|it|test|xit)(?:\.(?:skip|only))?$/.test(name)) { + if (name.endsWith('.only')) throw new Error(`Focused test is forbidden: ${file}`); + if (!node.arguments[0]) throw new Error(`Missing test title: ${file}`); + let skipped = name.endsWith('.skip') || name === 'xit'; + for (let parent: ts.Node | undefined = node.parent; parent; parent = parent.parent) { + if (ts.isCallExpression(parent)) { + const parentName = parent.expression.getText(source); + if (parentName === 'describe.only') + throw new Error(`Focused suite is forbidden: ${file}`); + if (parentName === 'describe.skip' || parentName === 'xdescribe') skipped = true; + } + } + for (const id of titleIds(node.arguments[0], node, source)) + cases.push({ id, file, skipped }); + } + } + ts.forEachChild(node, visit); + } + visit(source); + } + const seen = new Set(); + for (const item of cases) { + if (seen.has(item.id)) throw new Error(`Duplicate source case ID: ${item.id}`); + seen.add(item.id); + } + return cases; +} + +export function loadInventory(root: string): Inventory { + return JSON.parse(fs.readFileSync(path.join(root, 'test/inventory.json'), 'utf8')); +} + +export function validateInventory(root: string, inventory = loadInventory(root)): DiscoveredCase[] { + if (inventory.schemaVersion !== 1 || !Array.isArray(inventory.cases) || !inventory.cases.length) { + throw new Error('Invalid or empty inventory'); + } + const discovered = discoverCases(root); + const byId = new Map(discovered.map((item) => [item.id, item])); + const seen = new Set(); + for (const item of inventory.cases) { + const source = byId.get(item.id); + if (seen.has(item.id)) throw new Error(`Duplicate inventory case: ${item.id}`); + seen.add(item.id); + if (!source || source.file !== item.file) + throw new Error(`Stale inventory case/file: ${item.id}`); + if (!['test', 'fixture', 'utility'].includes(item.kind)) + throw new Error(`Invalid kind: ${item.id}`); + for (const platform of ['ios', 'android'] as const) { + const entry = item.platforms?.[platform]; + if ( + !entry || + !['local', 'regtest', 'mainnet'].includes(entry.backend) || + !entry.pack || + !['required', 'gap', 'inapplicable', 'manual'].includes(entry.disposition) || + !Array.isArray(entry.fixtures) || + !entry.fixtures.every((v) => typeof v === 'string') + ) { + throw new Error(`Invalid ${platform} assignment: ${item.id}`); + } + if (entry.disposition !== 'required' && (!entry.reason || !entry.reference)) { + throw new Error(`Unexplained ${platform} disposition: ${item.id}`); + } + if (source.skipped && entry.disposition === 'required') { + throw new Error(`Required case is statically skipped: ${platform}/${item.id}`); + } + } + } + const missing = discovered.filter((item) => !seen.has(item.id)); + if (missing.length) + throw new Error(`Uninventoried cases: ${missing.map((item) => item.id).join(', ')}`); + return discovered; +} diff --git a/test/reporting/report.ts b/test/reporting/report.ts new file mode 100644 index 00000000..72a2a63e --- /dev/null +++ b/test/reporting/report.ts @@ -0,0 +1,303 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import type { Backend, InventoryCase, Platform } from './inventory.ts'; + +export type Identity = { + runId: string; + platform: Platform; + backend: Backend; + appSha: string; + e2eSha: string; + artifactDigest: string; + backendRevision: string; + variant: string; +}; +export type Plan = { + schemaVersion: 1; + id: string; + identity: Identity; + inventoryDigest: string; + pack: string; + maxAttempts: number; + attemptTimeoutMinutes?: number; + cases: InventoryCase[]; + expected: string[]; +}; +export type Event = { + id: string; + status: 'passed' | 'failed' | 'skipped'; + durationMs: number; +}; +export type WorkerReport = { + schemaVersion: 1; + planId: string; + attempt: number; + cid: string; + startedAt: string; + finishedAt?: string; + failures: number; + errors: string[]; + events: Event[]; +}; + +export function digest(value: string): string { + return crypto.createHash('sha256').update(value).digest('hex'); +} + +export function writeJson(file: string, value: unknown): void { + fs.mkdirSync(path.dirname(file), { recursive: true }); + const temporary = `${file}.${process.pid}.tmp`; + fs.writeFileSync(temporary, JSON.stringify(value, null, 2) + '\n'); + fs.renameSync(temporary, file); +} + +export function createPlan( + cases: InventoryCase[], + identity: Identity, + inventoryDigest: string, + pack: string, + maxAttempts = 1, + attemptTimeoutMinutes?: number +): Plan { + if ( + !['ios', 'android'].includes(identity.platform) || + !['local', 'regtest', 'mainnet'].includes(identity.backend) || + Object.values(identity).some((value) => typeof value !== 'string' || !value.trim()) || + !/^[a-f0-9]{40}$/.test(identity.appSha) || + !/^[a-f0-9]{40}$/.test(identity.e2eSha) || + !/^[a-f0-9]{64}$/.test(identity.artifactDigest) || + !/^[a-f0-9]{64}$/.test(inventoryDigest) || + !Number.isInteger(maxAttempts) || + maxAttempts < 1 || + maxAttempts > 3 || + (attemptTimeoutMinutes !== undefined && + (!Number.isInteger(attemptTimeoutMinutes) || + attemptTimeoutMinutes < 1 || + attemptTimeoutMinutes > 110)) + ) { + throw new Error('Invalid or incomplete run identity/attempt policy'); + } + const selected = cases.filter((item) => { + const assignment = item.platforms[identity.platform]; + return ( + assignment.backend === identity.backend && + (pack === 'selected' || pack === 'nightly' || assignment.pack === pack) + ); + }); + const expected = selected + .filter((item) => ['required', 'gap'].includes(item.platforms[identity.platform].disposition)) + .map((item) => item.id) + .sort(); + if (!expected.length) + throw new Error(`No expected cases for ${identity.platform}/${identity.backend}/${pack}`); + if (new Set(expected).size !== expected.length) throw new Error('Duplicate expected case ID'); + const data = { + schemaVersion: 1 as const, + identity, + inventoryDigest, + pack, + maxAttempts, + ...(attemptTimeoutMinutes === undefined ? {} : { attemptTimeoutMinutes }), + cases: selected, + expected, + }; + return { ...data, id: digest(JSON.stringify(data)) }; +} + +export function validatePlan(plan: Plan): void { + const rebuilt = createPlan( + plan.cases, + plan.identity, + plan.inventoryDigest, + plan.pack, + plan.maxAttempts, + plan.attemptTimeoutMinutes + ); + if ( + rebuilt.id !== plan.id || + JSON.stringify(rebuilt.expected) !== JSON.stringify(plan.expected) + ) { + throw new Error('Run plan changed or is invalid'); + } +} + +export function reconcile(plan: Plan, reports: WorkerReport[]) { + validatePlan(plan); + const problems: string[] = []; + const expected = new Set(plan.expected); + const observations = new Map(); + const workers = new Set(); + const executions = new Set(); + let firstFailure = false; + const finalAttempt = Math.max(1, ...reports.map((r) => r.attempt)); + if (!reports.length) + problems.push('No worker reports: tests never started or reports are missing'); + for (const report of reports) { + if ( + report.schemaVersion !== 1 || + report.planId !== plan.id || + !Number.isInteger(report.attempt) || + report.attempt < 1 || + report.attempt > plan.maxAttempts || + !report.cid || + !Array.isArray(report.events) || + !Array.isArray(report.errors) || + !Number.isInteger(report.failures) || + report.failures < 0 + ) { + problems.push('Invalid worker report or mismatched run/platform/backend/source identity'); + continue; + } + const worker = `${report.attempt}/${report.cid}`; + if (workers.has(worker)) problems.push(`Duplicate worker report: ${worker}`); + workers.add(worker); + if (!report.finishedAt && report.attempt === finalAttempt) + problems.push(`Interrupted worker: ${worker}`); + firstFailure ||= report.failures > 0 || report.errors.length > 0; + if (report.attempt === finalAttempt && (report.failures || report.errors.length)) { + problems.push(`Final attempt failed: ${worker} (${report.errors.join('; ')})`); + } + for (const event of report.events) { + if ( + !expected.has(event.id) || + !['passed', 'failed', 'skipped'].includes(event.status) || + !Number.isFinite(event.durationMs) || + event.durationMs < 0 + ) { + problems.push(`Unexpected or invalid test event: ${event.id}`); + continue; + } + const execution = `${report.attempt}/${event.id}`; + if (executions.has(execution)) problems.push(`Duplicate case execution: ${execution}`); + executions.add(execution); + const history = observations.get(event.id) ?? []; + history.push({ + attempt: report.attempt, + event, + verified: + !!report.finishedAt && + report.errors.length === 0 && + report.failures <= report.events.filter((e) => e.status === 'failed').length, + }); + observations.set(event.id, history); + } + } + const results = plan.expected.map((id) => { + const history = (observations.get(id) ?? []).sort((a, b) => a.attempt - b.attempt); + let status = 'missing'; + let previouslyPassed = false; + for (const { attempt, event, verified } of history) { + firstFailure ||= event.status === 'failed'; + if (event.status === 'skipped') { + if (!previouslyPassed) { + problems.push(`Unexpected skip: ${id} (attempt ${attempt})`); + status = 'unexpected-skip'; + } + } else { + status = event.status === 'passed' && !verified ? 'unverified' : event.status; + previouslyPassed = status === 'passed'; + } + } + if (status !== 'passed') problems.push(`${id}: ${status}`); + return { + id, + status, + firstAttempt: history.find((h) => h.attempt === 1)?.event.status ?? 'missing', + history: history.map(({ attempt, event }) => ({ attempt, event })), + }; + }); + const dispositions = plan.cases.filter( + (item) => item.platforms[plan.identity.platform].disposition !== 'required' + ); + for (const item of dispositions) { + if (item.platforms[plan.identity.platform].disposition === 'gap') + problems.push(`Unresolved coverage gap: ${item.id}`); + } + return { + schemaVersion: 1, + planId: plan.id, + identity: plan.identity, + pack: plan.pack, + status: problems.length ? 'incomplete' : 'passed', + firstFailure, + expectedCount: plan.expected.length, + passedCount: results.filter((r) => r.status === 'passed').length, + durationMs: reports + .flatMap((r) => r.events ?? []) + .reduce((sum, e) => sum + (e.durationMs || 0), 0), + problems, + results, + dispositions, + }; +} + +export function markdownReport(result: ReturnType): string { + return [ + `## E2E ${result.pack}: ${result.status}`, + '', + `${result.identity.platform} / ${result.identity.backend} — ${result.passedCount}/${result.expectedCount} expected cases passed.`, + `First-attempt failure observed: ${result.firstFailure ? 'yes' : 'no'}.`, + '', + `Run: \`${result.identity.runId}\`; app: \`${result.identity.appSha}\`; tests: \`${result.identity.e2eSha}\`.`, + '', + '| Case | Result | First attempt |', + '| --- | --- | --- |', + ...result.results.map((r) => `| ${r.id} | ${r.status} | ${r.firstAttempt} |`), + '', + ...result.problems.map((p) => `- ${p}`), + '', + ...result.dispositions.map((item) => { + const entry = item.platforms[result.identity.platform]; + return `- ${item.id}: **${entry.disposition}** — ${entry.reason} (${entry.reference})`; + }), + '', + ].join('\n'); +} + +export type AttemptReceipt = { + attempt: number; + startedAt: string; + finishedAt?: string; + exitCode?: number; +}; + +export function reconcileAttempts(plan: Plan, reports: WorkerReport[], attempts: AttemptReceipt[]) { + validatePlan(plan); + const problems: string[] = []; + const perAttempt = attempts.map((attempt, index) => { + if (attempt.attempt !== index + 1 || attempt.attempt > plan.maxAttempts) + problems.push('Invalid, missing or out-of-order attempt receipt'); + const coverage = reconcile( + plan, + reports.filter((r) => r.attempt <= attempt.attempt) + ); + const passed = + attempt.exitCode === 0 && + !!attempt.finishedAt && + reports.some((r) => r.attempt === attempt.attempt) && + coverage.status === 'passed'; + return { ...attempt, passed, problems: coverage.problems, coverage }; + }); + if (!perAttempt.length) problems.push('Runner never started'); + if (perAttempt.slice(0, -1).some((a) => a.passed)) + problems.push('Attempt executed after a successful attempt'); + if (reports.some((r) => !attempts.some((a) => a.attempt === r.attempt))) + problems.push('Worker report without matching attempt receipt'); + const last = perAttempt.at(-1); + if (!last?.passed) + problems.push('Final attempt did not complete successfully with all selected cases passed'); + const fallback = reconcile(plan, []); + const result = last?.coverage ?? fallback; + return { + ...result, + status: problems.length ? 'incomplete' : 'passed', + firstFailure: perAttempt.length > 0 && !perAttempt[0].passed, + attemptsUsed: attempts.length, + maxAttempts: plan.maxAttempts, + attempts: perAttempt.map(({ coverage, ...receipt }) => receipt), + problems: [...problems, ...(last?.passed ? [] : result.problems)], + results: result.results, + durationMs: result.durationMs, + }; +} diff --git a/test/reporting/reporter.ts b/test/reporting/reporter.ts new file mode 100644 index 00000000..a7298db3 --- /dev/null +++ b/test/reporting/reporter.ts @@ -0,0 +1,100 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import WDIOReporter, { type HookStats, type RunnerStats, type TestStats } from '@wdio/reporter'; +import { assertExecutionIdentity } from './identity.ts'; +import { caseId } from './inventory.ts'; +import { digest, validatePlan, writeJson, type Plan, type WorkerReport } from './report.ts'; + +export default class InventoryReporter extends WDIOReporter { + private plan: Plan; + private report?: WorkerReport; + private file?: string; + + constructor(options: Record) { + super({ ...options, stdout: false }); + this.plan = JSON.parse(fs.readFileSync(process.env.E2E_PLAN!, 'utf8')); + validatePlan(this.plan); + assertExecutionIdentity(this.plan); + if ( + this.plan.identity.platform !== process.env.PLATFORM || + this.plan.identity.backend !== (process.env.BACKEND || 'local') || + this.plan.inventoryDigest !== digest(fs.readFileSync('test/inventory.json', 'utf8')) + ) { + throw new Error('Run plan does not match the executing platform/backend/inventory'); + } + } + + onRunnerStart(runner: RunnerStats) { + const attempt = Number(process.env.ATTEMPT || '1'); + if (!Number.isInteger(attempt) || attempt < 1 || attempt > this.plan.maxAttempts) { + throw new Error('Attempt is outside the declared run policy'); + } + if (!/^[\w-]+$/.test(runner.cid)) throw new Error('Invalid worker ID'); + this.file = path.join( + path.dirname(process.env.E2E_PLAN!), + 'workers', + `attempt-${attempt}-${runner.cid}.json` + ); + if (fs.existsSync(this.file)) + throw new Error('Refusing to overwrite an existing worker report'); + this.report = { + schemaVersion: 1, + planId: this.plan.id, + attempt, + cid: runner.cid, + startedAt: new Date().toISOString(), + failures: 0, + errors: [], + events: [], + }; + this.flush(); + } + + private record(test: TestStats, status: 'passed' | 'failed' | 'skipped') { + if (!this.report) throw new Error('Test event arrived before runner start'); + try { + this.report.events.push({ id: caseId(test.title), status, durationMs: test.duration || 0 }); + } catch (error) { + this.report.errors.push(String(error)); + } + this.flush(); + } + + onTestPass(test: TestStats) { + this.record(test, 'passed'); + } + onTestFail(test: TestStats) { + this.record(test, 'failed'); + } + onTestSkip(test: TestStats) { + this.record(test, 'skipped'); + } + + onTestPending(test: TestStats) { + this.record(test, 'skipped'); + } + + onTestRetry(test: TestStats) { + this.report?.errors.push(`Undeclared framework retry: ${test.title}`); + this.flush(); + } + + onHookEnd(hook: HookStats) { + if (hook.error || hook.errors?.length || hook.state === 'failed') { + this.report?.errors.push(`Hook failed: ${hook.title}`); + this.flush(); + } + } + + onRunnerEnd(runner: RunnerStats) { + if (!this.report) throw new Error('Runner ended without starting'); + this.report.failures = runner.failures ?? 0; + if (runner.error) this.report.errors.push(runner.error); + this.report.finishedAt = new Date().toISOString(); + this.flush(); + } + + private flush() { + if (this.file && this.report) writeJson(this.file, this.report); + } +} diff --git a/test/reporting/specs.ts b/test/reporting/specs.ts new file mode 100644 index 00000000..f68a029e --- /dev/null +++ b/test/reporting/specs.ts @@ -0,0 +1,10 @@ +import fs from 'node:fs'; +import { validatePlan, type Plan } from './report.ts'; +import { assertExecutionIdentity } from './identity.ts'; + +export function selectedSpecs(file: string): string[] { + const plan: Plan = JSON.parse(fs.readFileSync(file, 'utf8')); + validatePlan(plan); + assertExecutionIdentity(plan); + return [...new Set(plan.cases.filter((c) => plan.expected.includes(c.id)).map((c) => c.file))]; +} diff --git a/test/specs/migration.e2e.ts b/test/specs/migration.e2e.ts index 07dd6780..0b63034f 100644 --- a/test/specs/migration.e2e.ts +++ b/test/specs/migration.e2e.ts @@ -22,6 +22,7 @@ import { tap, typeText, waitForSetupWalletScreenFinish, + waitForWalletHome, } from '../helpers/actions'; import { expectMigrationBalances, @@ -356,6 +357,25 @@ async function setupLegacyWallet( await restoreRnWallet(IOS_RN_MNEMONIC); } + // Prove the source wallet is the funded fixture before installing the target. + // A dropped character in RN's passphrase field otherwise looks like lost + // funds after migration, even though the wrong wallet was restored first. + // The iOS accessibility tree exposes the amount on the touchable parent. + // The nested MoneyText used by Android is hidden on RN iOS v1.1.6. + const rnBalance = elementById('TotalBalance'); + await rnBalance.waitForDisplayed(); + await browser.waitUntil( + async () => { + const label = await rnBalance.getAttribute('label'); + return label.replace(/\s/g, '') === String(IOS_RN_BALANCE); + }, + { + timeout: 180_000, + interval: 1_000, + timeoutMsg: 'Restored RN wallet balance does not match the prepared migration fixture', + } + ); + console.info('=== iOS: RN wallet restored ==='); return { mnemonic: IOS_RN_MNEMONIC, balance: IOS_RN_BALANCE }; } @@ -665,8 +685,22 @@ async function restoreRnWallet( // Terms of service await elementById('Continue').waitForDisplayed(); - await tap('Check1'); - await tap('Check2'); + if (driver.isIOS) { + const next = elementById('Continue'); + // Legacy RN labels Continue as enabled only when both boxes are selected. + // Alternating toggles visits all four combinations if an initial tap was missed. + for (const toggle of ['Check1', 'Check2', 'Check1', 'Check2', 'Check1', 'Check2']) { + if ((await next.getAttribute('label')) === 'enabled') break; + await tap(toggle); + } + await browser.waitUntil(async () => (await next.getAttribute('label')) === 'enabled', { + timeout: 5_000, + timeoutMsg: 'Both legacy RN consent boxes must be selected', + }); + } else { + await tap('Check1'); + await tap('Check2'); + } await tap('Continue'); // Skip intro and go to restore @@ -685,7 +719,17 @@ async function restoreRnWallet( // Passphrase if provided if (passphrase) { await tap('AdvancedButton'); - await typeText('PassphraseInput', passphrase); + if (driver.isIOS) { + // RN can drop characters from XCUITest's simulated typing. Paste the + // complete value, then verify it before deriving/restoring the wallet. + await pasteIOSText('PassphraseInput', passphrase); + const entered = await elementById('PassphraseInput').getAttribute('value'); + if (entered !== passphrase) { + throw new Error('RN passphrase input does not match the migration fixture'); + } + } else { + await typeText('PassphraseInput', passphrase); + } await confirmInputOnKeyboard(); } @@ -845,7 +889,9 @@ async function sendRnOnchain( await mineBlocks(1); await electrumClient?.waitForSync(); await sleep(1000); - await dismissSheetRN(); + // Close already dismissed SendSuccess. Another Android Back from Home + // backgrounds the RN app, leaving the sweep fixture unable to read its balance. + await elementById('TotalBalance').waitForDisplayed(); console.info(`→ Sent ${sats} sats`); } @@ -1104,6 +1150,8 @@ async function verifyMigration( ): Promise { console.info('=== Verifying migration ==='); + if (driver.isIOS) await waitForWalletHome({ timeout: 180_000 }); + await expectMigrationBalances({ total: expectedBalance, spending: 0, diff --git a/test/specs/receive-ln-payments.e2e.ts b/test/specs/receive-ln-payments.e2e.ts index cf5aa5d2..7677ab13 100644 --- a/test/specs/receive-ln-payments.e2e.ts +++ b/test/specs/receive-ln-payments.e2e.ts @@ -43,7 +43,7 @@ describe('Receive LN payments (utility)', () => { await sleep(3000); }); - it(`should receive ${PAYMENT_COUNT} Lightning payments`, async () => { + it(`@utility_receive_ln_1 - should receive ${PAYMENT_COUNT} Lightning payments`, async () => { let succeeded = 0; let failed = 0; diff --git a/test/specs/transfer.e2e.ts b/test/specs/transfer.e2e.ts index 5f2e6293..c8341b5e 100644 --- a/test/specs/transfer.e2e.ts +++ b/test/specs/transfer.e2e.ts @@ -469,16 +469,29 @@ describe('@transfer - Transfer', () => { // check transfer card // await elementById('Suggestion-lightning_setting_up').waitForDisplayed(); - const totalBalance = await elementByIdWithin('TotalBalance-primary', 'MoneyText'); - const totalAmtAfterChannelOpen = await totalBalance.getText(); - await expect(totalBalance).not.toHaveText('100 000'); - await sleep(2500); - - // check activity - await elementById('ActivityShort-0').waitForDisplayed(); - await expectTextWithin('ActivityShort-0', 'Transfer'); - await elementById('ActivityShort-1').waitForDisplayed(); - await expectTextWithin('ActivityShort-1', 'Received'); + // QuickPay is shown once per wallet and can cover Home anywhere between + // channel opening and the last activity assertion. + let dismissedEarlyQuickPay = driver.isIOS && (await tryDismissQuickPayIntroIfVisible()); + const readOpeningActivity = async () => { + const totalBalance = await elementByIdWithin('TotalBalance-primary', 'MoneyText'); + await expect(totalBalance).not.toHaveText('100 000'); + const totalAmtAfterChannelOpen = await totalBalance.getText(); + await sleep(2500); + await elementById('ActivityShort-0').waitForDisplayed(); + await expectTextWithin('ActivityShort-0', 'Transfer'); + await elementById('ActivityShort-1').waitForDisplayed(); + await expectTextWithin('ActivityShort-1', 'Received'); + return { totalBalance, totalAmtAfterChannelOpen }; + }; + const { totalBalance, totalAmtAfterChannelOpen } = await readOpeningActivity().catch( + async (error) => { + if (!driver.isIOS || dismissedEarlyQuickPay || !(await tryDismissQuickPayIntroIfVisible())) { + throw error; + } + dismissedEarlyQuickPay = true; + return readOpeningActivity(); + } + ); await swipeFullScreen('down'); await swipeFullScreen('down'); @@ -488,7 +501,7 @@ describe('@transfer - Transfer', () => { await sleep(1000); await dismissBackgroundPaymentsTimedSheet({ triggerTimedSheet: true }); - await dismissQuickPayIntro({ triggerTimedSheet: true }); + if (!dismissedEarlyQuickPay) await dismissQuickPayIntro({ triggerTimedSheet: true }); await expectNoTextWithin('ActivitySpending', '0'); await waitForActiveChannel(lnd, ldkNodeId); diff --git a/test/unit/ci-retry.test.ts b/test/unit/ci-retry.test.ts new file mode 100644 index 00000000..150365f3 --- /dev/null +++ b/test/unit/ci-retry.test.ts @@ -0,0 +1,300 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import fs from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { + createPlan, + validatePlan, + reconcileAttempts, + type Identity, + type WorkerReport, + type AttemptReceipt, +} from '../reporting/report.ts'; +import { loadInventory } from '../reporting/inventory.ts'; +const cases = loadInventory(process.cwd()).cases.filter((c) => ['send_1', 'send_2'].includes(c.id)); +const identity: Identity = { + runId: 'retry', + platform: 'ios', + backend: 'local', + appSha: 'a'.repeat(40), + e2eSha: 'b'.repeat(40), + artifactDigest: 'c'.repeat(64), + backendRevision: 'local', + variant: 'local', +}; +const plan = createPlan(cases, identity, 'd'.repeat(64), 'selected', 3); +function receipt(attempt: number, exitCode = 0): AttemptReceipt { + return { + attempt, + exitCode, + startedAt: '2026-11-03T00:00:00Z', + finishedAt: '2026-11-03T00:01:00Z', + }; +} +function worker(attempt: number, failed = false): WorkerReport { + return { + schemaVersion: 1, + planId: plan.id, + attempt, + cid: '0-0', + startedAt: '2026-11-03T00:00:00Z', + finishedAt: '2026-11-03T00:01:00Z', + failures: failed ? 1 : 0, + errors: [], + events: plan.expected.map((id) => ({ + id, + status: failed ? 'failed' : 'passed', + durationMs: 10, + })), + }; +} +test('first failure remains after third-attempt recovery', () => { + const s = reconcileAttempts( + plan, + [worker(1, true), worker(2, true), worker(3)], + [receipt(1, 1), receipt(2, 1), receipt(3)] + ); + assert.equal(s.status, 'passed'); + assert.equal(s.firstFailure, true); + assert.equal(s.attemptsUsed, 3); + assert.equal(s.results[0].firstAttempt, 'failed'); + assert.equal(s.results[0].history.length, 3); +}); +test('startup failure without workers is retained after recovery', () => { + const s = reconcileAttempts(plan, [worker(2)], [receipt(1, 1), receipt(2)]); + assert.equal(s.status, 'passed'); + assert.equal(s.firstFailure, true); + assert.equal(s.results[0].firstAttempt, 'missing'); +}); +test('interrupted worker may recover only with a complete later attempt', () => { + const w = worker(1); + delete w.finishedAt; + const s = reconcileAttempts(plan, [w, worker(2)], [receipt(1, 1), receipt(2)]); + assert.equal(s.status, 'passed'); + assert.equal(s.firstFailure, true); + assert.ok(s.attempts[0].problems.some((p) => p.includes('Interrupted'))); +}); +for (const [name, reports, attempts] of [ + ['no runner', [], []], + ['runner failure despite passing tests', [worker(1)], [receipt(1, 1)]], + ['zero matches', [], [receipt(1)]], + ['attempt after pass', [worker(1), worker(2)], [receipt(1), receipt(2)]], + ['missing attempt', [worker(2)], [receipt(2)]], + ['fourth attempt', [worker(4)], [receipt(4)]], + ['interrupted process', [worker(1)], [{ ...receipt(1), finishedAt: undefined }]], + ['foreign report', [{ ...worker(1), planId: 'other' }], [receipt(1)]], + [ + 'partial retry', + [worker(1, true), { ...worker(2), events: worker(2).events.slice(0, 1) }], + [receipt(1, 1), receipt(2)], + ], +] as [string, WorkerReport[], AttemptReceipt[]][]) + test(name + ' fails closed', () => + assert.equal(reconcileAttempts(plan, reports, attempts).status, 'incomplete') + ); + +test('failed-only retries accumulate passes and preserve the original failure', () => { + const first = worker(1, true); + first.events[0].status = 'passed'; + const second = worker(2, true); + second.events[0].status = 'skipped'; + const third = worker(3); + third.events[0].status = 'skipped'; + const s = reconcileAttempts( + plan, + [first, second, third], + [receipt(1, 1), receipt(2, 1), receipt(3)] + ); + assert.equal(s.status, 'passed'); + assert.equal(s.firstFailure, true); + assert.deepEqual( + s.results[0].history.map((h) => h.event.status), + ['passed', 'skipped', 'skipped'] + ); + assert.deepEqual( + s.results[1].history.map((h) => h.event.status), + ['failed', 'failed', 'passed'] + ); + assert.equal(s.durationMs, 60); +}); + +test('passes from a failed hook or interrupted worker cannot justify a skipped retry', () => { + for (const first of [ + { ...worker(1), errors: ['Hook failed: afterEach'] }, + { ...worker(1), finishedAt: undefined }, + { ...worker(1), failures: 1 }, + ]) { + const retry = worker(2); + retry.events[0].status = 'skipped'; + assert.equal( + reconcileAttempts(plan, [first, retry], [receipt(1, 1), receipt(2)]).status, + 'incomplete' + ); + } +}); + +for (const failures of [0, 1, 2, 3, 'timeout', 'startup', 'hook'] as const) + test(`visible CI attempts retry only failures: ${failures}`, (t) => { + fs.mkdirSync('artifacts', { recursive: true }); + const root = fs.mkdtempSync(path.resolve('artifacts/retry-fixture-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + fs.mkdirSync(path.join(root, 'test/specs'), { recursive: true }); + fs.mkdirSync(path.join(root, 'aut/bitkit.app'), { recursive: true }); + fs.writeFileSync( + path.join(root, 'test/inventory.json'), + JSON.stringify({ schemaVersion: 1, cases }) + ); + fs.writeFileSync( + path.join(root, 'test/specs/send.e2e.ts'), + cases.map((c) => `ciIt('@${c.id} - fixture',async()=>{});`).join('\n') + ); + fs.writeFileSync(path.join(root, 'aut/bitkit.app/app'), 'fixture'); + fs.writeFileSync( + path.join(root, 'aut/e2e-build.json'), + JSON.stringify({ appSha: identity.appSha, platform: 'ios', backend: 'local' }) + ); + fs.writeFileSync( + path.join(root, 'ci_run_ios.sh'), + '#!/usr/bin/env bash\nexec node --experimental-strip-types stub.cjs\n', + { mode: 0o755 } + ); + fs.writeFileSync( + path.join(root, 'stub.cjs'), + String.raw` +const fs = require('fs'); +const Mocha = require(${JSON.stringify(path.resolve('node_modules/mocha'))}); +const {ciIt} = require(${JSON.stringify(path.resolve('test/helpers/suite.ts'))}); +const p = JSON.parse(fs.readFileSync(process.env.E2E_PLAN, 'utf8')); +const a = Number(process.env.ATTEMPT); +const mode = ${JSON.stringify(failures)}; +fs.mkdirSync('artifacts/attempt-' + a, {recursive: true}); +fs.writeFileSync('artifacts/attempt-' + a + '/evidence.txt', 'attempt ' + a); +if (mode === 'startup' && a === 1) process.exit(1); +const mocha = new Mocha({reporter: 'dot'}); +mocha.suite.emit('pre-require', global, 'fixture', mocha); +for (const id of p.expected) { + ciIt('@' + id + ' - fixture', async () => { + fs.appendFileSync('bodies.jsonl', JSON.stringify({attempt: a, id}) + '\n'); + if (id === 'send_2' && typeof mode === 'number' && a <= mode) throw Error('controlled case failure'); + }); +} +if (mode === 'hook') afterEach(() => {if (a === 1) throw Error('controlled hook failure')}); +const report = {schemaVersion: 1, planId: p.id, attempt: a, cid: '0-0', startedAt: new Date().toISOString(), failures: 0, errors: [], events: []}; +const runner = mocha.run(count => { + report.finishedAt = new Date().toISOString(); + report.failures = count; + fs.mkdirSync('artifacts/reporting/workers', {recursive: true}); + fs.writeFileSync('artifacts/reporting/workers/attempt-' + a + '.json', JSON.stringify(report)); + process.exitCode = count ? 1 : 0; + if (mode === 'timeout') setInterval(() => {}, 1000); +}); +for (const [event, status] of [['pass', 'passed'], ['fail', 'failed'], ['pending', 'skipped']]) { + runner.on(event, item => { + if (item.type === 'hook') {report.errors.push('Hook failed: ' + item.title); return} + report.events.push({id: item.title.match(/^@(\w+)/)[1], status, durationMs: 1}); + }); +} +` + ); + const env = { + ...process.env, + CI: '1', + PLATFORM: 'ios', + BACKEND: 'local', + E2E_ATTEMPT_TIMEOUT_MINUTES: '90', + E2E_APP_SHA: identity.appSha, + E2E_RUN_ID: 'wrapper-fixture', + E2E_BACKEND_REVISION: 'local', + E2E_CASE_IDS: JSON.stringify(cases.map((c) => c.id)), + E2E_PLAN: 'artifacts/reporting/plan.json', + GITHUB_ENV: path.join(root, 'env'), + GITHUB_STEP_SUMMARY: path.join(root, 'summary.md'), + ATTEMPT: '1', + }; + const clock = path.join(root, 'fast-clock.cjs'); + fs.writeFileSync( + clock, + `const real=global.setTimeout;global.setTimeout=(fn,ms,...args)=>real(fn,ms===5400000?1000:ms,...args);` + ); + const invoke = (cmd: string) => + spawnSync( + process.execPath, + [ + ...(failures === 'timeout' ? ['--require', clock] : []), + '--experimental-strip-types', + path.resolve('scripts/ci-run.ts'), + cmd, + ], + { cwd: root, env, encoding: 'utf8' } + ); + const prep = invoke('prepare'); + assert.equal(prep.status, 0, prep.stderr); + Object.assign( + env, + Object.fromEntries( + fs + .readFileSync(path.join(root, 'env'), 'utf8') + .trim() + .split('\n') + .map((line) => line.split('=')) + ) + ); + const prepared = JSON.parse(fs.readFileSync(path.join(root, env.E2E_PLAN), 'utf8')); + assert.equal(prepared.attemptTimeoutMinutes, 90); + env.E2E_ATTEMPT_TIMEOUT_MINUTES = '1'; // Attempt uses the sealed plan. + // A leftover lock must not suppress current execution, even under this plan path. + fs.mkdirSync(path.join(root, 'artifacts/reporting/completed'), { recursive: true }); + fs.writeFileSync(path.join(root, 'artifacts/reporting/completed/lock-send_1'), '1'); + let last; + for (let attempt = 1; attempt <= 3; attempt++) { + env.ATTEMPT = String(attempt); + last = invoke('attempt'); + if (last.status === 0) break; // The workflow's visible-step condition. + } + assert.equal(last?.status, failures === 3 || failures === 'timeout' ? 1 : 0, last?.stderr); + if (failures === 'timeout') assert.match(last!.stderr, /exceeded 90 minutes/); + const final = invoke('report'); + assert.equal(final.status, last!.status, final.stderr); + const s = JSON.parse( + fs.readFileSync(path.join(root, 'artifacts/reporting/summary.json'), 'utf8') + ); + const attempts = + typeof failures === 'number' ? Math.min(failures + 1, 3) : failures === 'timeout' ? 3 : 2; + assert.equal(s.attemptsUsed, attempts); + assert.equal(s.firstFailure, failures !== 0); + const bodies = fs + .readFileSync(path.join(root, 'bodies.jsonl'), 'utf8') + .trim() + .split('\n') + .map((line) => JSON.parse(line)); + assert.equal(bodies.filter((b) => b.id === 'send_1').length, failures === 'hook' ? 2 : 1); + assert.equal( + bodies.filter((b) => b.id === 'send_2').length, + failures === 'timeout' || failures === 'startup' || failures === 'hook' ? 1 : attempts + ); + assert.equal( + fs.readFileSync(path.join(root, 'artifacts/attempt-1/evidence.txt'), 'utf8'), + 'attempt 1' + ); + if (failures === 1 || failures === 2) { + assert.deepEqual( + s.results[0].history.map((h: any) => h.event.status), + ['passed', ...Array(attempts - 1).fill('skipped')] + ); + assert.equal(s.results[1].firstAttempt, 'failed'); + assert.match( + fs.readFileSync(path.join(root, 'artifacts/reporting/attempt-1-runner.log'), 'utf8'), + /controlled case failure/ + ); + } + }); + +test('attempt budget is bounded, sealed and backwards compatible', () => { + validatePlan(plan); // Existing plans retain their default 40-minute budget. + const p = createPlan(cases, identity, 'd'.repeat(64), 'selected', 3, 110); + validatePlan(p); + assert.throws(() => validatePlan({ ...p, attemptTimeoutMinutes: 40 }), /Run plan changed/); + for (const n of [0, -1, 111, Infinity, NaN, 1.5]) + assert.throws(() => createPlan(cases, identity, 'd'.repeat(64), 'selected', 3, n), /Invalid/); +}); diff --git a/test/unit/inventory.test.ts b/test/unit/inventory.test.ts new file mode 100644 index 00000000..154ba2f0 --- /dev/null +++ b/test/unit/inventory.test.ts @@ -0,0 +1,111 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { + discoverCases, + loadInventory, + validateInventory, + type Inventory, +} from '../reporting/inventory.ts'; + +function fixture(t: test.TestContext, source: string) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'inventory-test-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + fs.mkdirSync(path.join(root, 'test/specs'), { recursive: true }); + fs.writeFileSync(path.join(root, 'test/specs/example.ts'), source); + return root; +} +function inventory(id = 'example_1'): Inventory { + const assignment = { + backend: 'regtest' as const, + pack: 'core', + disposition: 'required' as const, + fixtures: [], + }; + return { + schemaVersion: 1, + cases: [ + { + id, + file: 'test/specs/example.ts', + kind: 'test', + platforms: { ios: { ...assignment }, android: { ...assignment } }, + }, + ], + }; +} + +test('repository inventory covers every source case, including skipped and conditional cases', () => { + const root = process.cwd(); + const cases = validateInventory(root); + assert.equal(cases.length, loadInventory(root).cases.length); + assert.ok(cases.some((c) => c.id === 'migration_setup_sweep')); + assert.ok(cases.some((c) => c.id === 'hardware_wallet_3' && c.skipped)); +}); + +test('discovery does not execute imports, hooks or runtime conditions', (t) => { + const root = fixture( + t, + "import dangerous from 'missing-network-module'; before(() => dangerous()); if (process.env.EXTRA) { ciIt('@example_1 - hidden', async () => {}); }" + ); + assert.equal(validateInventory(root, inventory()).length, 1); +}); + +test('dynamic IDs expand from finite factory arguments and loop values', (t) => { + const root = fixture( + t, + ` + function suite(config) { describe(config.label, () => { ciIt(\`\${config.tag} - scenario\`, async () => {}); }); } + suite({label:'first',tag:'@first_1'}); suite({label:'second',tag:'@second_1'}); + for (const method of ['restore','upgrade'] as const) { ciIt(\`@native_\${method} - \${process.env.VERSION}\`, async () => {}); } + ` + ); + assert.deepEqual( + discoverCases(root).map((c) => c.id), + ['first_1', 'second_1', 'native_restore', 'native_upgrade'] + ); +}); + +for (const [name, source, pattern] of [ + ['untagged case', "it('untracked test', () => {})", /stable leading case ID/], + [ + 'unknown dynamic ID', + 'ciIt(`@case_${process.env.NAME} - test`, () => {})', + /Cannot discover dynamic/, + ], + [ + 'duplicate ID', + "ciIt('@example_1 - a', () => {}); ciIt('@example_1 - b', () => {})", + /Duplicate source/, + ], + ['focused test', "ciIt.only('@example_1 - a', () => {})", /Focused test/], + [ + 'focused suite', + "describe.only('a', () => { ciIt('@example_1 - a', () => {}); })", + /Focused suite/, + ], +] as const) + test(`rejects ${name}`, (t) => assert.throws(() => discoverCases(fixture(t, source)), pattern)); + +test('new source cases cannot be hidden by workflow selection', (t) => { + const root = fixture(t, "ciIt('@example_1 - a', () => {}); ciIt('@new_1 - b', () => {})"); + assert.throws(() => validateInventory(root, inventory()), /Uninventoried cases: new_1/); +}); +test('removed or renamed tests and stale file mappings fail', (t) => { + const root = fixture(t, "ciIt('@renamed_1 - a', () => {})"); + assert.throws(() => validateInventory(root, inventory()), /Stale inventory/); +}); +test('every case requires both platforms and an explained exception', (t) => { + const root = fixture(t, "ciIt('@example_1 - a', () => {})"); + const data = inventory(); + data.cases[0].platforms.ios.disposition = 'inapplicable'; + assert.throws(() => validateInventory(root, data), /Unexplained ios disposition/); + delete (data.cases[0].platforms as Partial<(typeof data.cases)[0]['platforms']>).ios; + assert.throws(() => validateInventory(root, data), /Invalid ios assignment/); +}); +test('skip does not silently remove a required case', (t) => { + const root = fixture(t, "describe.skip('suite',()=>{ciIt('@example_1 - a', () => {})})"); + assert.throws(() => validateInventory(root, inventory()), /Required case is statically skipped/); +}); diff --git a/test/unit/pinned-fixture-source.test.cjs b/test/unit/pinned-fixture-source.test.cjs new file mode 100644 index 00000000..c526e70d --- /dev/null +++ b/test/unit/pinned-fixture-source.test.cjs @@ -0,0 +1,48 @@ +const test = require('node:test'), + assert = require('node:assert/strict'), + fs = require('node:fs'), + os = require('node:os'), + path = require('node:path'), + { spawnSync, execFileSync } = require('node:child_process'), + Y = require('yaml'); +test('migration fixture bypasses branch resolution only for explicit commit pins', () => { + const w = Y.parse(fs.readFileSync('.github/workflows/migration-wallet-setup.yml', 'utf8')); + const steps = w.jobs['prepare-wallets'].steps; + assert.equal(w.on.workflow_call.inputs.diagnostics_retention_days.type, 'number'); + assert.equal(w.on.workflow_call.inputs.diagnostics_retention_days.default, 0); + assert.equal( + steps.find((s) => s.name === 'Upload preparation diagnostics').with['retention-days'], + '${{ inputs.diagnostics_retention_days }}' + ); + assert.equal(steps.find((s) => s.name === 'Upload migration env file').with['retention-days'], 1); + assert.equal(steps.find((s) => s.id === 'e2e-ref').if, "inputs.e2e_sha == ''"); + assert.equal( + steps.find((s) => s.name === 'Clone E2E tests').with.ref, + '${{ inputs.e2e_sha || steps.e2e-ref.outputs.ref }}' + ); + const validation = steps.find((s) => s.name === 'Validate pinned commit'); + for (const pin of ['branch', 'a'.repeat(39), 'a'.repeat(40) + '; echo bad']) + assert.notEqual( + spawnSync('bash', ['-e', '-c', validation.run], { env: { ...process.env, E2E_SHA: pin } }) + .status, + 0 + ); + const head = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(); + assert.equal( + spawnSync('bash', ['-e', '-c', validation.run], { env: { ...process.env, E2E_SHA: head } }) + .status, + 0 + ); + assert.equal( + spawnSync('bash', ['scripts/verify-e2e-sha.sh', 'checkout'], { + env: { ...process.env, E2E_SHA: head }, + }).status, + 0 + ); + assert.notEqual( + spawnSync('bash', ['scripts/verify-e2e-sha.sh', 'checkout'], { + env: { ...process.env, E2E_SHA: 'f'.repeat(40) }, + }).status, + 0 + ); +}); diff --git a/test/unit/reporter-events.test.ts b/test/unit/reporter-events.test.ts new file mode 100644 index 00000000..c80b1fa3 --- /dev/null +++ b/test/unit/reporter-events.test.ts @@ -0,0 +1,157 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { executionIdentity } from '../reporting/identity.ts'; +import InventoryReporter from '../reporting/reporter.ts'; +import { loadInventory } from '../reporting/inventory.ts'; +import { + createPlan, + digest, + reconcile, + writeJson, + type WorkerReport, +} from '../reporting/report.ts'; + +function fixture(t: test.TestContext) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'reporter-events-')); + const previous = { + E2E_PLAN: process.env.E2E_PLAN, + PLATFORM: process.env.PLATFORM, + BACKEND: process.env.BACKEND, + ATTEMPT: process.env.ATTEMPT, + E2E_RUN_ID: process.env.E2E_RUN_ID, + E2E_APP_SHA: process.env.E2E_APP_SHA, + E2E_ARTIFACT_DIGEST: process.env.E2E_ARTIFACT_DIGEST, + E2E_BACKEND_REVISION: process.env.E2E_BACKEND_REVISION, + E2E_VARIANT: process.env.E2E_VARIANT, + }; + t.after(() => { + for (const [key, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + fs.rmSync(directory, { recursive: true, force: true }); + }); + Object.assign(process.env, { + E2E_RUN_ID: 'reporter-integration', + E2E_APP_SHA: 'a'.repeat(40), + E2E_ARTIFACT_DIGEST: 'c'.repeat(64), + E2E_BACKEND_REVISION: 'stag0', + E2E_VARIANT: 'default', + }); + const plan = createPlan( + loadInventory(process.cwd()).cases, + executionIdentity('ios', 'local'), + digest(fs.readFileSync('test/inventory.json', 'utf8')), + 'local-money' + ); + process.env.E2E_PLAN = path.join(directory, 'plan.json'); + process.env.PLATFORM = 'ios'; + process.env.BACKEND = 'local'; + process.env.ATTEMPT = '1'; + writeJson(process.env.E2E_PLAN, plan); + const reporter = new InventoryReporter({ writeStream: { write: () => true } }); + reporter.emit('runner:start', { + cid: '0-0', + capabilities: { platformName: 'iOS' }, + config: {}, + specs: ['test/specs/send.e2e.ts'], + }); + const receipt = () => + JSON.parse( + fs.readFileSync(path.join(directory, 'workers/attempt-1-0-0.json'), 'utf8') + ) as WorkerReport; + const passAll = () => { + for (const id of plan.expected) { + const e = { uid: id, cid: '0-0', title: `@${id} - test`, fullTitle: `fixture @${id} - test` }; + reporter.emit('test:start', e); + reporter.emit('test:pass', e); + reporter.emit('test:end', e); + } + }; + return { reporter, receipt, plan, passAll }; +} + +test('actual WDIO reporter events produce a complete exact-case receipt', (t) => { + const f = fixture(t); + f.passAll(); + f.reporter.emit('runner:end', { failures: 0 }); + assert.equal(reconcile(f.plan, [f.receipt()]).status, 'passed'); +}); +test('Mocha pending events are reported as unexpected skips', (t) => { + const f = fixture(t); + const id = f.plan.expected[0]; + f.reporter.emit('test:pending', { + uid: id, + cid: '0-0', + title: `@${id} - skipped`, + fullTitle: `@${id} - skipped`, + state: 'pending', + }); + f.reporter.emit('runner:end', { failures: 0 }); + const result = reconcile(f.plan, [f.receipt()]); + assert.match(result.problems.join(' '), /Unexpected skip/); +}); +test('hook failure is recorded even if runner failure count is zero', (t) => { + const f = fixture(t); + f.passAll(); + f.reporter.emit('hook:start', { uid: 'after', cid: '0-0', title: 'after hook' }); + f.reporter.emit('hook:end', { uid: 'after', errors: [new Error('teardown failed')] }); + f.reporter.emit('runner:end', { failures: 0 }); + const result = reconcile(f.plan, [f.receipt()]); + assert.equal(result.status, 'incomplete'); + assert.match(result.problems.join(' '), /Hook failed/); +}); +test('interruption retains incremental case evidence without a successful finish', (t) => { + const f = fixture(t); + f.passAll(); + assert.equal(f.receipt().events.length, f.plan.expected.length); + assert.equal(reconcile(f.plan, [f.receipt()]).status, 'incomplete'); +}); +test('framework retries cannot silently turn one-attempt fixture green', (t) => { + const f = fixture(t); + const id = f.plan.expected[0]; + const event = { uid: id, cid: '0-0', title: `@${id} - retry` }; + f.reporter.emit('test:start', event); + f.reporter.emit('test:retry', { ...event, errors: [new Error('first attempt')] }); + f.passAll(); + f.reporter.emit('runner:end', { failures: 0 }); + assert.equal(reconcile(f.plan, [f.receipt()]).status, 'incomplete'); +}); + +test('old plans reject changed current run, app, artifact, backend, platform or variant inputs', (t) => { + fixture(t); + for (const key of [ + 'E2E_RUN_ID', + 'E2E_APP_SHA', + 'E2E_ARTIFACT_DIGEST', + 'E2E_BACKEND_REVISION', + 'E2E_VARIANT', + 'BACKEND', + 'PLATFORM', + ]) { + const value = process.env[key]; + process.env[key] = 'changed'; + assert.throws( + () => new InventoryReporter({ writeStream: { write: () => true } }), + /Run plan does not match executing/ + ); + process.env[key] = value; + } +}); +test('old plan rejects changed actual E2E source revision even with unchanged inventory', (t) => { + const f = fixture(t); + const stale = createPlan( + f.plan.cases, + { ...f.plan.identity, e2eSha: 'f'.repeat(40) }, + f.plan.inventoryDigest, + f.plan.pack + ); + writeJson(process.env.E2E_PLAN!, stale); + assert.throws( + () => new InventoryReporter({ writeStream: { write: () => true } }), + /executing e2eSha/ + ); +}); diff --git a/test/unit/reporting-cli.test.ts b/test/unit/reporting-cli.test.ts new file mode 100644 index 00000000..ae4800a8 --- /dev/null +++ b/test/unit/reporting-cli.test.ts @@ -0,0 +1,62 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import test from 'node:test'; +import { writeJson } from '../reporting/report.ts'; + +test('real CLI rejects a retained successful plan under a replacement execution', (t) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'reporting-cli-')); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const planPath = path.join(directory, 'plan.json'); + const env = { + ...process.env, + PLATFORM: 'ios', + BACKEND: 'local', + E2E_RUN_ID: 'cli-original', + E2E_APP_SHA: 'a'.repeat(40), + E2E_ARTIFACT_DIGEST: 'b'.repeat(64), + E2E_BACKEND_REVISION: 'stag0', + E2E_VARIANT: 'default', + GITHUB_STEP_SUMMARY: path.join(directory, 'github.md'), + }; + const invoke = (args: string[]) => + spawnSync( + process.execPath, + ['--experimental-strip-types', 'scripts/e2e-inventory.ts', ...args, '--plan', planPath], + { env, encoding: 'utf8' } + ); + const preparation = invoke([ + 'plan', + '--platform', + 'ios', + '--backend', + 'local', + '--pack', + 'local-money', + ]); + assert.equal(preparation.status, 0, preparation.stderr); + const plan = JSON.parse(fs.readFileSync(planPath, 'utf8')); + writeJson(path.join(directory, 'workers/attempt-1-0-0.json'), { + schemaVersion: 1, + planId: plan.id, + attempt: 1, + cid: '0-0', + startedAt: new Date().toISOString(), + finishedAt: new Date().toISOString(), + failures: 0, + errors: [], + events: plan.expected.map((id: string) => ({ id, status: 'passed', durationMs: 1 })), + }); + const success = invoke(['report']); + assert.equal(success.status, 0, success.stderr); + assert.equal( + JSON.parse(fs.readFileSync(path.join(directory, 'summary.json'), 'utf8')).status, + 'passed' + ); + env.E2E_RUN_ID = 'cli-replacement'; + const stale = invoke(['report']); + assert.equal(stale.status, 1); + assert.match(stale.stderr, /does not match executing runId/); +}); diff --git a/test/unit/reporting.test.ts b/test/unit/reporting.test.ts new file mode 100644 index 00000000..70cf8e5a --- /dev/null +++ b/test/unit/reporting.test.ts @@ -0,0 +1,194 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { execFileSync } from 'node:child_process'; +import { + createPlan, + reconcile, + validatePlan, + type Identity, + type WorkerReport, +} from '../reporting/report.ts'; +import type { InventoryCase } from '../reporting/inventory.ts'; + +const assignment = { + backend: 'regtest' as const, + pack: 'fixture', + disposition: 'required' as const, + fixtures: [], +}; +const cases: InventoryCase[] = ['wallet_1', 'payment_1'].map((id) => ({ + id, + file: 'test/specs/example.ts', + kind: 'test', + platforms: { ios: { ...assignment }, android: { ...assignment } }, +})); +const identity: Identity = { + runId: 'night-2026-11-03', + platform: 'ios', + backend: 'regtest', + appSha: 'a'.repeat(40), + e2eSha: 'b'.repeat(40), + artifactDigest: 'c'.repeat(64), + backendRevision: 'stag0', + variant: 'default', +}; +function plan() { + return createPlan(cases, identity, 'd'.repeat(64), 'fixture', 3); +} +function report( + events: WorkerReport['events'], + overrides: Partial = {} +): WorkerReport { + return { + schemaVersion: 1, + planId: plan().id, + attempt: 1, + cid: '0-0', + startedAt: '2026-11-03T00:00:00Z', + finishedAt: '2026-11-03T00:01:00Z', + failures: 0, + errors: [], + events, + ...overrides, + }; +} +const passes = cases.map((c) => ({ id: c.id, status: 'passed' as const, durationMs: 10 })); + +test('exact expected cases pass and retain timing', () => { + const result = reconcile(plan(), [report(passes)]); + assert.equal(result.status, 'passed'); + assert.equal(result.passedCount, 2); + assert.equal(result.durationMs, 20); +}); +test('same inputs on the following night require fresh receipts', () => { + const next = createPlan( + cases, + { ...identity, runId: 'night-2026-11-04' }, + 'd'.repeat(64), + 'fixture', + 3 + ); + assert.notEqual(next.id, plan().id); + assert.equal(reconcile(next, [report(passes)]).status, 'incomplete'); +}); +for (const change of [ + { platform: 'android' }, + { appSha: 'e'.repeat(40) }, + { backendRevision: 'other' }, + { variant: 'rn-upgrade' }, + { artifactDigest: 'f'.repeat(64) }, +]) { + test(`receipts cannot cross identity boundary ${Object.keys(change)[0]}`, () => { + const next = createPlan( + cases, + { ...identity, ...change } as Identity, + 'd'.repeat(64), + 'fixture', + 3 + ); + assert.equal(reconcile(next, [report(passes)]).status, 'incomplete'); + }); +} +test('unknown/empty packs and incomplete build identity cannot create a green plan', () => { + assert.throws(() => createPlan(cases, identity, 'd'.repeat(64), 'typo'), /No expected cases/); + assert.throws( + () => createPlan(cases, { ...identity, appSha: '' }, 'd'.repeat(64), 'fixture'), + /identity/ + ); +}); +test('tampered expected cases invalidate the plan', () => { + const changed = plan(); + changed.expected.pop(); + assert.throws(() => validatePlan(changed), /changed or is invalid/); +}); +for (const [name, reports] of [ + ['no started worker', []], + ['missing case', [report(passes.slice(0, 1))]], + ['unexpected skip', [report([{ ...passes[0], status: 'skipped' }, passes[1]])]], + ['wrong case with same total count', [report([passes[0], { ...passes[1], id: 'different_1' }])]], + ['duplicate report', [report(passes), report(passes)]], + ['duplicate case across workers', [report(passes), report([passes[0]], { cid: '0-1' })]], + ['worker interrupted after assertions', [report(passes, { finishedAt: undefined })]], + ['setup failure', [report([], { failures: 1 })]], + ['teardown failure after passing cases', [report(passes, { failures: 1 })]], +] as [string, WorkerReport[]][]) + test(`${name} cannot report full coverage`, () => + assert.equal(reconcile(plan(), reports).status, 'incomplete')); + +test('a recovered retry keeps first-attempt failure evidence', () => { + const result = reconcile(plan(), [ + report([{ ...passes[0], status: 'failed' }, passes[1]], { failures: 1 }), + report(passes, { attempt: 2 }), + ]); + assert.equal(result.status, 'passed'); + assert.equal(result.firstFailure, true); + assert.equal(result.results.find((r) => r.id === 'wallet_1')?.firstAttempt, 'failed'); +}); +test('a skipped retry is accepted only with a verified same-run earlier pass', () => { + const retry = report([{ ...passes[0], status: 'skipped' }, passes[1]], { attempt: 2 }); + assert.equal(reconcile(plan(), [report(passes), retry]).status, 'passed'); + assert.equal(reconcile(plan(), [retry]).status, 'incomplete'); +}); +test('an old pass cannot mask a later failure', () => { + assert.equal( + reconcile(plan(), [ + report(passes), + report([{ ...passes[0], status: 'failed' }, passes[1]], { attempt: 2, failures: 1 }), + ]).status, + 'incomplete' + ); +}); +test('single-attempt fixture rejects retry receipts', () => { + const single = createPlan(cases, identity, 'd'.repeat(64), 'fixture'); + assert.equal( + reconcile(single, [report(passes, { planId: single.id, attempt: 2 })]).status, + 'incomplete' + ); +}); +test('known coverage gaps remain visible even if a case happened to pass', () => { + const data = structuredClone(cases); + data[0].platforms.ios = { + ...assignment, + disposition: 'gap', + reason: 'missing fixture validation', + reference: '#264', + }; + const p = createPlan(data, identity, 'd'.repeat(64), 'fixture'); + const result = reconcile(p, [report(passes, { planId: p.id })]); + assert.equal(result.status, 'incomplete'); + assert.match(result.problems.join(' '), /Unresolved coverage gap/); +}); +test('inventory retries reuse only current-plan locks; legacy paths stay unchanged', (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'report-lock-test-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const helper = path.resolve('test/helpers/suite.ts'); + const output = execFileSync( + process.execPath, + [ + '--experimental-strip-types', + '-e', + ` + const assert = require('node:assert/strict'); + const s = require(${JSON.stringify(helper)}); + const name = '@send_1 - sends coins'; + assert.equal(s.checkComplete(name), false); + s.markComplete(name); + assert.equal(s.checkComplete(name), true); + assert.equal(s.checkComplete('@send_2 - another case'), false); + process.env.E2E_PLAN = ${JSON.stringify(path.join(root, 'next-run/plan.json'))}; + assert.equal(s.checkComplete(name), false); + delete process.env.E2E_PLAN; + assert.equal(s.lockFile(name), '/tmp/lock/lock-_send_1_-_sends_coins'); + console.log('ok'); + `, + ], + { + env: { ...process.env, CI: '1', E2E_PLAN: path.join(root, 'run/plan.json') }, + encoding: 'utf8', + } + ); + assert.match(output, /ok/); +}); diff --git a/test/unit/wdio-scheduling.test.cjs b/test/unit/wdio-scheduling.test.cjs new file mode 100644 index 00000000..f18e1567 --- /dev/null +++ b/test/unit/wdio-scheduling.test.cjs @@ -0,0 +1,44 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { fileURLToPath } = require('node:url'); +const { loadInventory } = require('../reporting/inventory.ts'); +const { executionIdentity } = require('../reporting/identity.ts'); +const { createPlan, writeJson } = require('../reporting/report.ts'); + +test('actual WDIO config schedules selected specs together on one simulator', async (t) => { + const before = { ...process.env }; + t.after(() => { + for (const k of Object.keys(process.env)) if (!(k in before)) delete process.env[k]; + Object.assign(process.env, before); + }); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'wdio-plan-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + Object.assign(process.env, { + PLATFORM: 'ios', + BACKEND: 'local', + E2E_RUN_ID: 'schedule-test', + E2E_APP_SHA: 'a'.repeat(40), + E2E_ARTIFACT_DIGEST: 'b'.repeat(64), + E2E_BACKEND_REVISION: 'local', + E2E_VARIANT: 'default', + }); + const ids = ['send_1', 'send_2', 'lightning_1', 'transfer_2', 'boost_2']; + const cases = loadInventory(process.cwd()).cases.filter((c) => ids.includes(c.id)); + const plan = createPlan(cases, executionIdentity('ios', 'local'), 'c'.repeat(64), 'selected', 3); + process.env.E2E_PLAN = path.join(root, 'plan.json'); + writeJson(process.env.E2E_PLAN, plan); + const { ConfigParser } = await import('@wdio/config/node'); + const parser = new ConfigParser(path.resolve('wdio.conf.ts')); + await parser.initialize(); + const specs = parser.getSpecs(); + assert.equal(specs.length, 1, 'Exactly one worker group per simulator'); + assert.ok(Array.isArray(specs[0])); + assert.deepEqual( + specs[0].map((f) => path.relative(process.cwd(), fileURLToPath(f))).sort(), + [...new Set(cases.map((c) => c.file))].sort() + ); + assert.equal(parser.getConfig().maxInstances, 1); +}); diff --git a/tsconfig.json b/tsconfig.json index 07de3e38..c9151d8e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -22,5 +22,11 @@ "noFallthroughCasesInSwitch": true, "esModuleInterop": true }, - "include": ["test", "test/types/**/*.d.ts", "wdio.conf.ts"] + "include": [ + "test", + "test/types/**/*.d.ts", + "wdio.conf.ts", + "scripts/e2e-inventory.ts", + "scripts/ci-run.ts" + ] } diff --git a/wdio.conf.ts b/wdio.conf.ts index bdb546b7..11e6ec32 100644 --- a/wdio.conf.ts +++ b/wdio.conf.ts @@ -2,6 +2,8 @@ import path from 'node:path'; import fs from 'node:fs'; import { grantIOSCameraPermission } from './test/helpers/setup'; +import InventoryReporter from './test/reporting/reporter'; +import { selectedSpecs } from './test/reporting/specs'; const appiumPort = Number(process.env.APPIUM_PORT ?? '4723'); const isAndroid = process.env.PLATFORM === 'android'; @@ -47,7 +49,7 @@ export const config: WebdriverIO.Config = { // // Spec in [] will be run in a separate worker process. // Running all specs sequentially for now. - specs: [['./test/specs/**/*.ts']], + specs: process.env.E2E_PLAN ? [selectedSpecs(process.env.E2E_PLAN)] : [['./test/specs/**/*.ts']], // Patterns to exclude. exclude: [ // 'path/to/excluded/files' @@ -68,7 +70,7 @@ export const config: WebdriverIO.Config = { // and 30 processes will get spawned. The property handles how many capabilities // from the same test should run tests. // - maxInstances: 10, + maxInstances: process.env.E2E_PLAN ? 1 : 10, // // If you have trouble getting all important capabilities together, check out the // Sauce Labs platform configurator - a great tool to configure your capabilities: @@ -193,7 +195,7 @@ export const config: WebdriverIO.Config = { // Test reporter for stdout. // The only one supported by default is 'dot' // see also: https://webdriver.io/docs/dot-reporter - reporters: ['spec'], + reporters: process.env.E2E_PLAN ? ['spec', [InventoryReporter, {}]] : ['spec'], // Options to be passed to Mocha. // See the full list at http://mochajs.org/ @@ -381,6 +383,20 @@ export const config: WebdriverIO.Config = { console.log(`🧪 Start: ${test.parent} - ${test.title}`); }, + afterHook: async function (_test, _context, { error }) { + if (!process.env.E2E_PLAN || !error) return; + try { + const directory = path.join(path.dirname(process.env.E2E_PLAN), 'hook-failures'); + fs.mkdirSync(directory, { recursive: true }); + const screenshot = await driver.takeScreenshot(); + fs.writeFileSync(path.join(directory, `${Date.now()}.png`), screenshot, 'base64'); + } catch { + console.warn( + 'Hook failed before a screenshot could be collected; retain runner and simulator logs.' + ); + } + }, + afterTest: async function (test, _context, { error }) { const saveDiagnostics = Boolean(error) || process.env.SAVE_DIAGNOSTICS === 'true';