diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5cef10b1..333bb612 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,14 +5,29 @@ on: workflow_dispatch: inputs: ui_validation_mode: - description: Full release gate or focused native account-menu diagnosis + description: Full release gate, focused account-menu diagnosis, or automatic-cost control type: choice options: - full - account-menu-comparison + - automatic-cost default: full comparison_base_sha: - description: Exact ancestor SHA for account-menu comparison only + description: Exact ancestor SHA for manual comparison modes + type: string + default: '' + + cost_source_role: + description: Source measured by an automatic-cost control + type: choice + options: [baseline, candidate] + default: candidate + cost_expected_image: + description: Exact hosted ImageVersion required before cost workload + type: string + default: '' + cost_expected_toolchain: + description: Baseline toolchain SHA256; required for candidate cost control type: string default: '' @@ -38,6 +53,20 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + fetch-depth: ${{ inputs.ui_validation_mode == 'automatic-cost' && '0' || '1' }} + ref: ${{ inputs.ui_validation_mode == 'automatic-cost' && github.sha || '' }} + + - name: Verify manual automatic-cost identity + if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode == 'automatic-cost' }} + env: + AUTOMATIC_COST_MODE: ${{ inputs.ui_validation_mode }} + AUTOMATIC_COST_ROLE: ${{ inputs.cost_source_role }} + AUTOMATIC_COST_BASE: ${{ inputs.comparison_base_sha }} + AUTOMATIC_COST_IMAGE: ${{ inputs.cost_expected_image }} + AUTOMATIC_COST_TOOLCHAIN: ${{ inputs.cost_expected_toolchain }} + run: | + cp scripts/prepare-automatic-cost.py "$RUNNER_TEMP/automatic-cost-controller.py" + python3 "$RUNNER_TEMP/automatic-cost-controller.py" --receipt "$RUNNER_TEMP/automatic-cost.json" - name: Run pinned SwiftLint policy run: >- @@ -51,7 +80,7 @@ jobs: run: python3 -m unittest discover -s scripts/tests -p 'test_ci_policy.py' -v strict-concurrency: - if: ${{ github.event_name == 'workflow_dispatch' }} + if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'automatic-cost' }} name: Strict concurrency runs-on: macos-26 env: @@ -70,7 +99,7 @@ jobs: run: ./scripts/check-strict-concurrency.sh ios-tests: - if: ${{ github.event_name == 'workflow_dispatch' }} + if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'automatic-cost' }} name: iOS tests runs-on: macos-26 env: @@ -126,7 +155,7 @@ jobs: retention-days: 14 watch-tests: - if: ${{ github.event_name == 'workflow_dispatch' }} + if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'automatic-cost' }} name: watchOS tests runs-on: macos-26 env: @@ -199,7 +228,7 @@ jobs: retention-days: 14 smoke-tests: - if: ${{ github.event_name == 'workflow_dispatch' }} + if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'automatic-cost' }} name: SwiftPM smoke tests runs-on: macos-26 env: @@ -224,8 +253,8 @@ jobs: run: python3 -m unittest discover -s scripts/tests -p 'test_perfection_runner.py' -v ios-ui-families: - name: iOS UI journeys (${{ matrix.family }}) - if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'account-menu-comparison' && success() }} + name: iOS UI journeys (${{ matrix.family }}, ${{ matrix.shard }}) + if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'account-menu-comparison' && inputs.ui_validation_mode != 'automatic-cost' && success() }} needs: - swiftlint - strict-concurrency @@ -240,6 +269,7 @@ jobs: fail-fast: false matrix: family: [iphone, ipad] + shard: [original-1, original-2, original-3, supplemental, synthetic, native] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -249,59 +279,50 @@ jobs: - name: Verify pinned release toolchain and installed runtimes run: python3 scripts/verify-release-toolchain.py --output "$RUNNER_TEMP/release-toolchain.json" - - name: Create an owned UI simulator - id: ui-device - run: | - ./scripts/run-pro-release-validation.sh provision --family "${{ matrix.family }}" --output "$RUNNER_TEMP/ui-device" - echo "id=$(jq -r .device.deviceId "$RUNNER_TEMP/ui-device/ownership.json")" >> "$GITHUB_OUTPUT" + - name: Verify manual shard failure fixtures + if: ${{ matrix.family == 'iphone' && matrix.shard == 'original-1' }} + run: python3 -m unittest scripts.tests.test_ui_release_shards -v - name: Run GitHub Billing API fixtures - if: ${{ matrix.family == 'iphone' }} + if: ${{ matrix.family == 'iphone' && matrix.shard == 'original-1' }} run: xcrun swift run GitHubBillingFixtureTests - - name: Run UI journeys - id: ui-base - if: ${{ !cancelled() && steps.ui-device.outcome == 'success' }} - env: - UI_TEST_REQUIRE_DEVICE_ID: '1' - UI_TEST_OWNERSHIP_RECEIPT: ${{ runner.temp }}/ui-device/ownership.json - UI_TEST_RESULTS_DIR: ${{ runner.temp }}/ui-tests - UI_TEST_DEVICE_ID: ${{ steps.ui-device.outputs.id }} - run: ./scripts/run-ui-tests.sh "${{ matrix.family }}" - - - name: Run supplemental and commerce release phases - if: ${{ !cancelled() && steps.ui-base.outcome == 'success' }} + - name: Run the exact UI shard on a new qualified simulator + id: ui-shard run: >- - ./scripts/run-pro-release-validation.sh ui --family "${{ matrix.family }}" - --ownership "$RUNNER_TEMP/ui-device/ownership.json" - --base-results "$RUNNER_TEMP/ui-tests" --output "$RUNNER_TEMP/pro-ui" + python3 scripts/ui_release_shards.py run --family "${{ matrix.family }}" + --shard "${{ matrix.shard }}" --output "$RUNNER_TEMP/ui-shard" - - name: Shut down the owned UI simulator - if: ${{ always() && steps.ui-device.outcome == 'success' }} - run: >- - ./scripts/run-pro-release-validation.sh shutdown --family "${{ matrix.family }}" - --ownership "$RUNNER_TEMP/ui-device/ownership.json" --output "$RUNNER_TEMP/ui-shutdown" + - name: Publish compact shard proof + if: ${{ !cancelled() && steps.ui-shard.outcome == 'success' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ui-shard-proof-${{ matrix.family }}-${{ matrix.shard }} + path: ${{ runner.temp }}/ui-shard/proof.json + if-no-files-found: error + overwrite: true + retention-days: 14 - name: Preserve UI test results and failure screenshots if: ${{ always() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: CodexBarIOSUITests-${{ matrix.family }}-${{ github.run_id }}-${{ github.run_attempt }} + name: CodexBarIOSUITests-${{ matrix.family }}-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }} path: | - ${{ runner.temp }}/ui-tests - ${{ runner.temp }}/pro-ui - ${{ runner.temp }}/ui-device - ${{ runner.temp }}/ui-shutdown + ${{ runner.temp }}/ui-shard ${{ runner.temp }}/release-toolchain.json - !${{ runner.temp }}/ui-tests/*/DerivedData-*/** - !${{ runner.temp }}/pro-ui/DerivedData-*/** + !${{ runner.temp }}/ui-shard/DerivedData-*/** if-no-files-found: warn retention-days: 14 ios-ui-tests: name: Full iOS UI validation - if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'account-menu-comparison' }} + if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'account-menu-comparison' && inputs.ui_validation_mode != 'automatic-cost' }} needs: ios-ui-families + permissions: + actions: read + pull-requests: write + contents: read runs-on: ubuntu-latest timeout-minutes: 3 steps: @@ -317,6 +338,174 @@ jobs: exit 1 fi + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Require exact-head proofs for all twelve shards + env: + GH_TOKEN: ${{ github.token }} + run: | + gh run download "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" \ + --pattern 'ui-shard-proof-*' --dir "$RUNNER_TEMP/ui-proof" + python3 scripts/ui_release_shards.py aggregate --proofs "$RUNNER_TEMP/ui-proof" \ + --output "$RUNNER_TEMP/full-ui-proof.json" + + - name: Notify Chief of manual UI aggregate outcome + if: ${{ always() && !cancelled() && github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main' }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + CHIEF_NOTIFY_STATUS: ${{ job.status }} + with: + github-token: ${{ github.token }} + retries: 0 + debug: false + script: | + // Candidate actions/github-script body. The patch embeds these exact bytes. + // No checkout code, PR body, comment instructions, or payload URLs are executed. + const owner = "hemsoft-dev", repo = "codexbar-ios", repositoryId = 1285058368; + const workflowId = 319463386; + const botId = 41898282; + const params = {owner, repo}; + const reject = code => { throw new Error(code); }; + const integer = value => { + if (!/^[1-9][0-9]*$/.test(String(value))) reject("invalid-integer"); + const number = Number(value); + if (!Number.isSafeInteger(number)) reject("invalid-integer"); + return number; + }; + try { + const runId = integer(process.env.GITHUB_RUN_ID); + const attempt = integer(process.env.GITHUB_RUN_ATTEMPT); + const sha = context.sha; + const phase = "full-ui-summary"; + const claimedStatus = process.env.CHIEF_NOTIFY_STATUS; + if (context.eventName !== "workflow_dispatch" || context.repo.owner !== owner || + context.repo.repo !== repo || !/^[0-9a-f]{40}$/.test(sha) || + typeof context.ref !== "string" || !/^refs\/(heads|tags)\/.+/.test(context.ref) || + !["success", "failure", "cancelled"].includes(claimedStatus)) reject("context-refused"); + if (context.job !== "ios-ui-tests") reject("job-refused"); + const branch = context.ref.startsWith("refs/heads/") ? context.ref.slice("refs/heads/".length) : null; + const checkRun = run => { + if (run.id !== runId || run.run_attempt !== attempt || run.workflow_id !== workflowId || + run.repository?.id !== repositoryId || run.repository?.full_name !== `${owner}/${repo}` || + run.head_repository?.id !== repositoryId || run.event !== "workflow_dispatch" || + run.path !== ".github/workflows/ci.yml" || run.head_sha !== sha || + (branch !== null && run.head_branch !== branch)) reject("run-refused"); + }; + const skip = reason => { + core.notice(`Chief notifier not applicable (${reason}); native validation is unchanged.`); + return {state: "not-applicable", reason}; + }; + const getRun = async () => (await github.rest.actions.getWorkflowRun({...params, run_id: runId})).data; + const getRepo = async () => (await github.rest.repos.get(params)).data; + const run = await getRun(); + checkRun(run); + const metadata = await getRepo(); + if (metadata.id !== repositoryId || metadata.full_name !== `${owner}/${repo}` || + typeof metadata.default_branch !== "string" || !metadata.default_branch) reject("repository-refused"); + if (branch === null) return skip("non-branch-ref"); + // Deliberately no historical-PR lookup on main/default-branch validation. + if (branch === "main" || branch === metadata.default_branch) return skip("main-or-default-branch"); + // Bounded target resolution for this exact run branch, not repository monitoring. + const candidates = await github.paginate(github.rest.pulls.list, + {...params, state: "open", head: `${owner}:${branch}`, base: metadata.default_branch, per_page: 100}); + const isTarget = pr => pr.state === "open" && pr.merged_at === null && + pr.base?.repo?.id === repositoryId && pr.base?.ref === metadata.default_branch && + pr.head?.repo?.id === repositoryId && pr.head?.sha === sha && pr.head?.ref === branch; + const matches = candidates.filter(isTarget); + if (matches.length === 0) return skip("no-current-open-pr"); + if (matches.length !== 1) reject("ambiguous-pr"); + const prNumber = integer(matches[0].number); + const stillTarget = pr => pr.number === prNumber && isTarget(pr); + const getPr = async () => (await github.rest.pulls.get({...params, pull_number: prNumber})).data; + if (!stillTarget(await getPr())) return skip("pr-closed-or-head-moved"); + const jobs = await github.paginate( + "GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt_number}/jobs", + {...params, run_id: runId, attempt_number: attempt, per_page: 100}, + ); + const jobNamed = name => { + const found = jobs.filter(job => job.name === name); + if (found.length > 1) reject("ambiguous-job"); + if (found.length === 0) return null; + const job = found[0]; + integer(job.id); + if (job.run_id !== runId || job.run_attempt !== attempt || job.head_sha !== sha) reject("job-refused"); + return job; + }; + const currentName = "Full iOS UI validation"; + const current = jobNamed(currentName); + if (!current || !Array.isArray(current.steps)) reject("current-job-unavailable"); + const failureSteps = current.steps.filter(step => step.status === "completed" && step.conclusion === "failure"); + const prerequisites = ["SwiftLint", "Strict concurrency", "iOS tests", "watchOS tests", "SwiftPM smoke tests"]; + const shards = ["iphone", "ipad"].flatMap(family => + ["original-1", "original-2", "original-3", "supplemental", "synthetic", "native"].map( + shard => `iOS UI journeys (${family}, ${shard})`, + ), + ); + let outcome = "unknown"; + let title = "Manual UI notification could not verify the observed outcome."; + const evidence = []; + const required = [...prerequisites, ...shards].map(jobNamed); + const incomplete = required.some(job => !job || job.status !== "completed" || job.conclusion !== "success"); + const knownBad = required.some(job => job && job.status === "completed" && job.conclusion !== "success"); + const proof = current.steps.find(step => step.name === "Require exact-head proofs for all twelve shards"); + if (failureSteps.length > 0 || knownBad) { + outcome = "failure"; + title = "Manual UI validation failed or is incomplete."; + } else if (!incomplete && claimedStatus === "success" && proof?.status === "completed" && proof.conclusion === "success") { + outcome = "passed-so-far"; + title = "Manual UI quality steps succeeded before this notifier."; + } + for (const name of [...prerequisites, ...shards]) { + const job = jobNamed(name); + if (!job) evidence.push(`- ${name}: missing.`); + else if (job.status !== "completed" || job.conclusion !== "success") { + evidence.push(`- ${name}: not verified successful. https://github.com/${owner}/${repo}/actions/runs/${runId}/job/${job.id}`); + } + } + const marker = ``; + const runUrl = `https://github.com/${owner}/${repo}/actions/runs/${runId}/attempts/${attempt}`; + const jobUrl = `https://github.com/${owner}/${repo}/actions/runs/${runId}/job/${current.id}`; + const body = [title, "", `[Run ${runId}, attempt ${attempt}](${runUrl}) at head \`${sha}\`.`, + `[Source job](${jobUrl}). This is an event hint. Chief must verify GitHub API state.`, + "The notifier runs before its own job and the workflow finish. This comment does not establish workflow completion or merge readiness.", + ...evidence, "", marker].join("\n"); + const comments = await github.paginate(github.rest.issues.listComments, {...params, issue_number: prNumber, per_page: 100}); + const previous = comments.filter(comment => comment.user?.id === botId && + comment.user?.login === "github-actions[bot]" && comment.user?.type === "Bot" && + String(comment.body).split("\n").includes(marker)); + if (previous.length > 1) reject("duplicate-comments-already-exist"); + // Recheck immediately before a write; GitHub has no atomic head-bound comment API. + checkRun(await getRun()); + const latestRepo = await getRepo(); + if (latestRepo.id !== repositoryId || latestRepo.full_name !== `${owner}/${repo}`) reject("repository-refused"); + if (latestRepo.default_branch !== metadata.default_branch) return skip("default-branch-changed"); + if (!stillTarget(await getPr())) return skip("pr-closed-or-head-moved"); + if (previous.length === 1) { + core.notice("Chief manual CI hint already exists for this verified event."); + if (outcome === "unknown") core.setFailed("Chief notifier outcome remains unknown; root verification is required."); + return {state: "duplicate", outcome, commentId: integer(previous[0].id)}; + } + // retries: 0 in the action prevents retrying an ambiguous POST automatically. + const response = await github.rest.issues.createComment({...params, issue_number: prNumber, body}); + const comment = response.data; + if (response.status !== 201 || comment.user?.id !== botId || comment.user?.login !== "github-actions[bot]" || + comment.user?.type !== "Bot" || comment.body !== body || + comment.issue_url !== `https://api.github.com/repos/${owner}/${repo}/issues/${prNumber}`) reject("comment-response-unverified"); + core.notice("Chief manual CI hint posted; delivery to the watching thread is not yet proven."); + if (outcome === "unknown") core.setFailed("Chief notifier outcome remains unknown; root verification is required."); + return {state: "created", outcome, commentId: integer(comment.id)}; + } catch (error) { + // Never log API error text, request headers, tokens, response bodies or arbitrary PR text. + const codes = new Set(["invalid-integer", "context-refused", "job-refused", "run-refused", + "repository-refused", "ambiguous-pr", "ambiguous-job", "current-job-unavailable", + "duplicate-comments-already-exist", "comment-response-unverified"]); + const code = codes.has(error?.message) ? error.message : "github-api-unavailable"; + core.setFailed(`Chief notifier refused delivery (${code}). Outcome remains unknown; root must inspect this manual run.`); + return {state: "unverified", reason: code}; + } + account-menu-comparison: name: Account menu comparison if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode == 'account-menu-comparison' && success() }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 577230c1..37f30fcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,16 @@ building, testing, or releasing the app. ### Developer Experience +- Split manual iPhone and iPad UI qualification into measured shards while + preserving the 90-minute worker limit and automatic PR checks. Keep touch + target assertions from rejecting 44-point controls due to floating-point noise. + Distinguish reported zero Grok usage from missing percentages in UI fixtures. + Wait for billing warnings to disappear after reloading sign-in. + Add aggregate-only manual validation hints for matching open PR watches; + early failure notification delivery remains unqualified. + Require matching hosted image and toolchain identities for manual automatic-CI cost comparisons. + ([#517](https://github.com/hemsoft-dev/codexbar-ios/issues/517)) + - Add a Muse Code allowance reader for schema 1 of `muse usage --json`, with sanitized fixtures and distinct unavailable states, for use if Meta ships a supported iPhone source. Muse Code can't be added, refreshed or published, diff --git a/CI-POLICY.md b/CI-POLICY.md index 87c3f38c..64ec6ea9 100644 --- a/CI-POLICY.md +++ b/CI-POLICY.md @@ -306,3 +306,77 @@ the Germany preparation and storefront phases or function-risk scoring had run. Its 25-minute test step covered the native build, 1,106 ordinary tests, native preparation and 25 native commerce tests. The new budget leaves room for the remaining phases and scoring without retries. + +## Manual UI sharding + +[Issue #517](https://github.com/hemsoft-dev/codexbar-ios/issues/517) partitions only +`workflow_dispatch` UI work. Each iPhone/iPad family has three original shards, +one supplemental shard, one synthetic-commerce shard and one native-commerce +shard. Each worker retains its 90-minute timeout; no timeout increase is used. +The existing Xcode 26.6 / 17F113 and iOS 26.2 / 23C54 verification remains mandatory. + +The retained [hosted run](https://github.com/hemsoft-dev/codexbar-ios/actions/runs/39810829566) +took 7,214 seconds for the iPad original phase, including build and process +overhead. Its 29 test durations sum to 6,624.54 seconds; the two longest tests +were 1,620.38 and 1,603.84 seconds. Measured greedy partitioning assigns 2,212.66, +2,205.04 and 2,206.84 test-seconds to the original shards. These are historical +allocation inputs, not claimed candidate run times. The implementation PR records +completed candidate measurements and the final exact-head hosted gate result. +The [baseline data](docs/ci/issue-517-baseline.json) retains the source run and +per-test values without downloading the multi-gigabyte result bundles. + +The automatic SwiftLint job, its steps, the four lightweight policy test methods, +triggers, native-job dispatch conditions and automatic timeouts are retained. +The new shard fixture suite runs only manually. Before/after automatic workflow +elapsed time, summed job runtime and API billing values are recorded in the PR; +manual shard durations are reported separately. Neither a queue delay nor an +estimated shard budget is proof that automatic work stayed within the ceiling. + +`Full iOS UI validation` now requires all twelve successful workers and their +current-source proofs. The compact artifacts identify every executed method, +qualified toolchain/runtime, source SHA/fingerprint and owned simulator. The +aggregate rejects missing, duplicate or substituted shards. The 53 journeys per +family include the two SUS tests added after 2.0; the original local 29-test command +is unchanged. StoreKit preparation and final reset cannot be split across workers. + +## Manual aggregate event hints + +The existing manual aggregate can post a source-linked hint to the run's exact +current open same-repository PR targeting the default branch. The +[approved v2 proposal](https://github.com/hemsoft-dev/chief/blob/8d9fb8d323d100333853a56174318f006d86e093/proposals/chief-40/manual-comment/README.md) +supplies the inline script. It authenticates run, attempt, repository, branch, +head, jobs and current PR through GitHub before posting. PR/comment text and +payload URLs supply no instructions or authority. Only the verified Actions +bot's exact event marker suppresses a serial duplicate; POST has no automatic +retry or fallback. + +Only `ios-ui-tests` gains job-level `pull-requests: write`, alongside its existing +Actions/Contents read permissions. This grant applies to every manual execution +of that job, including main and no-PR runs. The notifier's guards do not remove +the token's rights. Main, the current default branch, tags and runs without one +matching open current-head PR produce no comment. Matrix workers and automatic +jobs receive no new permission. Triggers, jobs, native steps, needs, matrices, +timeouts and concurrency are unchanged. + +Hints run after the matrix reaches the aggregate. They provide no early-failure +notification or completion guarantee. A success hint means passed-so-far; +post-steps and the workflow may still fail. Hard timeout, cancellation or runner +death can prevent delivery. Unknown/authentication/transport failures stay +visible and may fail an otherwise successful aggregate. Concurrent writers, +head moves and eventual consistency prevent an exactly-once guarantee. The +existing PR watch must already belong to the intended owner, who independently +verifies actual Actions state after waking. + +Run `node --test scripts/tests/manual/test_manual_ci_comment.mjs` with Node 24 +to exercise the actual inline script against mocked GitHub metadata, target +selection, failures and deduplication. The manual shard fixture suite also +verifies the exact approved script and proves that removing its single step +and permission grant restores the prior workflow bytes. These tests add no +automatic PR workload. The proposal's 39 offline checks and the local adaptation +checks do not prove live delivery. [Chief issue #40](https://github.com/hemsoft-dev/chief/issues/40) +stays open until a real bot comment and the exact owner-thread wake are verified +on a normal required qualification run. Existing dispatches cannot acquire the +new step, and no dispatch is added solely to test notification. + +Rollback removes the final notifier step and its aggregate write grant, keeping +all quality gates, prior evidence and existing watches intact. diff --git a/CodexBarIOS/Services/ProCommerceFixture.swift b/CodexBarIOS/Services/ProCommerceFixture.swift index 7a16be45..f64bffee 100644 --- a/CodexBarIOS/Services/ProCommerceFixture.swift +++ b/CodexBarIOS/Services/ProCommerceFixture.swift @@ -1,5 +1,60 @@ #if DEBUG import Foundation +#if targetEnvironment(simulator) +import OSLog + +/// Observations only: never supplies readiness or changes a StoreKit operation. +@MainActor +enum ProNativeReadinessTrace { + enum Stage: String { + case snapshotBegin, snapshotEnd, appTransactionBegin, appTransactionReturn, appTransactionError + case entitlementsBegin, entitlementsEnd, latestBegin, latestReturn + case snapshotProductsBegin, snapshotProductsReturn, statusBegin, statusReturn, snapshotError + case catalogBegin, catalogEnd, catalogError, storefrontBegin, storefrontReturn + case productsBegin, productsReturn, modelsBegin, modelsEnd, finalStorefrontBegin, finalStorefrontReturn + case viewRefreshBegin, viewRefreshEnd, viewReadiness + } + + private static let logger = Logger(subsystem: "com.hemsoft.CodexBarIOS.native-readiness", category: "fixture") + private static let launch = UUID().uuidString + private static var sequence = 0 + private static var previousReadiness: String? + + static func runID(environment: [String: String]) -> UUID? { + guard environment["CODEXBAR_UI_TESTS"] == "1", + environment["CODEXBAR_UI_TEST_SCENARIO"] == "commerce-native-storekit", + let raw = environment["CODEXBAR_UI_TEST_RUN_ID"] else { return nil } + return UUID(uuidString: raw) + } + + static func emit(_ stage: Stage, count: Int = 0, operation: UUID? = nil) { + guard let run = runID(environment: ProcessInfo.processInfo.environment) else { return } + let wallBefore = Date().timeIntervalSince1970 + let uptime = ProcessInfo.processInfo.systemUptime + let wallAfter = Date().timeIntervalSince1970 + sequence += 1 + let record: [String: Any] = [ + "schema": 1, "stage": stage.rawValue, "run": run.uuidString, "launch": launch, + "sequence": sequence, "pid": ProcessInfo.processInfo.processIdentifier, "count": count, + "operation": operation?.uuidString ?? "none", + "wallBefore": wallBefore, "uptime": uptime, "wallAfter": wallAfter, + ] + guard let data = try? JSONSerialization.data(withJSONObject: record, options: [.sortedKeys]), + let message = String(data: data, encoding: .utf8) else { return } + logger.notice("CB517NATIVE \(message, privacy: .public)") + } + + static func readiness(_ ready: Bool, resolved: Bool, refreshing: Bool, catalog: Int) { + guard runID(environment: ProcessInfo.processInfo.environment) != nil else { return } + // Non-observed storage cannot invalidate SwiftUI or schedule more refreshes. + let state = "\(ready)-\(resolved)-\(refreshing)-\(catalog)" + guard previousReadiness != state else { return } + previousReadiness = state + // Fixed bit fields: ready=1, resolved=2, refreshing=4; catalog enum occupies bits 3+. + emit(.viewReadiness, count: (ready ? 1 : 0) | (resolved ? 2 : 0) | (refreshing ? 4 : 0) | (catalog << 3)) + } +} +#endif /// Explicit simulator fixtures share the production reducer but never its stores or listeners. @MainActor diff --git a/CodexBarIOS/Services/StoreKitProCatalog.swift b/CodexBarIOS/Services/StoreKitProCatalog.swift index 1100a356..c0f80044 100644 --- a/CodexBarIOS/Services/StoreKitProCatalog.swift +++ b/CodexBarIOS/Services/StoreKitProCatalog.swift @@ -9,14 +9,34 @@ final class StoreKitProCatalog { private var generation = 0 func load() async throws -> ProCatalog { +#if DEBUG && targetEnvironment(simulator) + let traceOperation = UUID() +#endif generation += 1 let request = generation - guard let storefront = await Storefront.current else { throw ProCommerceError.catalogUnavailable } +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.storefrontBegin, operation: traceOperation) +#endif + let initialStorefront = await Storefront.current +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.storefrontReturn, operation: traceOperation) +#endif + guard let storefront = initialStorefront else { throw ProCommerceError.catalogUnavailable } +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.productsBegin, operation: traceOperation) +#endif let fetched = try await Product.products(for: ProProductID.allCases.map(\.rawValue)) +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.productsReturn, count: fetched.count, operation: traceOperation) +#endif + // Some runtimes report a failed lookup by returning no products instead of throwing. guard !fetched.isEmpty else { throw ProCommerceError.catalogUnavailable } var models: [ProProductID: ProCatalogProduct] = [:] var retained: [ProProductID: Product] = [:] +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.modelsBegin, operation: traceOperation) +#endif for product in fetched { guard let identifier = ProProductID(rawValue: product.id) else { continue } let model = await Self.model(product, identifier: identifier, storefront: storefront.id) @@ -24,7 +44,17 @@ final class StoreKitProCatalog { models[identifier] = model retained[identifier] = product } +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.modelsEnd, operation: traceOperation) +#endif +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.finalStorefrontBegin, operation: traceOperation) +#endif let currentStorefront = await Storefront.current +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.finalStorefrontReturn, operation: traceOperation) +#endif + try Task.checkCancellation() guard request == generation, currentStorefront?.id == storefront.id else { throw ProCommerceError.termsChanged } let catalog = ProCatalog(products: models, storefrontID: storefront.id) diff --git a/CodexBarIOS/Services/StoreKitProStoreSource.swift b/CodexBarIOS/Services/StoreKitProStoreSource.swift index 21859511..3bb972f8 100644 --- a/CodexBarIOS/Services/StoreKitProStoreSource.swift +++ b/CodexBarIOS/Services/StoreKitProStoreSource.swift @@ -6,23 +6,82 @@ final class StoreKitProStoreSource: ProStoreSource { private var restoredClockEvidence: ProFreshClockEvidence? private let commerceCatalog = StoreKitProCatalog() - func catalog() async throws -> ProCatalog { try await commerceCatalog.load() } + func catalog() async throws -> ProCatalog { +#if DEBUG && targetEnvironment(simulator) + let traceOperation = UUID() +#endif +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.catalogBegin, operation: traceOperation) +#endif + do { + let catalog = try await commerceCatalog.load() +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.catalogEnd, count: catalog.products.count, operation: traceOperation) +#endif + return catalog + } catch { +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.catalogError, operation: traceOperation) +#endif + throw error + } + } func storefrontUpdates() -> AsyncStream { commerceCatalog.storefrontUpdates() } func snapshot() async -> ProStoreSnapshot { +#if DEBUG && targetEnvironment(simulator) + let traceOperation = UUID() +#endif + +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.snapshotBegin, operation: traceOperation) +#endif + defer { +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.snapshotEnd, operation: traceOperation) +#endif + } var snapshot = ProStoreSnapshot() await appendAppPurchase(to: &snapshot) snapshot.freshClockEvidence = restoredClockEvidence +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.entitlementsBegin, operation: traceOperation) +#endif for await result in Transaction.currentEntitlements { append(result, to: &snapshot) } +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.entitlementsEnd, operation: traceOperation) +#endif + // Latest signed records also expose revocations omitted by currentEntitlements, including after relaunch. for product in ProProductID.allCases { +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.latestBegin, operation: traceOperation) +#endif if let result = await Transaction.latest(for: product.rawValue) { append(result, to: &snapshot) } +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.latestReturn, operation: traceOperation) +#endif + } do { +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.snapshotProductsBegin, operation: traceOperation) +#endif let products = try await Product.products(for: ProProductID.allCases.map(\.rawValue)) +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.snapshotProductsReturn, count: products.count, operation: traceOperation) +#endif + if let subscription = products.first(where: { $0.subscription != nil })?.subscription { +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.statusBegin, operation: traceOperation) +#endif let statuses = try await subscription.status +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.statusReturn, count: statuses.count, operation: traceOperation) +#endif + snapshot.subscriptionStatusProducts = [] for status in statuses { if let evidence = Self.subscription(status) { @@ -39,6 +98,9 @@ final class StoreKitProStoreSource: ProStoreSource { snapshot.subscriptionStatusAvailable = true } } catch { +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.snapshotError, operation: traceOperation) +#endif // StoreKit can be unavailable offline. Only a previous verified decision may supply the allowance. } return snapshot @@ -132,8 +194,18 @@ final class StoreKitProStoreSource: ProStoreSource { } private func appendAppPurchase(to snapshot: inout ProStoreSnapshot) async { +#if DEBUG && targetEnvironment(simulator) + let traceOperation = UUID() +#endif do { - switch try await AppTransaction.shared { +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.appTransactionBegin, operation: traceOperation) +#endif + let result = try await AppTransaction.shared +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.appTransactionReturn, operation: traceOperation) +#endif + switch result { case let .verified(transaction): guard let environment = Self.environment(transaction.environment) else { return } snapshot.appendVerifiedAppPurchase( @@ -148,6 +220,9 @@ final class StoreKitProStoreSource: ProStoreSource { snapshot.appVerificationEnvironment = Self.environment(transaction.environment) } } catch { +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.appTransactionError, operation: traceOperation) +#endif // An unavailable AppTransaction does not replace a previously verified permanent grant. } } diff --git a/CodexBarIOS/Services/UITestFixtures.swift b/CodexBarIOS/Services/UITestFixtures.swift index d036f6dd..00188a06 100644 --- a/CodexBarIOS/Services/UITestFixtures.swift +++ b/CodexBarIOS/Services/UITestFixtures.swift @@ -807,8 +807,8 @@ final class UITestFixtures { let end = ISO8601DateFormatter().string(from: now.addingTimeInterval(5 * 86_400)) let noAllowance = scenario == "grok-no-allowance" let percentField = switch scenario { - case "grok-zero", "grok-existing-zero", "grok-spending-zero": "" - case "grok-spending-reported-zero": "\"creditUsagePercent\":0," + case "grok-spending-zero": "" + case "grok-zero", "grok-existing-zero", "grok-spending-reported-zero": "\"creditUsagePercent\":0," case "grok-no-allowance", "grok-percent-unavailable", "grok-existing-percent-unavailable", "grok-spending-unavailable": "\"creditUsagePercent\":null," diff --git a/CodexBarIOS/Views/ProPresentationView.swift b/CodexBarIOS/Views/ProPresentationView.swift index 82806561..8f26cbc2 100644 --- a/CodexBarIOS/Views/ProPresentationView.swift +++ b/CodexBarIOS/Views/ProPresentationView.swift @@ -45,7 +45,7 @@ struct ProPresentationView: View { } } .accessibilityIdentifier("codexbar-pro-presentation") - .accessibilityValue(isReady ? "ready" : "loading") + .accessibilityValue(readinessValue) .interactiveDismissDisabled(entitlement.command != nil) .background { ProSubscriptionSheetPresenter(isPresented: $showsManagement, groupID: managementGroupID) @@ -253,6 +253,21 @@ struct ProPresentationView: View { private var isBusy: Bool { entitlement.command != nil || isRefreshing || !entitlement.isResolved } + private var readinessValue: String { + let ready = isReady +#if DEBUG && targetEnvironment(simulator) + let catalog: Int + switch entitlement.catalog { + case .idle: catalog = 0 + case .loading: catalog = 1 + case .loaded: catalog = 2 + case .failed: catalog = 3 + } + ProNativeReadinessTrace.readiness(ready, resolved: entitlement.isResolved, refreshing: isRefreshing, catalog: catalog) +#endif + return ready ? "ready" : "loading" + } + private var isReady: Bool { guard entitlement.isResolved, !isRefreshing else { return false } switch entitlement.catalog { @@ -353,9 +368,20 @@ struct ProPresentationView: View { } private func refreshCommerce() async { +#if DEBUG && targetEnvironment(simulator) + let traceOperation = UUID() +#endif guard !isRefreshing, entitlement.command == nil else { return } isRefreshing = true - defer { isRefreshing = false } +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.viewRefreshBegin, operation: traceOperation) +#endif + defer { + isRefreshing = false +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.viewRefreshEnd, operation: traceOperation) +#endif + } await entitlement.refreshCommerce() } diff --git a/CodexBarIOSTests/StorefrontFixtureTests.swift b/CodexBarIOSTests/StorefrontFixtureTests.swift index 0c4179ec..d266fb95 100644 --- a/CodexBarIOSTests/StorefrontFixtureTests.swift +++ b/CodexBarIOSTests/StorefrontFixtureTests.swift @@ -2,6 +2,25 @@ import XCTest @testable import CodexBarIOS final class StorefrontFixtureTests: XCTestCase { + @MainActor + func testGrokUIFixturesSeparateReportedZeroFromMissingQuota() { + let account = ProviderAccountConfiguration(id: "fixture-grok-zero", providerID: .grok, + accountLabel: "Grok fixture", authMethod: .browserSession) + for scenario in ["grok-zero", "grok-existing-zero", "grok-spending-reported-zero"] { + let result = UITestFixtures.grokResult(for: account, scenario: scenario) + XCTAssertEqual(result.bars.map(\.used), [0], scenario) + XCTAssertTrue(result.usageMessages.isEmpty, scenario) + } + for scenario in [ + "grok-spending-zero", "grok-percent-unavailable", "grok-existing-percent-unavailable", + "grok-spending-unavailable", + ] { + let result = UITestFixtures.grokResult(for: account, scenario: scenario) + XCTAssertTrue(result.bars.isEmpty, scenario) + XCTAssertEqual(result.usageMessages, ["Grok did not report included usage."], scenario) + } + } + @MainActor func testStorefrontFreeSceneResolvesOneAccountAndSevenDayHistory() async throws { let scene = try XCTUnwrap(AppStoreScreenshotConfiguration.parse(arguments: [ diff --git a/CodexBarIOSUITests/CodexCreditsPoolUITests.swift b/CodexBarIOSUITests/CodexCreditsPoolUITests.swift index e466db29..f325b8e4 100644 --- a/CodexBarIOSUITests/CodexCreditsPoolUITests.swift +++ b/CodexBarIOSUITests/CodexCreditsPoolUITests.swift @@ -387,7 +387,8 @@ final class CodexCreditsPoolUITests: XCTestCase { keep("billing-\(title)-wrong-account", app: app) tap(app.buttons["Reload Sign-In"], in: app) XCTAssertTrue(app.buttons["subscription-billing-synthetic-match"].waitForExistence(timeout: 5)) - XCTAssertFalse(app.staticTexts.containing(NSPredicate(format: "label CONTAINS %@", "different account")).firstMatch.exists) + let mismatch = app.staticTexts.containing(NSPredicate(format: "label CONTAINS %@", "different account")).firstMatch + XCTAssertTrue(mismatch.waitForNonExistence(timeout: 5), app.debugDescription) keep("billing-\(title)-reloaded", app: app) tap(app.buttons["subscription-billing-synthetic-match"], in: app) XCTAssertTrue(app.buttons["subscription-billing-disconnect"].waitForExistence(timeout: 10), app.debugDescription) @@ -458,8 +459,9 @@ final class CodexCreditsPoolUITests: XCTestCase { let quota = personalQuota(in: app) XCTAssertTrue(quota.waitForExistence(timeout: 10), app.debugDescription) let menu = app.buttons["More options for Personal Codex"] - XCTAssertGreaterThanOrEqual(menu.frame.width, 44) - XCTAssertGreaterThanOrEqual(menu.frame.height, 44) + // XCUI geometry can report a 44-point target just below 44 after floating-point subtraction. + XCTAssertGreaterThanOrEqual(menu.frame.width + 0.000001, 44) + XCTAssertGreaterThanOrEqual(menu.frame.height + 0.000001, 44) keep("spacing-\(defaultText ? "default" : "accessibility2")-\(dark ? "dark" : "light")", app: app) let disclosure = app.descendants(matching: .any)["Personal Codex, Plan unavailable, Synthetic Codex usage, Normal status"] XCTAssertTrue(disclosure.exists, app.debugDescription) @@ -479,7 +481,7 @@ final class CodexCreditsPoolUITests: XCTestCase { )).firstMatch XCTAssertTrue(thirtyDay.waitForExistence(timeout: 10), badge.debugDescription) let badgeMenu = badge.buttons["More options for Personal Codex"] - XCTAssertGreaterThanOrEqual(badgeMenu.frame.height, 44) + XCTAssertGreaterThanOrEqual(badgeMenu.frame.height + 0.000001, 44) keep("spacing-badge-\(defaultText ? "default" : "accessibility2")-\(dark ? "dark" : "light")", app: badge) badge.terminate() } @@ -595,7 +597,7 @@ final class CodexCreditsPoolUITests: XCTestCase { XCTAssertTrue(quota.waitForExistence(timeout: 10), app.debugDescription) let renamedMenu = app.buttons["More options for \(title)"] XCTAssertTrue(renamedMenu.exists, app.debugDescription) - XCTAssertGreaterThanOrEqual(renamedMenu.frame.height, 44) + XCTAssertGreaterThanOrEqual(renamedMenu.frame.height + 0.000001, 44) XCTAssertTrue(quota.label.contains("12%")) XCTAssertEqual(quota.frame.width, savedWidth, accuracy: 1) XCTAssertEqual(quota.frame.minY, weekly.frame.minY, accuracy: 1) diff --git a/CodexBarIOSUITests/GrokSignInUITests.swift b/CodexBarIOSUITests/GrokSignInUITests.swift index 940ece8b..e4ce7c42 100644 --- a/CodexBarIOSUITests/GrokSignInUITests.swift +++ b/CodexBarIOSUITests/GrokSignInUITests.swift @@ -99,7 +99,7 @@ final class GrokSignInUITests: XCTestCase { let zeroMeter = zero.buttons["dashboard-metric-grok.included-usage"] XCTAssertTrue(zeroMeter.waitForExistence(timeout: 10), zero.debugDescription) XCTAssertTrue(zeroMeter.label.contains("0%"), zeroMeter.label) - XCTAssertTrue(zero.staticTexts["No included usage reported by Grok."].exists) + XCTAssertFalse(zero.staticTexts["Grok did not report included usage."].exists) XCTAssertTrue(zero.staticTexts["Extra Usage Credits"].exists) XCTAssertTrue(zero.buttons["dashboard-metric-cursor.grok-bot-weekly"].exists) if zeroMeter.frame.maxY > zero.frame.height * 0.8 { zero.swipeUp() } @@ -143,7 +143,7 @@ final class GrokSignInUITests: XCTestCase { let zero = launch(scenario: "grok-existing-zero") assertWeeklyBeforeCredits(in: zero, percent: "0%") - XCTAssertTrue(zero.staticTexts["No included usage reported by Grok."].exists) + XCTAssertFalse(zero.staticTexts["Grok did not report included usage."].exists) keepGrok("Existing Grok zero weekly before credits", app: zero) zero.terminate() @@ -200,7 +200,7 @@ final class GrokSignInUITests: XCTestCase { keep("Cursor retained stale data and reconnect\(suffix)", app: app) for _ in 0..<4 where !reconnect.isHittable { app.swipeUp() } XCTAssertTrue(reconnect.isHittable, app.debugDescription) - XCTAssertGreaterThanOrEqual(reconnect.frame.height, 44) + XCTAssertGreaterThanOrEqual(reconnect.frame.height + 0.000001, 44) reconnect.tap() XCTAssertTrue(app.navigationBars["Synthetic Cursor sign-in"].waitForExistence(timeout: 10), app.debugDescription) keep("Cursor guided synthetic account selection\(suffix)", app: app) diff --git a/CodexBarIOSUITests/ProCommerceJourneyTests.swift b/CodexBarIOSUITests/ProCommerceJourneyTests.swift index c9244cef..f1187697 100644 --- a/CodexBarIOSUITests/ProCommerceJourneyTests.swift +++ b/CodexBarIOSUITests/ProCommerceJourneyTests.swift @@ -374,13 +374,16 @@ final class ProCommerceNativeJourneyTests: XCTestCase { private struct ProCommerceJourneyDriver { static let proURL = URL(string: "codexbar://pro")! let app: XCUIApplication + private let nativeObserver: ProNativeWaitTrace? init(scenario: String, entry: String? = "limit", largeText: Bool = false, launch: Bool = true) { + let runID = UUID() + nativeObserver = scenario == "commerce-native-storekit" ? ProNativeWaitTrace(runID: runID) : nil app = XCUIApplication() app.launchArguments = ["-AppleLanguages", "(en)", "-AppleLocale", "en_US"] app.launchEnvironment = [ "CODEXBAR_UI_TESTS": "1", - "CODEXBAR_UI_TEST_RUN_ID": UUID().uuidString, + "CODEXBAR_UI_TEST_RUN_ID": runID.uuidString, "CODEXBAR_UI_TEST_RESET": "1", "CODEXBAR_UI_TEST_SCENARIO": scenario, "CODEXBAR_UI_TEST_DEFAULT_TEXT": largeText ? "0" : "1", @@ -400,7 +403,16 @@ private struct ProCommerceJourneyDriver { let expectation = XCTNSPredicateExpectation( predicate: NSPredicate(format: "value == %@", "ready"), object: element("codexbar-pro-presentation") ) - XCTAssertEqual(XCTWaiter.wait(for: [expectation], timeout: timeout), .completed, + if let nativeObserver { + expectation.handler = { + nativeObserver.record(.predicatePositive) + return true + } + } + nativeObserver?.record(.waitBegin, timeout: timeout) + let result = XCTWaiter.wait(for: [expectation], timeout: timeout) + nativeObserver?.record(.waitEnd, timeout: timeout, result: Int(result.rawValue)) + XCTAssertEqual(result, .completed, "Purchase information did not resolve. \(app.debugDescription)") } @@ -488,3 +500,38 @@ private struct ProCommerceJourneyDriver { XCTFail("Unreachable commerce control: \(target). UI: \(app.debugDescription)") } } + +/// XCTest's public handler is Sendable and does not promise MainActor execution. +/// Only these diagnostic fields are protected; no assertion or AX operation moves here. +private final class ProNativeWaitTrace: @unchecked Sendable { + enum Stage: String { case waitBegin, predicatePositive, waitEnd } + private let lock = NSLock() + private let runID: UUID + private let launch = UUID().uuidString + private var observedPositive = false + private var sequence = 0 + + init(runID: UUID) { self.runID = runID } + + func record(_ stage: Stage, timeout: TimeInterval = 0, result: Int = 0) { + let wallBefore = Date().timeIntervalSince1970 + let uptime = ProcessInfo.processInfo.systemUptime + let wallAfter = Date().timeIntervalSince1970 + lock.lock() + defer { lock.unlock() } + if stage == .predicatePositive { + guard !observedPositive else { return } + observedPositive = true + } + sequence += 1 + let record: [String: Any] = [ + "schema": 1, "stage": stage.rawValue, "run": runID.uuidString, "launch": launch, "sequence": sequence, + "pid": ProcessInfo.processInfo.processIdentifier, + "wallBefore": wallBefore, "uptime": uptime, "wallAfter": wallAfter, + "timeout": timeout, "result": result, + ] + guard let data = try? JSONSerialization.data(withJSONObject: record, options: [.sortedKeys]), + let message = String(data: data, encoding: .utf8) else { return } + print("CB517RUNNER \(message)") + } +} diff --git a/UI-TESTING.md b/UI-TESTING.md index a1b6afd4..13764725 100644 --- a/UI-TESTING.md +++ b/UI-TESTING.md @@ -417,14 +417,38 @@ quality jobs pass, `Full iOS UI validation` runs both device families. The release cannot proceed unless every job passes for the candidate SHA. If that SHA changes, dispatch a new run. -The two families run on independent manual workers, each with a 90-minute -limit and its own artifact. Fail-fast is disabled. An always-run aggregate -fails unless both workers succeed, including when a worker is cancelled or -skipped. Each worker requires all 53 exact journey identities plus the three -preparation/reset calls. The original runner retains its separate 29-pass contract. -Every phase requires zero skips or expected failures. GitHub retains both destinations' result bundles, logs, -summaries, and exported failure screenshots for 14 days. See -[CI-POLICY.md](CI-POLICY.md) for dispatch and SHA-verification commands. +Each family runs six independent manual shards: three original-journey partitions, +supplemental, synthetic commerce, and native commerce. The original 29 tests are +balanced using retained hosted durations into groups of 8, 8 and 13. Every shard +keeps the 90-minute limit, pinned Xcode 26.6 / 17F113 and iOS 26.2 / 23C54, +and provisions its own simulator. Fail-fast remains disabled. + +Native preparation, purchase/restore, purchase-error preparation, purchase-error +and the final normal StoreKit reset stay sequential on one worker. The reset is +attempted even after a journey failure. The two SUS evidence journeys are included +in the supplemental eight, so the full gate covers 53 distinct journeys per +family plus the three preparation/reset calls. Marketing capture remains excluded. +The original local runner retains its separate 29-pass contract. + +The always-run aggregate requires all twelve workers to succeed and validates their +compact proofs against the checked-out SHA and source fingerprint. Missing, +duplicate, stale, wrong-runtime or substituted-test proofs fail the gate. Every +phase still requires zero skips or expected failures. Full results, logs and +exported failure screenshots remain available for 14 days, separate from the small +proof artifacts used by the aggregate. See [CI-POLICY.md](CI-POLICY.md) for dispatch +and SHA-verification commands. + +Reproduce one shard from a clean committed checkout on air: + +```sh +DEVELOPER_DIR=/Applications/Xcode_26.6.app/Contents/Developer \ + python3 scripts/ui_release_shards.py run --family iphone --shard original-1 \ + --output /tmp/codexbar-ui-original-1 +``` + +Choose any shard listed in [the shard plan](scripts/ui-release-shards.json), and a +new output directory for each invocation. The command shuts down only its owned +simulator. It does not launch provider sign-in or touch live account storage. To reproduce a failure, rerun the same family with the command above. Every journey resets its own storage, so no simulator-wide data wipe is needed. @@ -534,3 +558,49 @@ disconnect and cancellation. Claude fixtures omit the unused legacy aliases. Set `CODEXBAR_UI_TEST_SUBJOURNEY` to `billing` in the copied xctestrun file to run only the guided billing recovery checks while diagnosing a connection flow. Use `renewals` for complete renewal-focused validation on both device families. + + +## Manual automatic-CI cost controls + +`ui_validation_mode=automatic-cost` runs only the existing SwiftLint job. It does +not run native quality jobs, UI workers, the full aggregate or its notifier. +Ordinary PR runs still execute checkout, pinned SwiftLint and the four policy +tests, with the existing runner, permissions and ten-minute limit. Full and +account-menu manual modes retain their native jobs, selectors and ceilings. + +Use one predeclared baseline/candidate pair. Both dispatches use the same frozen +candidate workflow ref; `comparison_base_sha` is its exact ancestor/default-base +SHA. `cost_source_role=baseline` materializes that ancestor before the existing +lint/policy workload; `candidate` materializes the exact dispatch SHA. Both first +verify an explicit `cost_expected_image` against the assigned runner's actual +`ImageVersion`. The candidate also requires `cost_expected_toolchain`, copied +from the successful baseline's `AUTOMATIC_COST_IDENTITY` log receipt. The hash +binds the default developer directory, Xcode/Swift builds, OS build, architecture, +CPU count and memory. A mismatch fails before the measured lint/policy workload; +it never silently selects another image or retries. + +The manual guard verifies that effective automatic job commands/configuration +match the baseline, rejects dirty or wrong source, and records exact source and +tool identities. Only this manual mode fetches history for ancestor validation. +The normal automatic checkout remains depth one with its original default ref. +Only cost mode explicitly selects `github.sha`, so a branch dispatch creates a +verified detached checkout of the immutable controller commit. After the identical candidate +checkout, both controls retain the verified controller outside the workspace, +remove the same candidate tracked paths, verify the index and worktree are empty +(except `.git`), and materialize their selected revision from that state. Ignored +or untracked leftovers fail rather than being silently deleted or reused. This +avoids charging only the baseline for rewriting an already populated candidate +checkout. All removal/materialization work still counts. Receipts bind the +common controller/start and empty-state checks to each measured source. The +Actions run SHA identifies the workflow/controller, while `measuredSource` +identifies the source actually linted. A baseline cost run is not a current-head +source-validation result. + +Acceptance remains completed Actions **workflow elapsed time and summed +non-skipped job runtime**: candidate must be no larger in either measure. Count +checkout, identity checks, source selection and all setup/cleanup time; make no +queue/network/guard deductions, tolerance, per-file normalization or synthetic +job-time claim. Both must pass on the same image, toolchain and hardware config. +Retain original adverse observations and any invalid/failed pair. Do not repeat +a pair or launch full native validation merely to obtain a green timing. A +successful cost-only run does not qualify any native/full release gate. diff --git a/docs/ci/issue-517-baseline.json b/docs/ci/issue-517-baseline.json new file mode 100644 index 00000000..e93baaf0 --- /dev/null +++ b/docs/ci/issue-517-baseline.json @@ -0,0 +1,82 @@ +{ + "manual_run": "https://github.com/hemsoft-dev/codexbar-ios/actions/runs/39810829566", + "manual_head": "9995bc14ee090d4688a7d31ae1a210dd31b78aad", + "ipad_original_step_seconds": 7214, + "original_test_seconds": { + "CodexBarIOSUITests/AccountJourneysUITests/testAccountSetupPersistsGroupAndCredentialAtAccessibilitySize": 190.38641798496246, + "CodexBarIOSUITests/AccountJourneysUITests/testCodingOnlyGeminiChoicesSurvivePartialAndFailedRefresh": 185.9075390100479, + "CodexBarIOSUITests/AccountJourneysUITests/testGeminiOnlyDashboardShowsCodingSetupAndKeepsSelectionOnRelaunch": 146.40053403377533, + "CodexBarIOSUITests/AccountJourneysUITests/testGitHubBillingActionableWarningStaysInline": 11.181080937385559, + "CodexBarIOSUITests/AccountJourneysUITests/testGitHubBillingAllowanceStates": 49.49990403652191, + "CodexBarIOSUITests/AccountJourneysUITests/testGitHubBillingPermissionDisclosure": 142.06247901916504, + "CodexBarIOSUITests/AccountJourneysUITests/testGitHubBillingProductSummaryAndRoutinePlacement": 56.41284894943237, + "CodexBarIOSUITests/AccountJourneysUITests/testRefreshRecoveryHistoryAndAccountDeepLink": 55.12314808368683, + "CodexBarIOSUITests/AccountJourneysUITests/testSavedGeminiRingIsSelectedInVisualizationMenu": 84.94209694862366, + "CodexBarIOSUITests/AccountJourneysUITests/testSixGoogleChoicesAndIndependentCustomizationPersist": 234.73875200748444, + "CodexBarIOSUITests/AccountJourneysUITests/testTwoCodexAccountsKeepSeparateUsageAfterRelaunch": 74.75701797008514, + "CodexBarIOSUITests/ClaudeUsageUITests/testWindowLabelsAndSavedCustomizationForProAndMax": 1603.8373860120773, + "CodexBarIOSUITests/CodexCreditsPoolUITests/testCompactCardHeadersAndIndependentControls": 1620.3830180168152, + "CodexBarIOSUITests/CodexCreditsPoolUITests/testOptInBalanceStatesAndSavedAccountChoices": 399.53378999233246, + "CodexBarIOSUITests/CodexCreditsPoolUITests/testPreciseSubscriptionPillsOnExpandedAndCollapsedCards": 267.53720700740814, + "CodexBarIOSUITests/CodexCreditsPoolUITests/testThirtyDayWindowLabelAndSavedVisibility": 122.59079003334045, + "CodexBarIOSUITests/GreptileAllowanceUITests/testDetailReconnectStaysInApp": 25.48348903656006, + "CodexBarIOSUITests/GreptileAllowanceUITests/testReviewHistoryAndBillingAvailabilityStates": 451.6969130039215, + "CodexBarIOSUITests/GrokSignInUITests/testCursorFreshPercentagesBotAndSavedChoices": 94.11898100376129, + "CodexBarIOSUITests/GrokSignInUITests/testCursorStaleSignInReconnectCancellationAndValidZero": 71.34450507164001, + "CodexBarIOSUITests/GrokSignInUITests/testGrokAndCursorMetersStaySeparateAndNoAllowanceIsUnavailable": 41.37902390956879, + "CodexBarIOSUITests/GrokSignInUITests/testGuidedSetupCancellationReconnectAndRemoval": 35.021477937698364, + "CodexBarIOSUITests/GrokSignInUITests/testSavedGrokLayoutAndWeeklyStates": 71.8157240152359, + "CodexBarIOSUITests/OpenCodeSignInUITests/testDisconnectedAccountOffersGuidedSignIn": 92.38425695896149, + "CodexBarIOSUITests/OpenCodeSignInUITests/testVerificationFailureKeepsAccountDisconnectedAndAllowsRetry": 255.2149029970169, + "CodexBarIOSUITests/SettingsEvidenceUITests/testEverySettingsDestinationAndDoneBack": 64.60541903972626, + "CodexBarIOSUITests/SettingsEvidenceUITests/testPendingDuplicateGroupBlocksDoneUntilCorrected": 46.53203594684601, + "CodexBarIOSUITests/SettingsEvidenceUITests/testSettingsDismissalSignalRejectsTheBackgroundGear": 15.621724009513855, + "CodexBarIOSUITests/SettingsEvidenceUITests/testSyntheticGrantedAndDeniedNotificationFeedback": 114.02973997592926 + }, + "automatic_run": "https://github.com/hemsoft-dev/codexbar-ios/actions/runs/40010770987", + "automatic_head": "9dec9e34b8b01093624ee714e76e129c5d5eb7bd", + "automatic_workflow_elapsed_seconds": 3215.0, + "automatic_summed_job_seconds": 43.0, + "automatic_jobs": [ + { + "name": "SwiftLint", + "seconds": 43.0, + "conclusion": "success" + }, + { + "name": "Strict concurrency", + "seconds": 0, + "conclusion": "skipped" + }, + { + "name": "watchOS tests", + "seconds": 0, + "conclusion": "skipped" + }, + { + "name": "iOS tests", + "seconds": 0, + "conclusion": "skipped" + }, + { + "name": "SwiftPM smoke tests", + "seconds": 0, + "conclusion": "skipped" + }, + { + "name": "iOS UI journeys (${{ matrix.family }})", + "seconds": 0, + "conclusion": "skipped" + }, + { + "name": "Account menu comparison", + "seconds": 0, + "conclusion": "skipped" + }, + { + "name": "Full iOS UI validation", + "seconds": 0, + "conclusion": "skipped" + } + ] +} diff --git a/scripts/prepare-automatic-cost.py b/scripts/prepare-automatic-cost.py new file mode 100644 index 00000000..a6228d83 --- /dev/null +++ b/scripts/prepare-automatic-cost.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Prepare one manual cost control; normal PR CI never calls this script.""" + +import argparse +import hashlib +import json +import os +import re +import subprocess +from pathlib import Path + + +MANUAL_GUARD = """ - name: Verify manual automatic-cost identity + if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode == 'automatic-cost' }} + env: + AUTOMATIC_COST_MODE: ${{ inputs.ui_validation_mode }} + AUTOMATIC_COST_ROLE: ${{ inputs.cost_source_role }} + AUTOMATIC_COST_BASE: ${{ inputs.comparison_base_sha }} + AUTOMATIC_COST_IMAGE: ${{ inputs.cost_expected_image }} + AUTOMATIC_COST_TOOLCHAIN: ${{ inputs.cost_expected_toolchain }} + run: | + cp scripts/prepare-automatic-cost.py "$RUNNER_TEMP/automatic-cost-controller.py" + python3 "$RUNNER_TEMP/automatic-cost-controller.py" --receipt "$RUNNER_TEMP/automatic-cost.json" + +""" + +class Refused(ValueError): + pass + + +def command(*args): + result = subprocess.run(args, check=True, capture_output=True, text=True, timeout=10) + if len(result.stdout.encode()) > 1_048_576: + raise Refused('command-output-cap') + return result.stdout.strip() + + +def automatic_job(workflow): + match = re.search(r'^ swiftlint:\n.*?(?=^ [\w-]+:\n|\Z)', workflow, re.M | re.S) + if not match: + raise Refused('missing-automatic-job') + block = match.group() + # These two declarations affect only the explicit manual comparison mode. + block = re.sub( + r"^ fetch-depth: \$\{\{ inputs.ui_validation_mode == 'automatic-cost' && '0' \|\| '1' \}\}\n", + '', block, flags=re.M, + ) + block = block.replace(" ref: ${{ inputs.ui_validation_mode == 'automatic-cost' && github.sha || '' }}\n", '', 1) + block = block.replace(MANUAL_GUARD, '', 1) + return block + + +def prepare(env, receipt): + if (env.get('GITHUB_ACTIONS') != 'true' + or env.get('GITHUB_EVENT_NAME') != 'workflow_dispatch' + or env.get('GITHUB_REPOSITORY') != 'hemsoft-dev/codexbar-ios' + or env.get('AUTOMATIC_COST_MODE') != 'automatic-cost'): + raise Refused('wrong-context') + candidate = env.get('GITHUB_SHA', '') + baseline = env.get('AUTOMATIC_COST_BASE', '') + role = env.get('AUTOMATIC_COST_ROLE') + if not all(re.fullmatch(r'[0-9a-f]{40}', x) for x in (candidate, baseline)): + raise Refused('invalid-source-sha') + if candidate == baseline or role not in ('baseline', 'candidate'): + raise Refused('invalid-source-role') + image = env.get('ImageVersion', '') + if not re.fullmatch(r'\d{8}\.\d{4}\.\d+', image) or image != env.get('AUTOMATIC_COST_IMAGE'): + raise Refused('image-mismatch') + if not env.get('ImageOS', '').startswith('macos26'): + raise Refused('wrong-runner-os') + developer = env.get('DEVELOPER_DIR', '') + if developer != '/Applications/Xcode.app/Contents/Developer': + raise Refused('different-automatic-toolchain-path') + root = Path(command('git', 'rev-parse', '--show-toplevel')).resolve() + if (root != Path.cwd().resolve() or root != Path(env.get('GITHUB_WORKSPACE', '/nonexistent')).resolve() + or command('git', 'rev-parse', 'HEAD') != candidate): + raise Refused('wrong-checkout') + if command('git', 'rev-parse', '--abbrev-ref', 'HEAD') != 'HEAD': + raise Refused('attached-checkout') + if command('git', 'status', '--porcelain'): + raise Refused('dirty-checkout') + command('git', 'merge-base', '--is-ancestor', baseline, candidate) + if receipt.resolve().parent != Path(env.get('RUNNER_TEMP', '/nonexistent')).resolve(): + raise Refused('receipt-outside-runner-temp') + controller = Path(__file__).resolve() + if controller.parent != receipt.resolve().parent: + raise Refused('controller-not-retained') + if controller.read_text().strip() != command('git', 'show', f'{candidate}:scripts/prepare-automatic-cost.py'): + raise Refused('controller-source-mismatch') + controller_sha = hashlib.sha256(controller.read_bytes()).hexdigest() + workflow = '.github/workflows/ci.yml' + before = automatic_job(command('git', 'show', f'{baseline}:{workflow}') + '\n') + after = automatic_job(command('git', 'show', f'{candidate}:{workflow}') + '\n') + if before != after: + raise Refused('automatic-workload-changed') + tools = { + 'developerDirectory': developer, + 'resolvedDeveloperDirectory': str(Path(developer).resolve()), + 'xcodeBuild': command('xcodebuild', '-version'), + 'swiftVersion': command('xcrun', 'swift', '--version'), + 'osVersion': command('sw_vers', '-productVersion'), + 'osBuild': command('sw_vers', '-buildVersion'), + 'architecture': command('uname', '-m'), + 'cpuCount': command('sysctl', '-n', 'hw.ncpu'), + 'memoryBytes': command('sysctl', '-n', 'hw.memsize'), + } + if any(len(value) > 4096 for value in tools.values()): + raise Refused('toolchain-output-cap') + fingerprint = hashlib.sha256(json.dumps(tools, sort_keys=True).encode()).hexdigest() + expected = env.get('AUTOMATIC_COST_TOOLCHAIN', '') + if role == 'candidate' and not re.fullmatch(r'[0-9a-f]{64}', expected): + raise Refused('missing-baseline-toolchain') + if expected and expected != fingerprint: + raise Refused('toolchain-mismatch') + source = baseline if role == 'baseline' else candidate + removed = len(command('git', 'ls-files', '-z').rstrip('\0').split('\0')) + command('git', 'rm', '-r', '-f', '--quiet', '--', '.') + if command('git', 'ls-files') or any(path.name != '.git' for path in root.iterdir()): + raise Refused('materialization-start-not-empty') + command('git', 'checkout', '--force', '--detach', source) + if command('git', 'rev-parse', 'HEAD') != source or command('git', 'status', '--porcelain'): + raise Refused('source-switch-unverified') + result = { + 'schema': 1, 'role': role, 'baseline': baseline, 'candidate': candidate, + 'measuredSource': source, 'imageVersion': image, 'imageOS': env['ImageOS'], + 'toolchain': tools, 'toolchainSHA256': fingerprint, + 'automaticJobSHA256': hashlib.sha256(before.encode()).hexdigest(), + 'sourcePreparation': {'controllerSHA256': controller_sha, 'commonStartingHEAD': candidate, + 'removedCandidatePaths': removed, 'indexEntriesBeforeMaterialization': 0, + 'worktreeEntriesBeforeMaterialization': 0}, + 'measurement': 'Complete Actions run elapsed and summed non-skipped job runtime; no deductions', + } + receipt.write_text(json.dumps(result, indent=2) + '\n') + print('AUTOMATIC_COST_IDENTITY=' + json.dumps(result, sort_keys=True)) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--receipt', type=Path, required=True) + args = parser.parse_args() + try: + prepare(os.environ, args.receipt) + except Refused as error: + print('AUTOMATIC_COST_REFUSED=' + str(error)) + return 1 + except (OSError, subprocess.SubprocessError): + print('AUTOMATIC_COST_REFUSED=operational-or-command-failure') + return 1 + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/pro_release_runner.py b/scripts/pro_release_runner.py index 113c2aa9..5f1dc5e4 100755 --- a/scripts/pro_release_runner.py +++ b/scripts/pro_release_runner.py @@ -1,8 +1,15 @@ #!/usr/bin/env python3 """Explicit manual release phases; owns only newly provisioned simulators.""" import argparse +from datetime import datetime, timedelta, timezone import json +import math import os +import re +import selectors +import shutil +import signal +import time from pathlib import Path import subprocess import sys @@ -192,9 +199,280 @@ def binaries(derived, signed): return result +# Native observation is manual-only and cannot alter XCTest's verdict. +OBSERVER_PHASES = {'ui-native', 'ui-native-purchase-error'} +OBSERVER_SUBSYSTEM = 'com.hemsoft.CodexBarIOS.native-readiness' +OBSERVER_PREFIX = 'CB517NATIVE ' +OBSERVER_LIMITS = dict(commandSeconds=15, phaseSeconds=60, stdoutBytes=8 * 1024 * 1024, + # Known retained export is197059893bytes; only the bounded session survives staging. + exportBytes=256 * 1024 * 1024, exportFiles=2048, sessionBytes=32 * 1024 * 1024) + + +def observer_export_cap(export): + if export and export.exists(): + size = 0 + for count, path in enumerate(export.rglob('*'), 1): + if count > OBSERVER_LIMITS['exportFiles'] or path.is_symlink(): + raise ValueError('Observer export byte/file cap exceeded') + if path.is_file(): + size += path.stat().st_size + if size > OBSERVER_LIMITS['exportBytes']: + raise ValueError('Observer export byte/file cap exceeded') + + +def observer_clock(row): + keys = ('wallBefore', 'uptime', 'wallAfter') + if (row.get('schema') != 1 or any(type(row.get(key)) not in (int, float) + or not math.isfinite(row[key]) for key in keys) or row['wallAfter'] < row['wallBefore']): + raise ValueError('Invalid observer clock tuple') + + +def observer_command(folder, label, argv, deadline, export=None): + """Finite process-group lifetime and byte/file caps, including partial output.""" + start = time.monotonic() + receipt = dict(command=argv, started=now(), limits=OBSERVER_LIMITS, complete=False, truncated=False, timedOut=False) + path = folder / (label + '.json') + failure, process = None, None + stdout, stderr = bytearray(), bytearray() + try: + if time.monotonic() >= deadline: + raise ValueError('Observer phase deadline exhausted before command') + with selectors.DefaultSelector() as selector: + process = subprocess.Popen(argv, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True) + selector.register(process.stdout, selectors.EVENT_READ, stdout) + selector.register(process.stderr, selectors.EVENT_READ, stderr) + end = min(deadline, time.monotonic() + OBSERVER_LIMITS['commandSeconds']) + while selector.get_map(): + if time.monotonic() >= end: + raise ValueError('Observer command deadline exceeded') + observer_export_cap(export) + for key, _ in selector.select(timeout=min(0.05, max(0, end - time.monotonic()))): + block = os.read(key.fileobj.fileno(), 65536) + if not block: + selector.unregister(key.fileobj) + else: + remaining = OBSERVER_LIMITS['stdoutBytes'] - len(stdout) - len(stderr) + key.data.extend(block[:max(0, remaining)]) + if len(block) > remaining: + raise ValueError('Observer output byte cap exceeded; partial output retained') + process.wait(timeout=max(0.001, end - time.monotonic())) + observer_export_cap(export) + if process.returncode: + raise ValueError(f'Observer command exited {process.returncode}') + receipt['complete'] = True + except (OSError, ValueError, subprocess.TimeoutExpired) as error: + failure = PhaseFailure(str(error)) + receipt.update(error=str(error), truncated='cap exceeded' in str(error), + timedOut='deadline' in str(error) or isinstance(error, subprocess.TimeoutExpired)) + finally: + cleanup_errors = [] + if process: + # Each step uses only this retained Popen handle, once. A failed + # kill/wait/close must not skip the other cleanup or erase evidence. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + except OSError as error: + cleanup_errors.append(dict(step='kill', error=str(error))) + try: + process.wait(timeout=2) + except (OSError, subprocess.TimeoutExpired) as error: + cleanup_errors.append(dict(step='wait', error=str(error))) + receipt['exit'] = process.returncode + for name, pipe in (('stdout', process.stdout), ('stderr', process.stderr)): + try: + pipe.close() + except OSError as error: + cleanup_errors.append(dict(step=name, error=str(error))) + if cleanup_errors: + receipt.update(complete=False, cleanupErrors=cleanup_errors) + details = '; '.join(item['step'] + ': ' + item['error'] for item in cleanup_errors) + message = 'Observer cleanup failed: ' + details + failure = PhaseFailure(f'{failure}; {message}' if failure else message, getattr(failure, 'code', 1)) + receipt.setdefault('error', message) + receipt.update(ended=now(), seconds=time.monotonic() - start, stdoutBytes=len(stdout), stderrBytes=len(stderr)) + write_errors = [] + for name, data in (('stdout', stdout), ('stderr', stderr)): + try: + (folder / (label + '.' + name)).write_bytes(data) + except OSError as error: + write_errors.append(dict(stream=name, error=str(error))) + if write_errors: + receipt.update(complete=False, writeErrors=write_errors) + details = '; '.join(item['stream'] + ': ' + item['error'] for item in write_errors) + message = 'Observer output write failed: ' + details + failure = PhaseFailure(f'{failure}; {message}' if failure else message, getattr(failure, 'code', 1)) + receipt.setdefault('error', message) + failure = persist(path, receipt, failure) + if failure: + raise failure + return bytes(stdout) + + +def observer_runner_records(path): + if path.stat().st_size > 32 * 1024 * 1024: + raise ValueError('Native XCTest log exceeds observer parse cap') + records = [] + for line in path.read_text().splitlines(): + if 'CB517RUNNER ' not in line: + continue + record = json.loads(line.split('CB517RUNNER ', 1)[1]) + if (not isinstance(record, dict) or record.get('stage') not in ('waitBegin', 'predicatePositive', 'waitEnd') + or not re_uuid(record.get('run', '')) or type(record.get('pid')) is not int): + raise ValueError('Invalid runner observation identity') + observer_clock(record) + records.append(record) + if not records or not {'waitBegin', 'waitEnd'} <= {row['stage'] for row in records}: + raise ValueError('Missing native wait boundary observations') + return records + + +def observer_log_bounds(receipt): + start, end = [datetime.fromisoformat(receipt[key]).astimezone(timezone.utc) for key in ('started', 'ended')] + # log show accepts whole-second dates. Query outward, then enforce the exact phase below. + stop = end.replace(microsecond=0) + (timedelta(seconds=1) if end.microsecond else timedelta()) + return [value.strftime('%Y-%m-%d %H:%M:%S%z') for value in (start.replace(microsecond=0), stop)] + + +def observer_event_in_phase(event, receipt): + if not isinstance(event, dict) or not isinstance(event.get('timestamp'), str): + raise ValueError('Missing observer event timestamp') + timestamp = datetime.strptime(event['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z') + if timestamp.utcoffset() is None: + raise ValueError('Observer event timestamp lacks timezone') + return datetime.fromisoformat(receipt['started']) <= timestamp <= datetime.fromisoformat(receipt['ended']) + + +def collect_native_observer(runner, folder, result, receipt): + """Read only the owned simulator, correlate app/runner UUID+PID, export before final hash.""" + output = folder / 'native-observer' + output.mkdir() + report = dict(schema=1, complete=False, limits=OBSERVER_LIMITS, source=runner.source, + device=runner.device, tools=runner.tools, phase=receipt['phase'], + started=receipt['started'], ended=receipt['ended']) + failure = None + staging = output / 'offline-export' + try: + deadline = time.monotonic() + OBSERVER_LIMITS['phaseSeconds'] + require_pinned(runner.tools, runner.device) + developer = Path(runner.tools['developer_dir']).resolve() + if Path(os.environ.get('DEVELOPER_DIR', '')).resolve() != developer: + raise ValueError('Observer DEVELOPER_DIR differs from qualified build') + ownership = read(runner.ownership) + created = ownership.get('created', {}) + argv = created.get('command', []) + if (ownership.get('owner') != str(ROOT) or ownership.get('tools') != runner.tools + or created.get('exit') != 0 + or len(argv) != 6 or argv[:3] != ['xcrun', 'simctl', 'create'] + or argv[3] != runner.device['deviceName'] or argv[5] != runner.device['runtimeBundlePath'] + or not re_uuid(runner.device['deviceId'])): + raise ValueError('Observer lacks exact simulator ownership') + validate_device(ownership['device'], runner.device) + if ownership['device']['deviceName'] != runner.device['deviceName']: + raise ValueError('Observer simulator name changed') + used = read(runner.ownership.parent / 'native-use.json') + if used.get('deviceId') != runner.device['deviceId'] or used.get('output') != str(runner.output): + raise ValueError('Observer native claim differs') + for tool in ('simctl', 'xcresulttool'): + found = observer_command(output, 'find-' + tool, ['/usr/bin/xcrun', '--find', tool], deadline) + if Path(found.decode().strip()).resolve() != developer / 'usr/bin' / tool: + raise ValueError('Observer tool outside qualified developer directory') + records = observer_runner_records(folder / 'xcodebuild.log') + runs = {row['run'] for row in records} + if len(runs) != 1 or len({row['pid'] for row in records}) != 1: + raise ValueError('Ambiguous native runner identity') + report['runner'] = records + interval = observer_log_bounds(receipt) + report['queryInterval'] = interval + prefix = ['/usr/bin/xcrun', 'simctl', 'spawn', runner.device['deviceId'], 'log', 'show', '--style', 'json', + '--start', interval[0], '--end', interval[1]] + raw = observer_command(output, 'app-trace', prefix + ['--predicate', f'subsystem == "{OBSERVER_SUBSYSTEM}"'], deadline) + events = json.loads(raw) + if not isinstance(events, list): + raise ValueError('Observer log is not a JSON event list') + app = [] + report['appOutsidePhaseCount'] = 0 + for event in events: + if not isinstance(event, dict): + raise ValueError('Invalid app log event shape') + if not observer_event_in_phase(event, receipt): + report['appOutsidePhaseCount'] += 1 + continue + message = event.get('eventMessage', '') + if not isinstance(message, str): + raise ValueError('Invalid app event message shape') + if not message.startswith(OBSERVER_PREFIX): + continue + row = json.loads(message[len(OBSERVER_PREFIX):]) + if (not isinstance(row, dict) or row.get('run') not in runs or row.get('pid') != event.get('processID') + or not event.get('processImagePath', '').endswith('/CodexBarIOS.app/CodexBarIOS') + or not re_uuid(row.get('launch', ''))): + raise ValueError('Observer app identity differs from runner') + observer_clock(row) + app.append(dict(marker=row, timestamp=event.get('timestamp'), bootUUID=event.get('bootUUID'), + processImageUUID=event.get('processImageUUID'))) + if not app or len({row['marker']['pid'] for row in app}) != 1 or len({row['marker']['launch'] for row in app}) != 1: + raise ValueError('Missing or ambiguous native app trace') + report['app'] = app + app_pid = app[0]['marker']['pid'] + # Strictly this app, not storekitd/trustd or other applications/accounts. + predicate = f'processID == {app_pid} AND (subsystem == "com.apple.securityd" OR subsystem == "com.apple.storekit")' + security = observer_command(output, 'app-sdk', prefix + ['--predicate', predicate], deadline) + sdk_events = json.loads(security) + if not isinstance(sdk_events, list): + raise ValueError('App SDK log is not a JSON list') + exact_sdk = [event for event in sdk_events if observer_event_in_phase(event, receipt)] + report['sdkOutsidePhaseCount'] = len(sdk_events) - len(exact_sdk) + report['sdkEventCount'] = len(exact_sdk) + if not result.exists(): + raise ValueError('No native xcresult for offline session export') + observer_command(output, 'session-export', ['/usr/bin/xcrun', 'xcresulttool', 'export', 'diagnostics', + '--path', str(result), '--output-path', str(staging)], deadline, export=staging) + # Offline extraction of already-captured xcresult diagnostics, no new sysdiagnose/host collection. + # Retain only this test bundle's Session logs. Other exported copies are removed below. + sessions = [path for path in staging.rglob('Session-*.log') + if any(part.startswith('CodexBarIOSUITests-') for part in path.parts)] + if len(sessions) != 1: + raise ValueError('Missing or ambiguous XCTest session log') + report['sessionBytes'] = sessions[0].stat().st_size + if report['sessionBytes'] > OBSERVER_LIMITS['sessionBytes']: + raise ValueError('XCTest session byte cap exceeded') + target = output / 'xctest-session.log' + shutil.copyfile(sessions[0], target) + session_text = target.read_text() + runner_pid = records[0]['pid'] + if f'CodexBarIOSUITests-Runner[{runner_pid}:' not in session_text or not re.search(rf'pid:\s*{app_pid}\b', session_text): + raise ValueError('XCTest session does not bind observed runner/app PIDs') + report['sessionSha256'] = digest(target) + stages = {row['marker']['stage'] for row in app} + if receipt.get('exit') == 0 and (not {'snapshotBegin', 'snapshotEnd', 'catalogBegin', 'catalogEnd', 'viewReadiness'} <= stages + or 'predicatePositive' not in {row['stage'] for row in records} + or not any(row['marker']['stage'] == 'viewReadiness' and row['marker'].get('count', 0) & 1 for row in app)): + raise ValueError('Passing native run lacks required app observations') + report['complete'] = True + except (ValueError, KeyError, TypeError, OSError, PhaseFailure) as error: + failure = error if isinstance(error, PhaseFailure) else PhaseFailure(str(error)) + report['error'] = str(error) + finally: + try: + if staging.exists(): + shutil.rmtree(staging) + except OSError as error: + report.update(complete=False, cleanupError=str(error)) + failure = failure or PhaseFailure(str(error)) + failure = persist(output / 'receipt.json', report, failure) + if failure: + raise failure + return report + + + class Runner: - def __init__(self, output, device, source, tools): + def __init__(self, output, device, source, tools, ownership=None): self.output, self.device, self.source, self.tools = output, device, source, tools + self.ownership = Path(ownership).resolve() if ownership is not None else output / 'ownership.json' self.manifest = load_manifest() self.receipts = [] @@ -311,6 +589,12 @@ def phase(self, name, build_path, base=False): failure = error if isinstance(error, PhaseFailure) else PhaseFailure(str(error)) receipt.update(exit=failure.code, error=str(error)) receipt['ended'] = now() + if name in OBSERVER_PHASES: + try: + receipt['nativeObserver'] = collect_native_observer(self, folder, result, receipt) + except (PhaseFailure, ValueError, KeyError, TypeError, OSError) as error: + receipt['observerError'] = str(error) + failure = failure or (error if isinstance(error, PhaseFailure) else PhaseFailure(str(error))) try: if result.exists(): receipt['result_initial_sha256'] = digest(result) @@ -415,7 +699,7 @@ def main(): shutdown(output, device) outcome.update(exit=0, complete=True) else: - runner = Runner(output, device, source, tools) + runner = Runner(output, device, source, tools, ownership=args.ownership) manifest = runner.manifest if args.mode == 'base': # Build the ordinary scheme's full test bundle (35 methods), then diff --git a/scripts/tests/manual/test_automatic_cost.py b/scripts/tests/manual/test_automatic_cost.py new file mode 100644 index 00000000..a384628a --- /dev/null +++ b/scripts/tests/manual/test_automatic_cost.py @@ -0,0 +1,242 @@ +"""Manual-only identity/failure controls for the actual cost preparation script.""" + +import contextlib +import hashlib +import importlib.util +import io +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location('cost', ROOT / 'prepare-automatic-cost.py') +COST = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(COST) +WORKFLOW = (ROOT.parent / '.github/workflows/ci.yml').read_text() + + +class CostControls(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.folder = Path(self.temp.name) + self.repo = self.folder / 'repo' + self.repo.mkdir() + self.previous = Path.cwd() + os.chdir(self.repo) + self.addCleanup(os.chdir, self.previous) + self.real_command = COST.command + for args in (('init', '-q'), ('config', 'user.name', 'Fixture'), + ('config', 'user.email', 'fixture@example.invalid')): + self.git(*args) + path = self.repo / '.github/workflows/ci.yml' + path.parent.mkdir(parents=True) + # Same effective automatic job, before the manual guard was introduced. + original = COST.automatic_job(WORKFLOW) + modified = WORKFLOW[WORKFLOW.index(' swiftlint:\n'):WORKFLOW.index(' strict-concurrency:\n')] + self.before = WORKFLOW.replace(modified, original) + path.write_text(self.before) + self.git('add', '.') + self.git('commit', '-qm', 'baseline') + self.base = self.git('rev-parse', 'HEAD') + path.write_text(WORKFLOW) + (self.repo / 'candidate.txt').write_text('candidate source\n') + controller = self.repo / 'scripts/prepare-automatic-cost.py' + controller.parent.mkdir(parents=True) + controller.write_bytes(Path(COST.__file__).read_bytes()) + self.controller = self.folder / 'controller.py' + self.controller.write_bytes(controller.read_bytes()) + self.git('add', '.') + self.git('commit', '-qm', 'candidate') + self.head = self.git('rev-parse', 'HEAD') + self.git('checkout', '--detach', self.head) + self.env = dict( + GITHUB_ACTIONS='true', GITHUB_EVENT_NAME='workflow_dispatch', + GITHUB_REPOSITORY='hemsoft-dev/codexbar-ios', GITHUB_SHA=self.head, + GITHUB_WORKSPACE=str(self.repo), + AUTOMATIC_COST_MODE='automatic-cost', AUTOMATIC_COST_ROLE='baseline', + AUTOMATIC_COST_BASE=self.base, AUTOMATIC_COST_IMAGE='20261021.0445.8', + AUTOMATIC_COST_TOOLCHAIN='', ImageVersion='20261021.0445.8', ImageOS='macos26-arm64', + DEVELOPER_DIR='/Applications/Xcode.app/Contents/Developer', RUNNER_TEMP=str(self.folder), + ) + self.calls = [] + self.materialization_states = [] + + def git(self, *args): + return subprocess.check_output(['git', *args], text=True, stderr=subprocess.PIPE).strip() + + def command(self, *args): + self.calls.append(args) + if args[:2] == ('git', 'checkout'): + self.materialization_states.append({ + 'index': self.git('ls-files'), + 'entries': sorted(p.name for p in self.repo.iterdir() if p.name != '.git'), + }) + if args == ('xcodebuild', '-version'): + return 'Xcode fixture\nBuild version fixture' + if args == ('xcrun', 'swift', '--version'): + return 'Swift fixture\nTarget: arm64-apple-macosx' + return self.real_command(*args) + + def run_control(self): + with patch.object(COST, '__file__', str(self.controller)), \ + patch.object(COST, 'command', self.command), contextlib.redirect_stdout(io.StringIO()): + return COST.prepare(self.env, self.folder / 'receipt.json') + + def test_real_git_baseline_then_candidate_bind_receipts_and_toolchain(self): + baseline = self.run_control() + self.assertEqual(self.git('rev-parse', 'HEAD'), self.base) + self.assertEqual(baseline['measuredSource'], self.base) + self.git('checkout', '--detach', self.head) + self.env.update(AUTOMATIC_COST_ROLE='candidate', AUTOMATIC_COST_TOOLCHAIN=baseline['toolchainSHA256']) + candidate = self.run_control() + self.assertEqual(candidate['measuredSource'], self.head) + self.assertEqual(candidate['toolchainSHA256'], baseline['toolchainSHA256']) + self.assertEqual(candidate['automaticJobSHA256'], hashlib.sha256(COST.automatic_job(WORKFLOW).encode()).hexdigest()) + self.assertEqual(json.loads((self.folder / 'receipt.json').read_text()), candidate) + self.assertEqual(self.git('status', '--porcelain'), '') + + def test_both_roles_materialize_from_the_same_empty_state(self): + baseline = self.run_control() + self.git('checkout', '--force', '--detach', self.head) + self.env.update(AUTOMATIC_COST_ROLE='candidate', AUTOMATIC_COST_TOOLCHAIN=baseline['toolchainSHA256']) + self.run_control() + self.assertEqual(len(self.materialization_states), 2) + self.assertEqual(self.materialization_states, [{'index': '', 'entries': []}] * 2) + self.assertTrue((self.repo / 'candidate.txt').is_file()) + self.assertEqual(self.git('status', '--porcelain'), '') + + def test_attached_checkout_is_never_cleared(self): + self.git('checkout', '-b', 'owned-branch') + with self.assertRaisesRegex(COST.Refused, 'attached-checkout'): + self.run_control() + self.assertTrue((self.repo / 'candidate.txt').exists()) + self.assertEqual(self.git('status', '--porcelain'), '') + + def test_modified_retained_controller_is_refused_before_clearing(self): + self.controller.write_text('different controller') + with self.assertRaisesRegex(COST.Refused, 'controller-source-mismatch'): + self.run_control() + self.assertTrue((self.repo / 'candidate.txt').exists()) + + def test_ignored_leftover_cannot_qualify_empty_materialization(self): + (self.repo / '.git/info/exclude').write_text('ignored-cache/\n') + ignored = self.repo / 'ignored-cache' + ignored.mkdir() + (ignored / 'retained').write_text('owned fixture') + with self.assertRaisesRegex(COST.Refused, 'materialization-start-not-empty'): + self.run_control() + self.assertTrue((ignored / 'retained').exists()) + self.assertEqual(self.materialization_states, []) + self.assertFalse((self.folder / 'receipt.json').exists()) + + def test_wrong_image_refused_before_any_command_or_source_switch(self): + self.env['ImageVersion'] = '20261013.0424.1' + with self.assertRaisesRegex(COST.Refused, 'image-mismatch'): + self.run_control() + self.assertEqual(self.calls, []) + self.assertEqual(self.git('rev-parse', 'HEAD'), self.head) + + def test_wrong_context_and_invalid_sources_never_run_commands(self): + for key, value in (('GITHUB_EVENT_NAME', 'pull_request'), ('GITHUB_REPOSITORY', 'other/repo'), + ('AUTOMATIC_COST_ROLE', 'other'), ('AUTOMATIC_COST_BASE', 'main; echo unsafe')): + with self.subTest(key=key), patch.dict(self.env, {key: value}): + with self.assertRaises(COST.Refused): + self.run_control() + self.assertEqual(self.calls, []) + + def test_dirty_or_wrong_checkout_refused(self): + (self.repo / 'untracked').write_text('owned test') + with self.assertRaisesRegex(COST.Refused, 'dirty-checkout'): + self.run_control() + (self.repo / 'untracked').unlink() + self.git('checkout', '--detach', self.base) + with self.assertRaisesRegex(COST.Refused, 'wrong-checkout'): + self.run_control() + + def test_missing_or_different_toolchain_does_not_switch_source(self): + self.env['AUTOMATIC_COST_ROLE'] = 'candidate' + for value, expected in (('', 'missing-baseline-toolchain'), ('a' * 64, 'toolchain-mismatch')): + self.env['AUTOMATIC_COST_TOOLCHAIN'] = value + with self.assertRaisesRegex(COST.Refused, expected): + self.run_control() + self.assertEqual(self.git('rev-parse', 'HEAD'), self.head) + self.assertFalse((self.folder / 'receipt.json').exists()) + + def test_changed_automatic_command_fails_before_workload(self): + path = self.repo / '.github/workflows/ci.yml' + path.write_text(WORKFLOW.replace('--reporter github-actions-logging', '--reporter json')) + self.git('add', '.') + self.git('commit', '-qm', 'different automatic workload') + self.head = self.git('rev-parse', 'HEAD') + self.env['GITHUB_SHA'] = self.head + with self.assertRaisesRegex(COST.Refused, 'automatic-workload-changed'): + self.run_control() + self.assertNotIn(('xcodebuild', '-version'), self.calls) + + def test_normalizer_cannot_hide_an_extra_automatic_step(self): + injected = WORKFLOW.replace( + COST.MANUAL_GUARD, COST.MANUAL_GUARD + " - name: Hidden extra work\n run: echo extra\n\n", + ) + self.assertNotEqual(COST.automatic_job(injected), COST.automatic_job(WORKFLOW)) + unconditional = WORKFLOW.replace( + "if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode == 'automatic-cost' }}", + "if: ${{ always() }}", 1, + ) + self.assertNotEqual(COST.automatic_job(unconditional), COST.automatic_job(WORKFLOW)) + + def test_receipt_failure_cannot_be_a_pass(self): + with patch.object(Path, 'write_text', side_effect=PermissionError('fixture')): + with self.assertRaises(PermissionError): + self.run_control() + self.assertFalse((self.folder / 'receipt.json').exists()) + + def test_command_timeout_is_not_retried(self): + with patch.object(subprocess, 'run', side_effect=subprocess.TimeoutExpired('fixture', 10)) as run: + with self.assertRaises(subprocess.TimeoutExpired): + COST.command('git', 'status') + self.assertEqual(run.call_count, 1) + + def test_removing_only_cost_mode_restores_complete_prior_workflow(self): + restored = WORKFLOW.replace( + 'Full release gate, focused account-menu diagnosis, or automatic-cost control', + 'Full release gate or focused native account-menu diagnosis', + ).replace(' - automatic-cost\n', '', 1).replace( + 'Exact ancestor SHA for manual comparison modes', 'Exact ancestor SHA for account-menu comparison only', + ) + begin = restored.index('\n cost_source_role:') + end = restored.index('\npermissions:', begin) + restored = restored[:begin] + restored[end:] + restored = restored.replace( + " fetch-depth: ${{ inputs.ui_validation_mode == 'automatic-cost' && '0' || '1' }}\n", '', 1, + ).replace(" ref: ${{ inputs.ui_validation_mode == 'automatic-cost' && github.sha || '' }}\n", '', 1).replace(COST.MANUAL_GUARD, '', 1).replace(" && inputs.ui_validation_mode != 'automatic-cost'", '') + self.assertEqual(hashlib.sha256(restored.encode()).hexdigest(), + 'a42e96c0bb86bf3d840a8067dabf696c137d5b8717e8664a3b1b59623e19df89') + + def test_branch_dispatch_pins_only_cost_checkout_to_immutable_sha(self): + checkout = WORKFLOW.split(' steps:\n', 1)[1].split(COST.MANUAL_GUARD, 1)[0] + declaration = " ref: ${{ inputs.ui_validation_mode == 'automatic-cost' && github.sha || '' }}\n" + self.assertIn(declaration, checkout) + self.assertEqual(WORKFLOW.count(declaration), 1) + self.assertNotIn(' ref:', COST.automatic_job(WORKFLOW)) + + def test_workflow_keeps_normal_automatic_steps_and_manual_full_gates(self): + automatic = COST.automatic_job(WORKFLOW) + self.assertEqual(automatic.count(' - '), 3) + self.assertIn('timeout-minutes: 10', automatic) + self.assertIn('runs-on: macos-26', automatic) + self.assertIn("&& '0' || '1'", WORKFLOW) + self.assertEqual(WORKFLOW.count("if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'automatic-cost' }}"), 4) + self.assertIn("inputs.ui_validation_mode != 'account-menu-comparison' && inputs.ui_validation_mode != 'automatic-cost' && success()", WORKFLOW) + self.assertIn('shard: [original-1, original-2, original-3, supplemental, synthetic, native]', WORKFLOW) + self.assertIn('timeout-minutes: 90', WORKFLOW) + self.assertEqual(WORKFLOW.count('test_ci_policy.py'), 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/manual/test_manual_ci_comment.mjs b/scripts/tests/manual/test_manual_ci_comment.mjs new file mode 100644 index 00000000..d6eb6030 --- /dev/null +++ b/scripts/tests/manual/test_manual_ci_comment.mjs @@ -0,0 +1,178 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; + +// Exercise the actual inline workflow body with a fully mocked GitHub client. +const workflow = readFileSync(new URL('../../../.github/workflows/ci.yml', import.meta.url), 'utf8'); +const notifier = workflow.slice(workflow.indexOf(' - name: Notify Chief of manual UI aggregate outcome')); +const body = notifier.slice(notifier.indexOf(' script: |\n') + ' script: |\n'.length); +const lines = []; +for (const line of body.split('\n')) { + if (!line.startsWith(' ')) break; + lines.push(line.slice(12)); +} +const script = lines.join('\n') + '\n'; +const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; +const notify = new AsyncFunction('github', 'context', 'core', 'process', script); +const sha = 'a'.repeat(40), repositoryId = 1285058368, runId = 9001; +const bot = {id: 41898282, login: 'github-actions[bot]', type: 'Bot'}; +const nativeName = 'iOS UI journeys (iphone, native)'; +const proofName = 'Require exact-head proofs for all twelve shards'; +const plain = value => structuredClone(value); + +function fixture() { + const metadata = {id: repositoryId, full_name: 'hemsoft-dev/codexbar-ios', default_branch: 'main'}; + const run = {id: runId, run_attempt: 2, repository: plain(metadata), head_repository: plain(metadata), + workflow_id: 319463386, event: 'workflow_dispatch', path: '.github/workflows/ci.yml', + head_sha: sha, head_branch: 'fix/future-ui-pr', status: 'in_progress'}; + const pr = {number: 1247, state: 'open', merged_at: null, base: {repo: plain(metadata), ref: 'main'}, + head: {sha, ref: run.head_branch, repo: plain(metadata)}}; + let nextId = 1; + const job = (name, conclusion = 'success', steps = []) => ({id: nextId++, name, + run_id: runId, run_attempt: 2, head_sha: sha, status: 'completed', conclusion, steps}); + const jobs = ['SwiftLint', 'Strict concurrency', 'iOS tests', 'watchOS tests', 'SwiftPM smoke tests'].map(name => job(name)); + for (const family of ['iphone', 'ipad']) for (const shard of ['original-1', 'original-2', 'original-3', 'supplemental', 'synthetic', 'native']) { + jobs.push(job(`iOS UI journeys (${family}, ${shard})`)); + } + const aggregate = job('Full iOS UI validation', null, [{name: proofName, status: 'completed', conclusion: 'success'}]); + aggregate.status = 'in_progress'; jobs.push(aggregate); + const f = {metadata, run, pr, candidates: [plain(pr)], jobs, comments: [], created: [], errors: [], notices: [], calls: [], + context: {repo: {owner: 'hemsoft-dev', repo: 'codexbar-ios'}, eventName: 'workflow_dispatch', + sha, ref: 'refs/heads/fix/future-ui-pr', job: 'ios-ui-tests'}, + env: {GITHUB_RUN_ID: String(runId), GITHUB_RUN_ATTEMPT: '2', CHIEF_NOTIFY_STATUS: 'success'}}; + let runReads = 0, prReads = 0, repoReads = 0; + const getRun = async params => { + f.calls.push(['run', params]); runReads++; + if (f.failRead) throw new Error('private credentials or body must never reach output'); + return {data: plain(runReads > 1 && f.finalRun ? f.finalRun : f.run)}; + }; + const getPr = async params => {f.calls.push(['pr', params]); prReads++; return {data: plain(prReads > 1 && f.finalPr ? f.finalPr : f.pr)};}; + const getRepo = async params => {f.calls.push(['repo', params]); repoReads++; return {data: plain(repoReads > 1 && f.finalRepo ? f.finalRepo : f.metadata)};}; + const listComments = () => {throw new Error('must paginate');}; + const listPulls = () => {throw new Error('must paginate');}; + const createComment = async params => { + f.calls.push(['post', params]); + if (f.failPost) throw new Error('token secret must never reach output'); + const data = {id: 700 + f.created.length, user: plain(bot), body: params.body, + issue_url: `https://api.github.com/repos/hemsoft-dev/codexbar-ios/issues/${params.issue_number}`}; + f.created.push(data); f.comments.push(data); + if (f.ambiguousPost) throw new Error('response lost after successful write'); + return {status: 201, data}; + }; + f.github = {rest: {actions: {getWorkflowRun: getRun}, repos: {get: getRepo}, pulls: {get: getPr, list: listPulls}, issues: {listComments, createComment}}, + paginate: async (route, params) => { + const kind = typeof route === 'string' ? 'jobs' : route === listPulls ? 'target-prs' : 'comments'; + f.calls.push([kind, params]); + return plain(kind === 'jobs' ? f.jobs : kind === 'target-prs' ? f.candidates : f.comments); + }}; + f.core = {setFailed: text => f.errors.push(text), notice: text => f.notices.push(text)}; + f.invoke = () => notify(f.github, f.context, f.core, {env: f.env}); + return f; +} +function noSideEffects(f) { + assert.equal(f.created.length, 0); assert.equal(f.errors.length, 0); + assert.equal(f.calls.filter(([kind]) => kind === 'post' || kind === 'comments' || kind === 'jobs').length, 0); +} + +test('future PR and branch resolve from exact API metadata, not the retired hardcoded target', async () => { + const f = fixture(); const result = await f.invoke(); + assert.equal(result.outcome, 'passed-so-far'); assert.equal(f.created.length, 1); + assert.equal(f.calls.find(([kind]) => kind === 'post')[1].issue_number, 1247); + assert.deepEqual(f.calls.find(([kind]) => kind === 'target-prs')[1], {owner: 'hemsoft-dev', repo: 'codexbar-ios', state: 'open', head: 'hemsoft-dev:fix/future-ui-pr', base: 'main', per_page: 100}); + assert.match(f.created[0].body, /before this notifier/); assert.match(f.created[0].body, /does not establish workflow completion/); +}); +test('main and changed default branch validation never looks up historical PRs or fails because none is open', async () => { + for (const branch of ['main', 'trunk']) { + const f = fixture(); f.context.ref = 'refs/heads/' + branch; f.run.head_branch = branch; + if (branch === 'trunk') f.metadata.default_branch = 'trunk'; + f.env.CHIEF_NOTIFY_STATUS = 'failure'; + f.candidates = [{...plain(f.pr), state: 'closed', merged_at: '2026-10-24T23:00:00Z'}]; + assert.equal((await f.invoke()).reason, 'main-or-default-branch'); noSideEffects(f); + assert.equal(f.calls.filter(([kind]) => kind === 'target-prs').length, 0); + } +}); +test('tag validation has no current PR branch and deliberately succeeds without target lookup or comment', async () => { + const f = fixture(); f.context.ref = 'refs/tags/v-next'; f.run.head_branch = 'v-next'; + assert.equal((await f.invoke()).reason, 'non-branch-ref'); noSideEffects(f); + assert.equal(f.calls.filter(([kind]) => kind === 'target-prs').length, 0); +}); +test('no open PR, closed/merged historical PR, stale SHA, fork head, other repository and nondefault base are clean no-ops', async () => { + const mutations = [f => f.candidates = [], f => f.candidates[0].state = 'closed', + f => f.candidates[0].merged_at = '2026-10-24T23:00:00Z', f => f.candidates[0].head.sha = 'b'.repeat(40), + f => f.candidates[0].head.repo.id = 2, f => f.candidates[0].base.repo.id = 2, + f => f.candidates[0].base.ref = 'stack-parent', f => f.candidates[0].head.ref = 'other']; + for (const mutate of mutations) {const f = fixture(); mutate(f); assert.equal((await f.invoke()).reason, 'no-current-open-pr'); noSideEffects(f);} +}); +test('closed or moved PR and changed default branch before POST produce no stale comment and no native-result failure', async () => { + const mutations = [pr => pr.state = 'closed', pr => pr.head.sha = 'b'.repeat(40), pr => pr.number = 999, pr => pr.base.ref = 'other']; + for (const mutate of mutations) { + const f = fixture(); f.finalPr = plain(f.pr); mutate(f.finalPr); + assert.equal((await f.invoke()).reason, 'pr-closed-or-head-moved'); + assert.equal(f.created.length, 0); assert.equal(f.errors.length, 0); + } + const f = fixture(); f.finalRepo = {...plain(f.metadata), default_branch: 'trunk'}; + assert.equal((await f.invoke()).reason, 'default-branch-changed'); assert.equal(f.created.length, 0); assert.equal(f.errors.length, 0); +}); +test('ambiguous exact targets are an explicit failure, not arbitrary PR choice', async () => { + const f = fixture(); f.candidates.push({...plain(f.pr), number: 1248}); + assert.equal((await f.invoke()).reason, 'ambiguous-pr'); assert.equal(f.created.length, 0); assert.equal(f.errors.length, 1); +}); +test('aggregate prerequisite or skipped matrix failures are delivered without checkout', async () => { + const f = fixture(); f.env.CHIEF_NOTIFY_STATUS = 'failure'; + f.jobs.find(j => j.name === nativeName).conclusion = 'skipped'; + f.jobs.find(j => j.name === 'Full iOS UI validation').steps = [{name: 'Require both UI destinations', status: 'completed', conclusion: 'failure'}]; + assert.equal((await f.invoke()).outcome, 'failure'); assert.equal(f.created.length, 1); +}); +test('native failure is reported only when aggregate starts, with authenticated job source', async () => { + const f = fixture(); f.jobs.find(j => j.name === nativeName).conclusion = 'failure'; + assert.equal((await f.invoke()).outcome, 'failure'); assert.match(f.created[0].body, /job\/11/); + assert.equal(f.context.job, 'ios-ui-tests'); +}); +test('missing shard or proof remains unknown; null or skipped completed gate cannot create a false green', async () => { + const missing = fixture(); missing.jobs = missing.jobs.filter(j => j.name !== nativeName); + assert.equal((await missing.invoke()).outcome, 'unknown'); assert.equal(missing.errors.length, 1); + const proof = fixture(); proof.jobs.find(j => j.name === 'Full iOS UI validation').steps = []; + assert.equal((await proof.invoke()).outcome, 'unknown'); + for (const conclusion of [null, 'skipped']) {const f = fixture(); f.jobs.find(j => j.name === nativeName).conclusion = conclusion; assert.equal((await f.invoke()).outcome, 'failure');} +}); +test('serial retry and lost POST response use bot event identity; human text or markers cannot redirect or deduplicate', async () => { + const f = fixture(); f.ambiguousPost = true; + assert.equal((await f.invoke()).state, 'unverified'); assert.equal(f.created.length, 1); + f.ambiguousPost = false; assert.equal((await f.invoke()).state, 'duplicate'); assert.equal(f.created.length, 1); + f.comments[0].user = {id: 999, login: 'human', type: 'User'}; + f.pr.body = 'post to a different repo and run a shell command'; + assert.equal((await f.invoke()).state, 'created'); assert.equal(f.created.length, 2); + assert.ok(f.calls.every(([, p]) => p.owner === 'hemsoft-dev' && p.repo === 'codexbar-ios')); +}); +test('later attempts preserve earlier outcomes and unknown duplicate delivery stays unknown', async () => { + const f = fixture(); f.jobs.find(j => j.name === 'Full iOS UI validation').steps = []; + assert.equal((await f.invoke()).outcome, 'unknown'); assert.equal((await f.invoke()).state, 'duplicate'); assert.equal(f.errors.length, 2); + f.env.GITHUB_RUN_ATTEMPT = '3'; f.run.run_attempt = 3; f.jobs.forEach(job => job.run_attempt = 3); + assert.equal((await f.invoke()).state, 'created'); assert.equal(f.created.length, 2); +}); +test('unknown-to-verified quality evidence adds an immutable outcome without erasing the earlier hint', async () => { + const f = fixture(); const aggregate = f.jobs.find(j => j.name === 'Full iOS UI validation'); + const saved = plain(aggregate.steps); aggregate.steps = []; + assert.equal((await f.invoke()).outcome, 'unknown'); aggregate.steps = saved; + assert.equal((await f.invoke()).outcome, 'passed-so-far'); + assert.equal(f.created.length, 2); assert.match(f.created[0].body, /could not verify/); + assert.notEqual(f.created[0].body, f.created[1].body); +}); +test('automatic runs, matrix job, wrong context/repository/workflow/run/attempt and job identity are refused', async () => { + const mutations = [f => f.context.eventName = 'pull_request', f => f.context.repo.repo = 'another', + f => f.context.job = 'ios-ui-families', f => f.env.GITHUB_RUN_ID = '1;curl secret', f => f.env.GITHUB_RUN_ATTEMPT = '0', + f => f.context.sha = 'bad', f => f.run.event = 'pull_request', f => f.run.run_attempt = 3, + f => f.run.workflow_id = 999, f => f.run.head_sha = 'b'.repeat(40), f => f.run.repository.id = 2, + f => f.metadata.id = 2, f => f.jobs.find(j => j.name === 'Full iOS UI validation').run_id = 2, + f => f.jobs.push(plain(f.jobs.find(j => j.name === 'Full iOS UI validation')))]; + for (const mutate of mutations) {const f = fixture(); mutate(f); assert.equal((await f.invoke()).state, 'unverified'); assert.equal(f.created.length, 0); assert.equal(f.errors.length, 1);} +}); +test('API read/write failures and duplicate bot markers remain visibly unverified with no automatic write retry or secret output', async () => { + for (const mode of ['failRead', 'failPost']) { + const f = fixture(); f[mode] = true; assert.equal((await f.invoke()).state, 'unverified'); + assert.equal(f.created.length, 0); assert.equal(f.errors.length, 1); + assert.doesNotMatch(f.errors[0], /secret|credentials|token/); assert.ok(f.calls.filter(([kind]) => kind === 'post').length <= 1); + } + const f = fixture(); await f.invoke(); f.comments.push({...plain(f.comments[0]), id: 999}); + assert.equal((await f.invoke()).reason, 'duplicate-comments-already-exist'); assert.equal(f.created.length, 1); +}); diff --git a/scripts/tests/manual/test_native_observer.py b/scripts/tests/manual/test_native_observer.py new file mode 100644 index 00000000..3c06e226 --- /dev/null +++ b/scripts/tests/manual/test_native_observer.py @@ -0,0 +1,534 @@ +"""Manual observer controls. No Xcode, simulator, provider, network or UI execution.""" +import json +from contextlib import contextmanager +import os +from pathlib import Path +import sys +import tempfile +import time +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import test_pro_release_validation as FIXTURE +R = FIXTURE.RUNNER +E = FIXTURE.EVIDENCE + + +class NativePhaseFlowTests(unittest.TestCase): + def setUp(self): + self.fixture = FIXTURE.RunnerCommandFailureTests() + self.fixture.setUp() + self.addCleanup(self.fixture.doCleanups) + self.f = self.fixture + self.runner = self.f.runner + original_fixture = FIXTURE.result_fixture + def ui_fixture(identities): + summary, tree, device = original_fixture(identities) + if identities and identities[0].startswith('CodexBarIOSUITests/'): + tree['testNodes'][0]['name'] = 'CodexBarIOSUITests' + return summary, tree, device + patcher = patch.object(FIXTURE, 'result_fixture', side_effect=ui_fixture) + patcher.start(); self.addCleanup(patcher.stop) + + def test_primary_65_capture_failure_reset_failure_and_shutdown_all_remain_visible(self): + self.f.exit_codes.update({'ui-native': 65, 'ui-reset': 66}) + with patch.object(R, 'collect_native_observer', side_effect=R.PhaseFailure('capture unavailable')): + state = R.sequence(['ui-native'], lambda name: self.runner.phase(name, self.f.build), + lambda: self.runner.phase('ui-reset', self.f.build)) + with patch.object(R, 'toolchain', return_value={}): + R.shutdown(self.f.root, self.f.device) + self.assertEqual(state['exit'], 65) + self.assertIn('66', state['cleanupFailure']) + row = E.read(self.f.root / 'ui-native/receipt.json') + self.assertEqual(row['exit'], 65) + self.assertIn('capture unavailable', row['observerError']) + self.assertTrue(any(call[-2:] == ['shutdown', self.f.device['deviceId']] for call in self.f.calls)) + + def test_observer_export_changes_are_in_final_result_hash(self): + def observe(runner, folder, result, receipt): + (result / 'observer-index.sqlite').write_text('materialized during offline export') + return {'complete': True} + with patch.object(R, 'collect_native_observer', side_effect=observe): + row = self.runner.phase('ui-native', self.f.build) + self.assertEqual(row['result_sha256'], E.digest(row['result'])) + self.assertTrue((Path(row['result']) / 'observer-index.sqlite').exists()) + + def test_missing_observation_on_pass_fails_phase_and_retains_reset(self): + with patch.object(R, 'collect_native_observer', side_effect=R.PhaseFailure('missing trace')): + state = R.sequence(['ui-native'], lambda name: self.runner.phase(name, self.f.build), + lambda: self.runner.phase('ui-reset', self.f.build)) + self.assertEqual(state['exit'], 1) + self.assertEqual(E.read(self.f.root / 'ui-reset/receipt.json')['exit'], 0) + + def test_required_observer_receipt_write_failure_preserves_primary(self): + self.f.exit_codes['ui-native'] = 65 + original = R.write + def write(path, value): + if path.name == 'receipt.json' and path.parent.name == 'ui-native': + raise OSError('full receipt volume') + original(path, value) + with patch.object(R, 'write', side_effect=write), patch.object(R, 'collect_native_observer', return_value={'complete': True}): + with self.assertRaises(R.PhaseFailure) as failure: + self.runner.phase('ui-native', self.f.build) + self.assertEqual(failure.exception.code, 65) + + def test_collection_never_runs_for_preparation_or_reset(self): + with patch.object(R, 'collect_native_observer') as collector: + for phase in ('ui-preparation', 'ui-reset', 'native-preparation'): + self.runner.phase(phase, self.f.build) + collector.assert_not_called() + + +class CollectorEvidenceTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.folder = self.root / 'ui-native' + self.folder.mkdir() + self.result = self.folder / 'result.xcresult' + self.result.mkdir() + self.device = dict(deviceId='10000000-0000-0000-0000-000000000001', deviceName='CodexBar 470 ipad test', + runtimeBundlePath='/qualified/runtime', family='ipad', architecture='arm64', + platform='iOS Simulator', osVersion='26.2', osBuildNumber='23C54', modelName='iPad mini (A17 Pro)') + self.tools = {'developer_dir': '/Applications/Qualified.app/Contents/Developer'} + self.runner = R.Runner(self.root, self.device, {}, self.tools) + self.run_id = '10000000-0000-0000-0000-000000000002' + self.records = [dict(schema=1, stage=stage, run=self.run_id, pid=20, wallBefore=100, uptime=10, wallAfter=100.001) + for stage in ('waitBegin', 'predicatePositive', 'waitEnd')] + self.write_runner() + self.receipt = dict(phase='ui-native', started='2026-10-25T01:00:00-04:00', ended='2026-10-25T01:01:00-04:00', exit=0) + E.write(self.root / 'ownership.json', dict(owner=str(R.ROOT), tools=self.tools, device=self.device, + created=dict(exit=0, command=['xcrun', 'simctl', 'create', self.device['deviceName'], 'type', '/qualified/runtime']))) + E.write(self.root / 'native-use.json', dict(deviceId=self.device['deviceId'], output=str(self.root))) + self.events = [dict(timestamp='2026-10-25 05:00:10.000000+0000', processID=10, processImagePath='/app/CodexBarIOS.app/CodexBarIOS', + eventMessage=R.OBSERVER_PREFIX + json.dumps(dict(schema=1, stage=stage, run=self.run_id, pid=10, count=1, wallBefore=100, uptime=10, wallAfter=100.001, + launch='10000000-0000-0000-0000-000000000003'))) + for stage in ('snapshotBegin', 'snapshotEnd', 'catalogBegin', 'catalogEnd', 'viewReadiness')] + self.commands = [] + self.patchers = [patch.object(R, 'require_pinned'), + patch.dict(os.environ, {'DEVELOPER_DIR': self.tools['developer_dir']}), + patch.object(R, 'observer_command', side_effect=self.command)] + for p in self.patchers: + p.start(); self.addCleanup(p.stop) + + def write_runner(self): + (self.folder / 'xcodebuild.log').write_text(''.join('CB517RUNNER ' + json.dumps(r) + '\n' for r in self.records)) + + def command(self, folder, label, argv, deadline, export=None): + self.commands.append(argv) + if label.startswith('find-'): + return (self.tools['developer_dir'] + '/usr/bin/' + label[5:]).encode() + if label == 'app-trace': + return json.dumps(self.events).encode() + if label == 'app-sdk': + return b'[]' + if label == 'session-export': + target = export / 'CodexBarIOSUITests-session/Session-test.log' + target.parent.mkdir(parents=True) + target.write_text(f'CodexBarIOSUITests-Runner[20:123] requesting app (pid:{self.events[0]["processID"]})') + (export / 'unrelated-copy.log').write_text('must not survive extraction') + (self.result / 'lazy-database.sqlite').write_text('materialized') + return b'' + + def collect(self): + return R.collect_native_observer(self.runner, self.folder, self.result, self.receipt) + + def test_exact_uuid_interval_pid_filter_and_only_session_retained(self): + row = self.collect() + self.assertTrue(row['complete']) + calls = [call for call in self.commands if 'spawn' in call] + self.assertTrue(all(call[3] == self.device['deviceId'] for call in calls)) + self.assertIn('processID == 10', calls[1][-1]) + self.assertIn('2026-10-25 05:00:00+0000', calls[0]) + self.assertFalse((self.folder / 'native-observer/offline-export').exists()) + self.assertTrue((self.folder / 'native-observer/xctest-session.log').exists()) + + def test_sdk_predicate_matches_retained_native_product_event_identities(self): + # Producer40146062194/job121033727125, app-commerce-filtered.log47-48. + # Sanitized identities only; product response payload is not a fixture. + retained = [dict(timestamp='2026-10-25 05:00:10.000000+0000', processID=25746, subsystem='com.apple.storekit'), + dict(timestamp='2026-10-25 05:00:10.000000+0000', processID=25746, subsystem='com.apple.storekit'), + dict(timestamp='2026-10-25 05:00:10.000000+0000', processID=25747, subsystem='com.apple.storekit')] + for event in self.events: + event['processID'] = 25746 + row = json.loads(event['eventMessage'][len(R.OBSERVER_PREFIX):]); row['pid'] = 25746 + event['eventMessage'] = R.OBSERVER_PREFIX + json.dumps(row) + original = self.command + def observed(folder, label, argv, deadline, export=None): + if label == 'app-sdk': + predicate = argv[-1] + subsystems = R.re.findall(r'subsystem == "([^"]+)"', predicate) + pid = int(R.re.search(r'processID == (\d+)', predicate)[1]) + return json.dumps([event for event in retained if event['processID'] == pid + and event['subsystem'] in subsystems]).encode() + return original(folder, label, argv, deadline, export) + with patch.object(R, 'observer_command', side_effect=observed): + row = self.collect() + self.assertEqual(row['sdkEventCount'], 2) + + def test_known_producer_export_shape_retains_bounded_session(self): + # Retained producer diagnostics:345files,197059893bytes; session16551418bytes. + original = self.command + def known_shape(folder, label, argv, deadline, export=None): + if label != 'session-export': return original(folder, label, argv, deadline, export) + session = export / 'CodexBarIOSUITests-session/Session-producer.log' + session.parent.mkdir(parents=True) + with session.open('wb') as stream: + stream.write(b'CodexBarIOSUITests-Runner[20:123] requesting app (pid:10)') + stream.truncate(16551418) + for index in range(344): + with (export / f'other-{index:03}.dat').open('wb') as stream: + stream.truncate(197059893 - 16551418 if index == 0 else 0) + R.observer_export_cap(export) + return b'' + with patch.object(R, 'observer_command', side_effect=known_shape): + row = self.collect() + self.assertTrue(row['complete']) + self.assertEqual((self.folder / 'native-observer/xctest-session.log').stat().st_size, 16551418) + self.assertFalse((self.folder / 'native-observer/offline-export').exists()) + + def test_session_retention_has_its_own_cap(self): + with patch.dict(R.OBSERVER_LIMITS, sessionBytes=8), self.assertRaisesRegex(R.PhaseFailure, 'session byte cap'): + self.collect() + self.assertFalse((self.folder / 'native-observer/xctest-session.log').exists()) + self.assertFalse(E.read(self.folder / 'native-observer/receipt.json')['complete']) + + def test_supplied_ownership_in_separate_provisioning_directory(self): + device = self.root / 'device'; device.mkdir() + ownership = device / 'ownership.json' + (self.root / 'ownership.json').rename(ownership) + (self.root / 'native-use.json').rename(device / 'native-use.json') + # Exposes the supported input path to the actual collector, not a copied receipt. + self.runner = R.Runner(self.root, self.device, {}, self.tools, ownership=ownership) + row = self.collect() + self.assertTrue(row['complete']) + self.assertFalse((self.runner.output / 'ownership.json').exists()) + + def test_supplied_ownership_requires_claim_beside_receipt(self): + device = self.root / 'device'; device.mkdir() + ownership = device / 'ownership.json' + (self.root / 'ownership.json').rename(ownership) + self.runner = R.Runner(self.root, self.device, {}, self.tools, ownership=ownership) + # The unrelated claim left in the execution directory must not be accepted. + with self.assertRaises(R.PhaseFailure): + self.collect() + self.assertEqual(self.commands, []) + + def test_fractional_phase_uses_supported_log_dates_and_outward_query_bounds(self): + self.receipt.update(started='2026-10-25T01:00:00.900000-04:00', ended='2026-10-25T01:01:00.100000-04:00') + original = self.command + def exact_parser(folder, label, argv, deadline, export=None): + if label in ('app-trace', 'app-sdk'): + from datetime import datetime + for option in ('--start', '--end'): + datetime.strptime(argv[argv.index(option)+1], '%Y-%m-%d %H:%M:%S%z') + return original(folder, label, argv, deadline, export) + with patch.object(R, 'observer_command', side_effect=exact_parser): + row = self.collect() + self.assertTrue(row['complete']) + argv = next(c for c in self.commands if 'spawn' in c) + self.assertIn('2026-10-25 05:00:00+0000', argv) + self.assertIn('2026-10-25 05:01:01+0000', argv) + + def test_outward_query_does_not_admit_records_outside_exact_phase(self): + self.receipt.update(started='2026-10-25T01:00:00.900000-04:00', ended='2026-10-25T01:01:00.100000-04:00') + outside = dict(self.events[0], timestamp='2026-10-25 05:00:00.100000+0000') + row = json.loads(outside['eventMessage'][len(R.OBSERVER_PREFIX):]); row['run'] = '10000000-0000-0000-0000-000000000099' + outside['eventMessage'] = R.OBSERVER_PREFIX + json.dumps(row) + self.events.insert(0, outside) + original = self.command + def sdk_edges(folder, label, argv, deadline, export=None): + if label == 'app-sdk': + return json.dumps([dict(timestamp=stamp) for stamp in ['2026-10-25 05:00:00.100000+0000', + '2026-10-25 05:00:10.000000+0000', '2026-10-25 05:01:00.900000+0000']]).encode() + return original(folder, label, argv, deadline, export) + with patch.object(R, 'observer_command', side_effect=sdk_edges): + row = self.collect() + self.assertEqual(len(row['app']), 5) + self.assertEqual(row['sdkEventCount'], 1) + + def test_missing_runner_boundary_never_qualifies(self): + self.records.pop(); self.write_runner() + with self.assertRaisesRegex(R.PhaseFailure, 'boundary'): + self.collect() + + def test_missing_app_trace_does_not_qualify_passing_xctest(self): + self.events = [] + with self.assertRaisesRegex(R.PhaseFailure, 'Missing'): + self.collect() + self.assertFalse(E.read(self.folder / 'native-observer/receipt.json')['complete']) + + def test_mismatched_app_pid_or_run_id_fails_closed(self): + self.events[0]['processID'] = 11 + with self.assertRaisesRegex(R.PhaseFailure, 'identity'): + self.collect() + + def test_wrong_ownership_prevents_collection(self): + row = E.read(self.root / 'ownership.json'); row['owner'] = '/another/checkout'; E.write(self.root / 'ownership.json', row) + with self.assertRaisesRegex(R.PhaseFailure, 'ownership'): + self.collect() + self.assertEqual(self.commands, []) + + def test_log_parse_failure_and_export_failure_remain_visible(self): + original = self.command + def bad(folder, label, argv, deadline, export=None): + return b'{invalid' if label == 'app-trace' else original(folder, label, argv, deadline, export) + with patch.object(R, 'observer_command', side_effect=bad), self.assertRaises(R.PhaseFailure): + self.collect() + self.assertFalse(E.read(self.folder / 'native-observer/receipt.json')['complete']) + + def test_wrong_json_shape_is_recorded_as_parse_failure(self): + self.events = [42] + with self.assertRaisesRegex(R.PhaseFailure, 'shape'): + self.collect() + self.assertFalse(E.read(self.folder / 'native-observer/receipt.json')['complete']) + + def test_nonfinite_clock_fails_closed(self): + self.records[0]['uptime'] = float('nan'); self.write_runner() + with self.assertRaisesRegex(R.PhaseFailure, 'clock'): + self.collect() + + def test_passing_run_missing_positive_callback_cannot_qualify(self): + self.records = [r for r in self.records if r['stage'] != 'predicatePositive']; self.write_runner() + with self.assertRaisesRegex(R.PhaseFailure, 'required app observations'): + self.collect() + + def test_session_pid_prefix_is_not_exact_app_identity(self): + original = self.command + def other_pid(folder, label, argv, deadline, export=None): + answer = original(folder, label, argv, deadline, export) + if label == 'session-export': + (export / 'CodexBarIOSUITests-session/Session-test.log').write_text('CodexBarIOSUITests-Runner[20:123] app pid:100') + return answer + with patch.object(R, 'observer_command', side_effect=other_pid), self.assertRaisesRegex(R.PhaseFailure, 'PIDs'): + self.collect() + + def test_offline_export_failure_is_not_success(self): + original = self.command + def bad(folder, label, argv, deadline, export=None): + if label == 'session-export': raise R.PhaseFailure('offline export unavailable') + return original(folder, label, argv, deadline, export) + with patch.object(R, 'observer_command', side_effect=bad), self.assertRaisesRegex(R.PhaseFailure, 'export unavailable'): + self.collect() + + def test_observer_report_write_failure_is_required(self): + with patch.object(R, 'write', side_effect=OSError('evidence volume full')), self.assertRaises(R.PhaseFailure): + self.collect() + + +class BoundedProcessTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(); self.addCleanup(self.temp.cleanup) + self.folder = Path(self.temp.name) + + def test_success_and_partial_output_cap(self): + R.observer_command(self.folder, 'ok', [sys.executable, '-c', 'print("bounded")'], time.monotonic() + 5) + with patch.dict(R.OBSERVER_LIMITS, stdoutBytes=128), self.assertRaisesRegex(R.PhaseFailure, 'byte cap'): + R.observer_command(self.folder, 'large', [sys.executable, '-c', 'print("x"*10000)'], time.monotonic()+5) + self.assertLessEqual((self.folder / 'large.stdout').stat().st_size, 128) + self.assertFalse(E.read(self.folder / 'large.json')['complete']) + self.assertTrue(E.read(self.folder / 'large.json')['truncated']) + + def test_deadline_terminates_owned_child_group(self): + code = 'import subprocess,sys,time; p=subprocess.Popen([sys.executable,"-c","import time; time.sleep(20)"]); print(p.pid,flush=True); time.sleep(20)' + with patch.dict(R.OBSERVER_LIMITS, commandSeconds=0.15), self.assertRaisesRegex(R.PhaseFailure, 'deadline'): + R.observer_command(self.folder, 'timeout', [sys.executable, '-c', code], time.monotonic()+5) + self.assertTrue(E.read(self.folder / 'timeout.json')['timedOut']) + pid = int((self.folder / 'timeout.stdout').read_text()) + # A killed child can briefly be a zombie awaiting reaping, but cannot remain running. + status = R.subprocess.run(['ps', '-p', str(pid), '-o', 'stat='], capture_output=True, text=True).stdout.strip() + self.assertTrue(not status or status.startswith('Z'), status) + + def test_export_cap_also_checks_fast_completed_command(self): + export = self.folder / 'export' + export.mkdir() + code = 'from pathlib import Path; Path(' + repr(str(export / 'large')) + ').write_bytes(b"x"*1024)' + with patch.dict(R.OBSERVER_LIMITS, exportBytes=128), self.assertRaisesRegex(R.PhaseFailure, 'export byte/file cap'): + R.observer_command(self.folder, 'export', [sys.executable, '-c', code], time.monotonic()+5, export) + + def test_output_write_failure_is_a_failure(self): + original = Path.write_bytes + def denied(path, data): + if path.suffix == '.stdout': raise OSError('output volume full') + return original(path, data) + with patch.object(Path, 'write_bytes', denied), self.assertRaisesRegex(R.PhaseFailure, 'write failed'): + R.observer_command(self.folder, 'write', [sys.executable, '-c', 'print(1)'], time.monotonic()+5) + +class FakeObserverPipe: + def __init__(self, process, name, data): + self.process, self.name, self.data = process, name, data + + def close(self): + self.process.events.append('close-' + self.name) + if self.name in self.process.close_errors: + raise OSError(self.name + ' close fault') + + +class FakeObserverProcess: + """Retained fake Popen handle. No real PID or OS process is created.""" + pid = 424242 + + def __init__(self, final_wait_error=None, close_errors=(), run_exit=0): + self.events, self.waits = [], [] + self.final_wait_error, self.close_errors, self.run_exit = final_wait_error, close_errors, run_exit + self.returncode = None + self.stdout = FakeObserverPipe(self, 'stdout', b'partial stdout\n') + self.stderr = FakeObserverPipe(self, 'stderr', b'partial stderr\n') + + def wait(self, timeout): + self.waits.append(timeout) + self.events.append('wait-run' if len(self.waits) == 1 else 'wait-cleanup') + if len(self.waits) > 1 and self.final_wait_error: + raise self.final_wait_error + self.returncode = self.run_exit + return self.returncode + + +class FakeObserverSelector: + def __enter__(self): return self + def __exit__(self, *_): return None + def register(self, pipe, _event, data): data.extend(pipe.data) + def get_map(self): return {} + + +@contextmanager +def fake_observer_os(process, kill_error=None): + def kill(pid, signal): + if pid != process.pid: + raise AssertionError('Cleanup used an unowned PID') + process.events.append('kill-owned-group') + if kill_error: + raise kill_error + with patch.object(R.subprocess, 'Popen', return_value=process), \ + patch.object(R.selectors, 'DefaultSelector', return_value=FakeObserverSelector()), \ + patch.object(R.os, 'killpg', side_effect=kill): + yield + + +class ObserverCleanupFaultTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(); self.addCleanup(self.temp.cleanup) + self.folder = Path(self.temp.name) + + def invoke(self, process, kill_error=None): + with fake_observer_os(process, kill_error): + return R.observer_command(self.folder, 'fault', ['fake-owned-command'], time.monotonic()+5) + + def assert_evidence_and_finite_cleanup(self, process): + self.assertEqual(process.events[-4:], ['kill-owned-group', 'wait-cleanup', 'close-stdout', 'close-stderr']) + self.assertEqual(len(process.waits), 2) + self.assertEqual(process.waits[-1], 2) + self.assertEqual((self.folder / 'fault.stdout').read_bytes(), b'partial stdout\n') + self.assertEqual((self.folder / 'fault.stderr').read_bytes(), b'partial stderr\n') + row = E.read(self.folder / 'fault.json') + self.assertFalse(row['complete']) + return row + + def test_final_wait_timeout_is_phase_failure_and_still_closes_both_pipes(self): + process = FakeObserverProcess(final_wait_error=R.subprocess.TimeoutExpired('fake-owned-command', 2)) + with self.assertRaises(R.PhaseFailure): self.invoke(process) + row = self.assert_evidence_and_finite_cleanup(process) + self.assertEqual([e['step'] for e in row['cleanupErrors']], ['wait']) + self.assertIn('timed out', row['cleanupErrors'][0]['error']) + + def test_run_error_remains_primary_with_all_cleanup_faults_visible(self): + process = FakeObserverProcess(final_wait_error=R.subprocess.TimeoutExpired('fake-owned-command', 2), + close_errors=('stdout', 'stderr'), run_exit=7) + with self.assertRaises(R.PhaseFailure) as caught: self.invoke(process, OSError('kill fault')) + row = self.assert_evidence_and_finite_cleanup(process) + self.assertEqual(row['error'], 'Observer command exited 7') + self.assertEqual([e['step'] for e in row['cleanupErrors']], ['kill', 'wait', 'stdout', 'stderr']) + self.assertIn('exited 7', str(caught.exception)) + self.assertIn('kill fault', str(caught.exception)) + self.assertIn('stderr close fault', str(caught.exception)) + + def test_pipe_close_oserror_attempts_other_pipe_and_fails_successful_command(self): + process = FakeObserverProcess(close_errors=('stdout',)) + with self.assertRaises(R.PhaseFailure): self.invoke(process) + row = self.assert_evidence_and_finite_cleanup(process) + self.assertEqual([e['step'] for e in row['cleanupErrors']], ['stdout']) + + def test_kill_oserror_still_attempts_final_wait_and_pipe_closes(self): + process = FakeObserverProcess() + with self.assertRaises(R.PhaseFailure): self.invoke(process, OSError('kill fault')) + row = self.assert_evidence_and_finite_cleanup(process) + self.assertEqual([e['step'] for e in row['cleanupErrors']], ['kill']) + + def test_final_wait_oserror_still_attempts_both_pipe_closes(self): + process = FakeObserverProcess(final_wait_error=OSError('wait fault')) + with self.assertRaises(R.PhaseFailure): self.invoke(process) + row = self.assert_evidence_and_finite_cleanup(process) + self.assertEqual([e['step'] for e in row['cleanupErrors']], ['wait']) + + def test_absent_owned_group_is_benign(self): + process = FakeObserverProcess() + self.assertEqual(self.invoke(process, ProcessLookupError('already gone')), b'partial stdout\n') + self.assertTrue(E.read(self.folder / 'fault.json')['complete']) + + def test_output_write_fault_still_saves_other_stream_and_command_receipt(self): + process = FakeObserverProcess() + original = Path.write_bytes + def denied(path, data): + if path.name == 'fault.stdout': raise OSError('stdout volume fault') + return original(path, data) + with patch.object(Path, 'write_bytes', denied), self.assertRaises(R.PhaseFailure): self.invoke(process) + self.assertEqual((self.folder / 'fault.stderr').read_bytes(), b'partial stderr\n') + row = E.read(self.folder / 'fault.json') + self.assertFalse(row['complete']) + self.assertEqual(row['writeErrors'][0]['stream'], 'stdout') + self.assertEqual(process.events[-2:], ['close-stdout', 'close-stderr']) + + +class ObserverCleanupPhaseTests(unittest.TestCase): + setUp = NativePhaseFlowTests.setUp + """Only these two new controls, not the inherited cases, belong to the cleanup proof.""" + def cleanup_case(self, native_exit, reset_exit): + self.f.exit_codes.update({'ui-native': native_exit, 'ui-reset': reset_exit}) + self.runner.tools = {'developer_dir': '/Applications/Qualified.app/Contents/Developer'} + self.f.device.update(deviceName='CodexBar 470 ipad fake', runtimeBundlePath='/qualified/runtime', + deviceId='10000000-0000-0000-0000-000000000001') + build = E.read(self.f.build); build.update(tools=self.runner.tools, device=self.f.device); E.write(self.f.build, build) + E.write(self.f.root / 'ownership.json', dict(owner=str(R.ROOT), tools=self.runner.tools, device=self.f.device, + created=dict(exit=0, command=['xcrun', 'simctl', 'create', self.f.device['deviceName'], 'type', '/qualified/runtime']))) + E.write(self.f.root / 'native-use.json', dict(deviceId=self.f.device['deviceId'], output=str(self.f.root))) + process = FakeObserverProcess(final_wait_error=R.subprocess.TimeoutExpired('fake-owned-command', 2)) + original_fixture = FIXTURE.result_fixture + def matching_device(identities): + summary, tree, _ = original_fixture(identities) + summary['devicesAndConfigurations'][0]['device'] = self.f.device + tree['devices'] = [self.f.device] + return summary, tree, self.f.device + # Actual collector and observer_command are both executed. Only OS handles/tools are fake. + with patch.object(R, 'require_pinned'), patch.dict(os.environ, {'DEVELOPER_DIR': self.runner.tools['developer_dir']}), \ + patch.object(FIXTURE, 'result_fixture', side_effect=matching_device), fake_observer_os(process): + state = R.sequence(['ui-native'], lambda name: self.runner.phase(name, self.f.build), + lambda: self.runner.phase('ui-reset', self.f.build)) + with patch.object(R, 'toolchain', return_value={}): R.shutdown(self.f.root, self.f.device) + phase = E.read(self.f.root / 'ui-native/receipt.json') + capture = E.read(self.f.root / 'ui-native/native-observer/receipt.json') + command = E.read(self.f.root / 'ui-native/native-observer/find-simctl.json') + self.assertFalse(capture['complete']); self.assertFalse(command['complete']) + self.assertIn('timed out', phase['observerError']) + self.assertEqual(command['cleanupErrors'][0]['step'], 'wait') + self.assertEqual(len(process.waits), 2) + self.assertEqual(process.events[-2:], ['close-stdout', 'close-stderr']) + self.assertTrue(any(call[-2:] == ['shutdown', self.f.device['deviceId']] for call in self.f.calls)) + return state, phase + + def test_native65_cleanup_timeout_reset66_and_shutdown_preserve_all_evidence(self): + state, phase = self.cleanup_case(65, 66) + self.assertEqual(state['exit'], 65); self.assertEqual(phase['exit'], 65) + self.assertIn('66', state['cleanupFailure']) + + def test_passing_xctest_cleanup_timeout_fails_observer_and_preserves_reset(self): + state, phase = self.cleanup_case(0, 0) + self.assertEqual(state['exit'], 1); self.assertEqual(phase['exit'], 1) + self.assertIsNone(state['cleanupFailure']) + self.assertNotIn('validationError', phase) + + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/test_ci_policy.py b/scripts/tests/test_ci_policy.py index 99254fed..5d3164e6 100644 --- a/scripts/tests/test_ci_policy.py +++ b/scripts/tests/test_ci_policy.py @@ -58,7 +58,7 @@ def test_ci_supports_automatic_and_manual_runs(self) -> None: def test_native_jobs_are_dispatch_only_and_keep_their_workloads(self) -> None: for job_id in ("strict-concurrency", "ios-tests", "watch-tests", "smoke-tests"): - self.assertIn(" if: ${{ github.event_name == 'workflow_dispatch' }}\n", job_block(self.workflow, job_id)) + self.assertIn(" if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'automatic-cost' }}\n", job_block(self.workflow, job_id)) lint_job = job_block(self.workflow, "swiftlint") smoke_job = job_block(self.workflow, "smoke-tests") self.assertIn("swiftlint lint", lint_job) @@ -80,7 +80,8 @@ def test_ui_job_runs_only_after_a_successful_manual_gate(self) -> None: ui_job = job_block(self.workflow, "ios-ui-families") self.assertIn( " if: ${{ github.event_name == 'workflow_dispatch' && " - "inputs.ui_validation_mode != 'account-menu-comparison' && success() }}\n", + "inputs.ui_validation_mode != 'account-menu-comparison' && " + "inputs.ui_validation_mode != 'automatic-cost' && success() }}\n", ui_job, ) for required_job in ( @@ -91,7 +92,7 @@ def test_ui_job_runs_only_after_a_successful_manual_gate(self) -> None: "smoke-tests", ): self.assertIn(f" - {required_job}\n", ui_job) - self.assertIn('run: ./scripts/run-ui-tests.sh "${{ matrix.family }}"', ui_job) + self.assertIn('python3 scripts/ui_release_shards.py run --family "${{ matrix.family }}"', ui_job) self.assertIn("family: [iphone, ipad]", ui_job) self.assertIn("fail-fast: false", ui_job) self.assertIn("retention-days: 14", ui_job) diff --git a/scripts/tests/test_ui_release_shards.py b/scripts/tests/test_ui_release_shards.py new file mode 100644 index 00000000..759a927d --- /dev/null +++ b/scripts/tests/test_ui_release_shards.py @@ -0,0 +1,172 @@ +"""Manual-only shard completeness, provenance and cleanup failure controls.""" +import copy +import hashlib +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import Mock, patch + +SCRIPTS = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPTS)) +import ui_release_shards as SHARDS +from release_evidence import load_manifest, read +from pro_release_runner import PhaseFailure + + +class UIShardTests(unittest.TestCase): + def setUp(self): + self.source = dict(revision='candidate', status='', files={'a.swift': 'hash'}) + self.plan = SHARDS.load_plan() + self.tools = dict(xcode=read(SCRIPTS / 'function-risk/policy.json')['tools']['xcode'], + swift='qualified', developer_dir='/Applications/Xcode_26.6.app/Contents/Developer', + sdk='26.5', sdk_build='qualified', architecture='arm64', image='fixture') + + def proof(self, family, name): + return dict(schema=1, complete=True, shutdown=True, revision='candidate', + sourceFingerprint=SHARDS.fingerprint(self.source), tools=self.tools, + family=family, shard=name, started='2026-10-24T12:00:00+00:00', ended='2026-10-24T13:00:00+00:00', + device=dict(family=family, architecture='arm64', platform='iOS Simulator', osVersion='26.2', + osBuildNumber='23C54', deviceId=family+'-'+name), + phases=[dict(phase=phase, tests=tests, receiptSha256='a'*64, + started='2026-10-24T12:01:00+00:00', ended='2026-10-24T12:01:00+00:00') + for phase, tests in self.plan[name]]) + + def test_all_journeys_are_partitioned_once_and_keep_native_sequence(self): + manifest = load_manifest() + actual = [test for phases in self.plan.values() for phase, tests in phases + if manifest['phases'][phase]['kind'] == 'journey' for test in tests] + expected = [test for phase in manifest['ui_journeys'] for test in manifest['phases'][phase]['tests']] + self.assertEqual(sorted(actual), sorted(expected)) + self.assertEqual(len(actual), 53) + self.assertEqual(len(set(actual)), 53) + self.assertEqual([len(self.plan[name][0][1]) for name in SHARDS.SHARDS[:3]], [8, 8, 13]) + self.assertEqual([phase for phase, _ in self.plan['native']], + ['ui-preparation', 'ui-native', 'ui-error-preparation', 'ui-native-purchase-error', 'ui-reset']) + + def test_original_partition_omission_duplicate_and_substitution_fail(self): + original = SHARDS.read(SHARDS.PLAN) + for mutation in ('missing', 'duplicate', 'other'): + altered = copy.deepcopy(original) + tests = altered['shards']['original-1']['tests'] + if mutation == 'missing': tests.pop() + elif mutation == 'duplicate': tests.append(tests[0]) + else: tests[0] = 'CodexBarIOSUITests/Other/testSubstituted' + with self.subTest(mutation=mutation), patch.object(SHARDS, 'read', return_value=altered), self.assertRaises(ValueError): + SHARDS.load_plan() + + def test_native_schedule_cannot_omit_cleanup_or_change_preparation_order(self): + original = SHARDS.read(SHARDS.PLAN) + for mutation in ('reset', 'order', 'matrix'): + altered = copy.deepcopy(original) + if mutation == 'reset': altered['shards']['native']['cleanup'] = None + elif mutation == 'order': altered['shards']['native']['phases'].reverse() + else: del altered['shards']['synthetic'] + with self.subTest(mutation=mutation), patch.object(SHARDS, 'read', return_value=altered), self.assertRaises(ValueError): + SHARDS.load_plan() + + def test_native_failure_preserves_primary_and_always_attempts_reset(self): + runner = Mock(manifest=load_manifest()) + def phase(name, build): + if name == 'ui-native': raise PhaseFailure('purchase failed', 65) + if name == 'ui-reset': raise PhaseFailure('reset failed', 66) + runner.phase.side_effect = phase + with self.assertRaises(PhaseFailure) as caught: + SHARDS.execute(runner, 'native', self.plan['native']) + self.assertEqual(caught.exception.code, 65) + self.assertEqual([call.args[0] for call in runner.phase.call_args_list], ['ui-preparation', 'ui-native', 'ui-reset']) + self.assertTrue(runner.build.call_args.kwargs['native']) + + def test_non_native_shard_selects_exact_tests_and_does_not_build_storekit(self): + runner = Mock(manifest=load_manifest()) + SHARDS.execute(runner, 'original-1', self.plan['original-1']) + self.assertFalse(runner.build.call_args.kwargs['native']) + self.assertEqual(runner.build.call_args.args[2], self.plan['original-1'][0][1]) + self.assertEqual(runner.phase.call_count, 1) + + def test_failed_execution_shuts_down_without_a_passing_proof(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / 'run' + device = self.proof('iphone', 'native')['device'] + with patch.object(SHARDS, 'source_snapshot', return_value=self.source), \ + patch.object(SHARDS, 'toolchain', return_value=self.tools), \ + patch.object(SHARDS, 'provision', return_value=device), \ + patch.object(SHARDS, 'claim_native_device'), patch.object(SHARDS, 'command'), \ + patch.object(SHARDS, 'Runner'), \ + patch.object(SHARDS, 'execute', side_effect=PhaseFailure('journey failed', 65)), \ + patch.object(SHARDS, 'shutdown', side_effect=PhaseFailure('shutdown failed', 66)) as shutdown: + self.assertEqual(SHARDS.run_shard(output, 'iphone', 'native'), 65) + shutdown.assert_called_once() + self.assertFalse((output / 'proof.json').exists()) + self.assertFalse(read(output / 'outcome.json')['complete']) + + def test_all_twelve_proofs_are_required_for_the_current_source(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + proofs = [self.proof(family, name) for family in SHARDS.FAMILIES for name in SHARDS.SHARDS] + for index, proof in enumerate(proofs): + folder = root / str(index);folder.mkdir();(folder / 'proof.json').write_text(json.dumps(proof)) + self.assertEqual(SHARDS.aggregate(root, self.source)['journeysPerFamily'], 53) + path = root / '0/proof.json' + for key, value in [('complete', False), ('shutdown', False), ('revision', 'other'), + ('sourceFingerprint', 'other'), ('family', 'other')]: + bad = copy.deepcopy(proofs[0]);bad[key] = value;path.write_text(json.dumps(bad)) + with self.subTest(key=key), self.assertRaises(ValueError): SHARDS.aggregate(root, self.source) + path.unlink() + with self.assertRaises(ValueError): SHARDS.aggregate(root, self.source) + + def test_wrong_runtime_substituted_tests_and_missing_reset_are_rejected(self): + for mutation in ('runtime', 'test', 'reset', 'duplicate-device', 'time'): + proof = self.proof('ipad', 'native') + if mutation == 'runtime': proof['device']['osVersion'] = '26.5' + elif mutation == 'test': proof['phases'][1]['tests'] = ['CodexBarIOSUITests/Other/testSubstituted'] + elif mutation == 'reset': proof['phases'].pop() + elif mutation == 'duplicate-device': proof['device']['family'] = 'iphone' + else: proof['phases'][0]['started'] = '2026-10-24T11:00:00+00:00' + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + SHARDS.validate_proof(proof, self.source, 'ipad', 'native', self.plan['native']) + + def test_automatic_job_and_workload_inventory_remain_unchanged(self): + workflow = (SCRIPTS.parent / '.github/workflows/ci.yml').read_text() + from scripts.tests.test_ci_policy import job_block, workflow_trigger_keys + self.assertEqual(workflow_trigger_keys(workflow), {'pull_request', 'workflow_dispatch'}) + lint = job_block(workflow, 'swiftlint') + self.assertIn('timeout-minutes: 10', lint) + self.assertNotIn('ui_release_shards', lint) + self.assertEqual(workflow.count('test_ci_policy.py'), 1) + worker = job_block(workflow, 'ios-ui-families') + self.assertIn("github.event_name == 'workflow_dispatch'", worker) + self.assertIn('timeout-minutes: 90', worker) + self.assertIn('shard: [' + ', '.join(SHARDS.SHARDS) + ']', worker) + self.assertIn('verify-release-toolchain.py', worker) + self.assertIn('ui_release_shards.py aggregate', job_block(workflow, 'ios-ui-tests')) + + def test_manual_notifier_permission_and_full_workflow_scope(self): + workflow = (SCRIPTS.parent / '.github/workflows/ci.yml').read_text() + from scripts.tests.test_ci_policy import job_block + aggregate = job_block(workflow, 'ios-ui-tests') + marker = ' - name: Notify Chief of manual UI aggregate outcome\n' + self.assertEqual(workflow.count(marker), 1) + self.assertEqual(workflow.count('pull-requests: write'), 1) + self.assertNotIn('issues: write', workflow) + self.assertIn(' pull-requests: write\n', aggregate) + notifier = aggregate[aggregate.index(marker):] + self.assertIn('always() && !cancelled()', notifier) + self.assertIn("github.ref != 'refs/heads/main'", notifier) + self.assertIn(' retries: 0\n', notifier) + self.assertIn(' debug: false\n', notifier) + script = notifier.split(' script: |\n', 1)[1] + script = '\n'.join(line[12:] for line in script.splitlines() if line.startswith(' ')) + '\n' + self.assertEqual(hashlib.sha256(script.encode()).hexdigest(), + '4f299b77c869f5674c5fc63a9af4cd208a0ac17c39a43b8b88b8538763229ead') + # Removing the approved step and its job-level grant must recover the + # pre-relay workflow plus the explicit manual cost-only mode. The separate + # manual cost controls recover the complete older workflow byte-for-byte. + restored = workflow.replace(notifier, '').replace(' pull-requests: write\n', '', 1) + self.assertEqual(hashlib.sha256(restored.encode()).hexdigest(), + '0c10b64a94fbcdd2cf9eb120f42f31bcb3e3d3b91ea7e5de8a9c3b848188c357') + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/ui-release-shards.json b/scripts/ui-release-shards.json new file mode 100644 index 00000000..09d70ef9 --- /dev/null +++ b/scripts/ui-release-shards.json @@ -0,0 +1,65 @@ +{ + "schema": 1, + "timing_run": "https://github.com/hemsoft-dev/codexbar-ios/actions/runs/39810829566", + "shards": { + "original-1": { + "phase": "ui-original", + "tests": [ + "CodexBarIOSUITests/AccountJourneysUITests/testGeminiOnlyDashboardShowsCodingSetupAndKeepsSelectionOnRelaunch", + "CodexBarIOSUITests/AccountJourneysUITests/testGitHubBillingPermissionDisclosure", + "CodexBarIOSUITests/AccountJourneysUITests/testRefreshRecoveryHistoryAndAccountDeepLink", + "CodexBarIOSUITests/AccountJourneysUITests/testTwoCodexAccountsKeepSeparateUsageAfterRelaunch", + "CodexBarIOSUITests/CodexCreditsPoolUITests/testCompactCardHeadersAndIndependentControls", + "CodexBarIOSUITests/GrokSignInUITests/testGuidedSetupCancellationReconnectAndRemoval", + "CodexBarIOSUITests/OpenCodeSignInUITests/testDisconnectedAccountOffersGuidedSignIn", + "CodexBarIOSUITests/SettingsEvidenceUITests/testPendingDuplicateGroupBlocksDoneUntilCorrected" + ] + }, + "original-2": { + "phase": "ui-original", + "tests": [ + "CodexBarIOSUITests/AccountJourneysUITests/testAccountSetupPersistsGroupAndCredentialAtAccessibilitySize", + "CodexBarIOSUITests/AccountJourneysUITests/testSavedGeminiRingIsSelectedInVisualizationMenu", + "CodexBarIOSUITests/ClaudeUsageUITests/testWindowLabelsAndSavedCustomizationForProAndMax", + "CodexBarIOSUITests/CodexCreditsPoolUITests/testThirtyDayWindowLabelAndSavedVisibility", + "CodexBarIOSUITests/GreptileAllowanceUITests/testDetailReconnectStaysInApp", + "CodexBarIOSUITests/GrokSignInUITests/testGrokAndCursorMetersStaySeparateAndNoAllowanceIsUnavailable", + "CodexBarIOSUITests/GrokSignInUITests/testSavedGrokLayoutAndWeeklyStates", + "CodexBarIOSUITests/SettingsEvidenceUITests/testEverySettingsDestinationAndDoneBack" + ] + }, + "original-3": { + "phase": "ui-original", + "tests": [ + "CodexBarIOSUITests/AccountJourneysUITests/testCodingOnlyGeminiChoicesSurvivePartialAndFailedRefresh", + "CodexBarIOSUITests/AccountJourneysUITests/testGitHubBillingActionableWarningStaysInline", + "CodexBarIOSUITests/AccountJourneysUITests/testGitHubBillingAllowanceStates", + "CodexBarIOSUITests/AccountJourneysUITests/testGitHubBillingProductSummaryAndRoutinePlacement", + "CodexBarIOSUITests/AccountJourneysUITests/testSixGoogleChoicesAndIndependentCustomizationPersist", + "CodexBarIOSUITests/CodexCreditsPoolUITests/testOptInBalanceStatesAndSavedAccountChoices", + "CodexBarIOSUITests/CodexCreditsPoolUITests/testPreciseSubscriptionPillsOnExpandedAndCollapsedCards", + "CodexBarIOSUITests/GreptileAllowanceUITests/testReviewHistoryAndBillingAvailabilityStates", + "CodexBarIOSUITests/GrokSignInUITests/testCursorFreshPercentagesBotAndSavedChoices", + "CodexBarIOSUITests/GrokSignInUITests/testCursorStaleSignInReconnectCancellationAndValidZero", + "CodexBarIOSUITests/OpenCodeSignInUITests/testVerificationFailureKeepsAccountDisconnectedAndAllowsRetry", + "CodexBarIOSUITests/SettingsEvidenceUITests/testSettingsDismissalSignalRejectsTheBackgroundGear", + "CodexBarIOSUITests/SettingsEvidenceUITests/testSyntheticGrantedAndDeniedNotificationFeedback" + ] + }, + "supplemental": { + "phase": "ui-supplemental" + }, + "synthetic": { + "phase": "ui-synthetic" + }, + "native": { + "phases": [ + "ui-preparation", + "ui-native", + "ui-error-preparation", + "ui-native-purchase-error" + ], + "cleanup": "ui-reset" + } + } +} diff --git a/scripts/ui_release_shards.py b/scripts/ui_release_shards.py new file mode 100644 index 00000000..3109bcb1 --- /dev/null +++ b/scripts/ui_release_shards.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Manual-only UI shard execution and fail-closed cross-worker proof aggregation.""" +import argparse +import copy +from datetime import datetime +import hashlib +import json +from pathlib import Path +import subprocess +import sys + +from pro_release_runner import Runner, PhaseFailure, claim_native_device, command, persist, provision, sequence, shutdown +from release_evidence import ROOT, digest, load_manifest, now, read, require_pinned, source_snapshot, toolchain, write + +PLAN = ROOT / 'scripts/ui-release-shards.json' +SHARDS = ('original-1', 'original-2', 'original-3', 'supplemental', 'synthetic', 'native') +FAMILIES = ('iphone', 'ipad') + + +def load_plan(): + manifest, plan = load_manifest(), read(PLAN) + if plan.get('schema') != 1 or tuple(plan.get('shards', {})) != SHARDS: + raise ValueError('UI shard inventory differs from the manual matrix') + result = {} + for name, config in plan['shards'].items(): + if name.startswith('original-'): + if config.get('phase') != 'ui-original': + raise ValueError('Original shards must select original journeys') + tests = config.get('tests', []) + if not tests or len(tests) != len(set(tests)): + raise ValueError('Empty or duplicate original shard selectors') + result[name] = [('ui-original', sorted(tests))] + else: + expected = (['ui-supplemental'] if name == 'supplemental' else ['ui-synthetic'] if name == 'synthetic' + else ['ui-preparation', 'ui-native', 'ui-error-preparation', 'ui-native-purchase-error', 'ui-reset']) + selected = ([config.get('phase')] if 'phase' in config else config.get('phases', []) + [config.get('cleanup')]) + if selected != expected: + raise ValueError('Missing, reordered or unexpected shard phases') + result[name] = [(phase, sorted(manifest['phases'][phase]['tests'])) for phase in expected] + original = [test for name in SHARDS[:3] for _, tests in result[name] for test in tests] + if sorted(original) != sorted(manifest['phases']['ui-original']['tests']): + raise ValueError('Original shard partition omitted, duplicated or substituted a journey') + return result + + +def fingerprint(source): + return hashlib.sha256(json.dumps(source['files'], sort_keys=True).encode()).hexdigest() + + +def execute(runner, name, phases): + native = name == 'native' + scheme = runner.manifest['phases'][phases[0][0]]['scheme'] + selectors = sorted({test for _, tests in phases for test in tests}) + runner.manifest = copy.deepcopy(runner.manifest) + for phase, tests in phases: + runner.manifest['phases'][phase].update(tests=tests, count=len(tests)) + build = runner.build(name, scheme, selectors, native=native) + if native: + state = sequence([phase for phase, _ in phases[:-1]], lambda phase: runner.phase(phase, build), + lambda: runner.phase('ui-reset', build)) + if state['exit']: + raise PhaseFailure(state['primaryFailure'] or state['cleanupFailure'], state['exit']) + else: + runner.phase(phases[0][0], build) + + +def validate_proof(proof, source, family, name, phases): + if (proof.get('schema') != 1 or proof.get('complete') is not True or proof.get('shutdown') is not True + or proof.get('revision') != source['revision'] or source['status'] + or proof.get('sourceFingerprint') != fingerprint(source) + or proof.get('family') != family or proof.get('shard') != name): + raise ValueError('Incomplete, stale or substituted shard proof') + device = proof['device'] + require_pinned(proof['tools'], device) + if device['family'] != family or device['architecture'] != 'arm64' or device['platform'] != 'iOS Simulator': + raise ValueError('Shard device family/platform differs') + rows = proof['phases'] + if [(row['phase'], row['tests']) for row in rows] != phases: + raise ValueError('Shard execution identities or schedule differ') + start, end = datetime.fromisoformat(proof['started']), datetime.fromisoformat(proof['ended']) + if end < start: + raise ValueError('Shard time interval is reversed') + previous = start + for row in rows: + a, b = datetime.fromisoformat(row['started']), datetime.fromisoformat(row['ended']) + if not previous <= a <= b <= end or len(row['receiptSha256']) != 64: + raise ValueError('Shard phase time/provenance differs') + previous = b + + +def run_shard(output, family, name): + output.mkdir(parents=True, exist_ok=False) + failure, runner, device, source, tools = None, None, None, None, None + started = now() + phases = load_plan()[name] + try: + source, tools = source_snapshot(), toolchain() + require_pinned(tools) + write(output / 'source.json', source) + write(output / 'toolchain.json', tools) + device = provision(output, family, tools) + require_pinned(tools, device) + write(output / 'device.json', device) + command(output, 'boot-device', ['xcrun', 'simctl', 'boot', device['deviceId']]) + command(output, 'boot-ready', ['xcrun', 'simctl', 'bootstatus', device['deviceId'], '-b']) + if name == 'native': + claim_native_device(output / 'ownership.json', output, device) + runner = Runner(output, device, source, tools) + execute(runner, name, phases) + if [row['phase'] for row in runner.receipts] != [phase for phase, _ in phases]: + raise ValueError('Shard completed without all required phases') + runner.unchanged() + except (PhaseFailure, ValueError, KeyError, TypeError, OSError, subprocess.CalledProcessError) as error: + failure = error if isinstance(error, PhaseFailure) else PhaseFailure(str(error)) + finally: + if device: + try: + shutdown(output, device) + except (PhaseFailure, ValueError, KeyError, TypeError, OSError, subprocess.CalledProcessError) as error: + failure = failure or PhaseFailure(str(error)) + if failure is None: + try: + rows = [read(item['receipt']) for item in runner.receipts] + proof = dict(schema=1, complete=True, shutdown=True, started=started, ended=now(), + revision=source['revision'], sourceFingerprint=fingerprint(source), tools=tools, + device=device, family=family, shard=name, + phases=[dict(phase=row['phase'], tests=row['executedTests'], started=row['started'], ended=row['ended'], + receiptSha256=digest(item['receipt'])) for row, item in zip(rows, runner.receipts)]) + validate_proof(proof, source, family, name, phases) + write(output / 'proof.json', proof) + except (ValueError, KeyError, TypeError, OSError) as error: + failure = PhaseFailure(str(error)) + failure = persist(output / 'outcome.json', dict(complete=failure is None, exit=failure.code if failure else 0, + error=str(failure) if failure else None, started=started, ended=now()), failure) + if failure: + print(str(failure), file=sys.stderr) + return failure.code if failure else 0 + + +def aggregate(directory, source): + plan = load_plan() + expected = {(family, name) for family in FAMILIES for name in SHARDS} + proofs = [read(path) for path in directory.rglob('proof.json')] + keys = [(proof.get('family'), proof.get('shard')) for proof in proofs] + if len(keys) != len(expected) or set(keys) != expected: + raise ValueError('Missing, duplicate or unexpected UI shard proofs') + devices = [proof['device']['deviceId'] for proof in proofs] + if len(devices) != len(set(devices)): + raise ValueError('Each UI shard requires a newly owned simulator') + for proof in proofs: + family, name = proof['family'], proof['shard'] + validate_proof(proof, source, family, name, plan[name]) + return dict(complete=True, revision=source['revision'], families=list(FAMILIES), shards=len(proofs), + journeysPerFamily=sum(load_manifest()['phases'][phase]['count'] for phase in load_manifest()['ui_journeys'])) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('mode', choices=['run', 'aggregate']) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--family', choices=FAMILIES) + parser.add_argument('--shard', choices=SHARDS) + parser.add_argument('--proofs', type=Path) + args = parser.parse_args() + try: + if args.mode == 'run': + if not args.family or not args.shard: + raise ValueError('A shard and family are required') + return run_shard(args.output.resolve(), args.family, args.shard) + if args.proofs is None: + raise ValueError('Proof directory is required') + result = aggregate(args.proofs, source_snapshot()) + with args.output.open('x') as stream: + json.dump(result, stream, indent=2) + print(json.dumps(result)) + return 0 + except (ValueError, KeyError, TypeError, OSError, subprocess.CalledProcessError) as error: + print(str(error), file=sys.stderr) + return 1 + + +if __name__ == '__main__': + sys.exit(main())