From a1e80c358f753f9c6319faf6bc0b90f6cdc580ea Mon Sep 17 00:00:00 2001 From: HemSoft Date: Sat, 24 Oct 2026 18:02:06 -0400 Subject: [PATCH 01/11] fix: shard manual iOS UI qualification by measured runtime (#517) --- .github/workflows/ci.yml | 72 ++++---- CHANGELOG.md | 4 + CI-POLICY.md | 32 ++++ UI-TESTING.md | 44 ++++- docs/ci/issue-517-baseline.json | 82 +++++++++ scripts/pro-release-phases.json | 6 +- scripts/release_evidence.py | 6 +- scripts/tests/test_ci_policy.py | 2 +- scripts/tests/test_pro_release_validation.py | 2 +- scripts/tests/test_ui_release_shards.py | 146 +++++++++++++++ scripts/ui-release-shards.json | 65 +++++++ scripts/ui_release_shards.py | 183 +++++++++++++++++++ 12 files changed, 593 insertions(+), 51 deletions(-) create mode 100644 docs/ci/issue-517-baseline.json create mode 100644 scripts/tests/test_ui_release_shards.py create mode 100644 scripts/ui-release-shards.json create mode 100644 scripts/ui_release_shards.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5cef10b1..e0e83dad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -224,7 +224,7 @@ jobs: run: python3 -m unittest discover -s scripts/tests -p 'test_perfection_runner.py' -v ios-ui-families: - name: iOS UI journeys (${{ matrix.family }}) + name: iOS UI journeys (${{ matrix.family }}, ${{ matrix.shard }}) if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'account-menu-comparison' && success() }} needs: - swiftlint @@ -240,6 +240,7 @@ jobs: fail-fast: false matrix: family: [iphone, ipad] + shard: [original-1, original-2, original-3, supplemental, synthetic, native] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -249,52 +250,39 @@ jobs: - name: Verify pinned release toolchain and installed runtimes run: python3 scripts/verify-release-toolchain.py --output "$RUNNER_TEMP/release-toolchain.json" - - name: Create an owned UI simulator - id: ui-device - run: | - ./scripts/run-pro-release-validation.sh provision --family "${{ matrix.family }}" --output "$RUNNER_TEMP/ui-device" - echo "id=$(jq -r .device.deviceId "$RUNNER_TEMP/ui-device/ownership.json")" >> "$GITHUB_OUTPUT" + - name: Verify manual shard failure fixtures + if: ${{ matrix.family == 'iphone' && matrix.shard == 'original-1' }} + run: python3 -m unittest scripts.tests.test_ui_release_shards -v - name: Run GitHub Billing API fixtures - if: ${{ matrix.family == 'iphone' }} + if: ${{ matrix.family == 'iphone' && matrix.shard == 'original-1' }} run: xcrun swift run GitHubBillingFixtureTests - - name: Run UI journeys - id: ui-base - if: ${{ !cancelled() && steps.ui-device.outcome == 'success' }} - env: - UI_TEST_REQUIRE_DEVICE_ID: '1' - UI_TEST_OWNERSHIP_RECEIPT: ${{ runner.temp }}/ui-device/ownership.json - UI_TEST_RESULTS_DIR: ${{ runner.temp }}/ui-tests - UI_TEST_DEVICE_ID: ${{ steps.ui-device.outputs.id }} - run: ./scripts/run-ui-tests.sh "${{ matrix.family }}" - - - name: Run supplemental and commerce release phases - if: ${{ !cancelled() && steps.ui-base.outcome == 'success' }} + - name: Run the exact UI shard on a new qualified simulator + id: ui-shard run: >- - ./scripts/run-pro-release-validation.sh ui --family "${{ matrix.family }}" - --ownership "$RUNNER_TEMP/ui-device/ownership.json" - --base-results "$RUNNER_TEMP/ui-tests" --output "$RUNNER_TEMP/pro-ui" + python3 scripts/ui_release_shards.py run --family "${{ matrix.family }}" + --shard "${{ matrix.shard }}" --output "$RUNNER_TEMP/ui-shard" - - name: Shut down the owned UI simulator - if: ${{ always() && steps.ui-device.outcome == 'success' }} - run: >- - ./scripts/run-pro-release-validation.sh shutdown --family "${{ matrix.family }}" - --ownership "$RUNNER_TEMP/ui-device/ownership.json" --output "$RUNNER_TEMP/ui-shutdown" + - name: Publish compact shard proof + if: ${{ !cancelled() && steps.ui-shard.outcome == 'success' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ui-shard-proof-${{ matrix.family }}-${{ matrix.shard }} + path: ${{ runner.temp }}/ui-shard/proof.json + if-no-files-found: error + overwrite: true + retention-days: 14 - name: Preserve UI test results and failure screenshots if: ${{ always() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: CodexBarIOSUITests-${{ matrix.family }}-${{ github.run_id }}-${{ github.run_attempt }} + name: CodexBarIOSUITests-${{ matrix.family }}-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }} path: | - ${{ runner.temp }}/ui-tests - ${{ runner.temp }}/pro-ui - ${{ runner.temp }}/ui-device - ${{ runner.temp }}/ui-shutdown + ${{ runner.temp }}/ui-shard ${{ runner.temp }}/release-toolchain.json - !${{ runner.temp }}/ui-tests/*/DerivedData-*/** - !${{ runner.temp }}/pro-ui/DerivedData-*/** + !${{ runner.temp }}/ui-shard/DerivedData-*/** if-no-files-found: warn retention-days: 14 @@ -302,6 +290,9 @@ jobs: name: Full iOS UI validation if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'account-menu-comparison' }} needs: ios-ui-families + permissions: + actions: read + contents: read runs-on: ubuntu-latest timeout-minutes: 3 steps: @@ -317,6 +308,19 @@ jobs: exit 1 fi + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Require exact-head proofs for all twelve shards + env: + GH_TOKEN: ${{ github.token }} + run: | + gh run download "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" \ + --pattern 'ui-shard-proof-*' --dir "$RUNNER_TEMP/ui-proof" + python3 scripts/ui_release_shards.py aggregate --proofs "$RUNNER_TEMP/ui-proof" \ + --output "$RUNNER_TEMP/full-ui-proof.json" + account-menu-comparison: name: Account menu comparison if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode == 'account-menu-comparison' && success() }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 79e07dbb..d802ed83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,10 @@ building, testing, or releasing the app. ### Developer Experience +- Split manual iPhone and iPad UI qualification into measured shards while + preserving the 90-minute worker limit and automatic PR checks. + ([#517](https://github.com/hemsoft-dev/codexbar-ios/issues/517)) + - Trace native StoreKit status delivery on the existing test receiver, preserve status and clock diagnostics without transaction identifiers, and verify stable test-store cleanup. ([#470](https://github.com/hemsoft-dev/codexbar-ios/issues/470)) - Record the App Review submission of 2.0.0 build 5 and keep the shipped diff --git a/CI-POLICY.md b/CI-POLICY.md index 87c3f38c..f3812f46 100644 --- a/CI-POLICY.md +++ b/CI-POLICY.md @@ -306,3 +306,35 @@ the Germany preparation and storefront phases or function-risk scoring had run. Its 25-minute test step covered the native build, 1,106 ordinary tests, native preparation and 25 native commerce tests. The new budget leaves room for the remaining phases and scoring without retries. + +## Manual UI sharding + +[Issue #517](https://github.com/hemsoft-dev/codexbar-ios/issues/517) partitions only +`workflow_dispatch` UI work. Each iPhone/iPad family has three original shards, +one supplemental shard, one synthetic-commerce shard and one native-commerce +shard. Each worker retains its 90-minute timeout; no timeout increase is used. +The existing Xcode 26.6 / 17F113 and iOS 26.2 / 23C54 verification remains mandatory. + +The retained [hosted run](https://github.com/hemsoft-dev/codexbar-ios/actions/runs/39810829566) +took 7,214 seconds for the iPad original phase, including build and process +overhead. Its 29 test durations sum to 6,624.54 seconds; the two longest tests +were 1,620.38 and 1,603.84 seconds. Measured greedy partitioning assigns 2,212.66, +2,205.04 and 2,206.84 test-seconds to the original shards. These are historical +allocation inputs, not claimed candidate run times. The implementation PR records +completed candidate measurements and the final exact-head hosted gate result. +The [baseline data](docs/ci/issue-517-baseline.json) retains the source run and +per-test values without downloading the multi-gigabyte result bundles. + +The automatic SwiftLint job, its steps, the four lightweight policy test methods, +triggers, native-job dispatch conditions and automatic timeouts are retained. +The new shard fixture suite runs only manually. Before/after automatic workflow +elapsed time, summed job runtime and API billing values are recorded in the PR; +manual shard durations are reported separately. Neither a queue delay nor an +estimated shard budget is proof that automatic work stayed within the ceiling. + +`Full iOS UI validation` now requires all twelve successful workers and their +current-source proofs. The compact artifacts identify every executed method, +qualified toolchain/runtime, source SHA/fingerprint and owned simulator. The +aggregate rejects missing, duplicate or substituted shards. The 53 journeys per +family include the two SUS tests added after 2.0; the original local 29-test command +is unchanged. StoreKit preparation and final reset cannot be split across workers. diff --git a/UI-TESTING.md b/UI-TESTING.md index 909e1025..731d2463 100644 --- a/UI-TESTING.md +++ b/UI-TESTING.md @@ -21,8 +21,8 @@ attachments and the accessibility hierarchy from the result bundle. Open the ## Complete release phases The base command above still executes exactly 29 original journeys. The -[phase manifest](scripts/pro-release-phases.json) requires 51 user journeys per -family for a release: original 29, supplemental 6, synthetic commerce 14, and +[phase manifest](scripts/pro-release-phases.json) requires 53 user journeys per +family for a release: original 29, supplemental 8, synthetic commerce 14, and native purchase/restore and purchase-error journeys 1 each. The separate marketing capture method is excluded. Normal preparation, purchase-error preparation and a final normal reset add three separate XCTest invocations; @@ -417,14 +417,38 @@ quality jobs pass, `Full iOS UI validation` runs both device families. The release cannot proceed unless every job passes for the candidate SHA. If that SHA changes, dispatch a new run. -The two families run on independent manual workers, each with a 90-minute -limit and its own artifact. Fail-fast is disabled. An always-run aggregate -fails unless both workers succeed, including when a worker is cancelled or -skipped. Each worker requires all 51 exact journey identities plus the three -preparation/reset calls. The original runner retains its separate 29-pass contract. -Every phase requires zero skips or expected failures. GitHub retains both destinations' result bundles, logs, -summaries, and exported failure screenshots for 14 days. See -[CI-POLICY.md](CI-POLICY.md) for dispatch and SHA-verification commands. +Each family runs six independent manual shards: three original-journey partitions, +supplemental, synthetic commerce, and native commerce. The original 29 tests are +balanced using retained hosted durations into groups of 8, 8 and 13. Every shard +keeps the 90-minute limit, pinned Xcode 26.6 / 17F113 and iOS 26.2 / 23C54, +and provisions its own simulator. Fail-fast remains disabled. + +Native preparation, purchase/restore, purchase-error preparation, purchase-error +and the final normal StoreKit reset stay sequential on one worker. The reset is +attempted even after a journey failure. The two SUS evidence journeys are included +in the supplemental eight, so the full gate covers 53 distinct journeys per +family plus the three preparation/reset calls. Marketing capture remains excluded. +The original local runner retains its separate 29-pass contract. + +The always-run aggregate requires all twelve workers to succeed and validates their +compact proofs against the checked-out SHA and source fingerprint. Missing, +duplicate, stale, wrong-runtime or substituted-test proofs fail the gate. Every +phase still requires zero skips or expected failures. Full results, logs and +exported failure screenshots remain available for 14 days, separate from the small +proof artifacts used by the aggregate. See [CI-POLICY.md](CI-POLICY.md) for dispatch +and SHA-verification commands. + +Reproduce one shard from a clean committed checkout on air: + +```sh +DEVELOPER_DIR=/Applications/Xcode_26.6.app/Contents/Developer \ + python3 scripts/ui_release_shards.py run --family iphone --shard original-1 \ + --output /tmp/codexbar-ui-original-1 +``` + +Choose any shard listed in [the shard plan](scripts/ui-release-shards.json), and a +new output directory for each invocation. The command shuts down only its owned +simulator. It does not launch provider sign-in or touch live account storage. To reproduce a failure, rerun the same family with the command above. Every journey resets its own storage, so no simulator-wide data wipe is needed. diff --git a/docs/ci/issue-517-baseline.json b/docs/ci/issue-517-baseline.json new file mode 100644 index 00000000..e93baaf0 --- /dev/null +++ b/docs/ci/issue-517-baseline.json @@ -0,0 +1,82 @@ +{ + "manual_run": "https://github.com/hemsoft-dev/codexbar-ios/actions/runs/39810829566", + "manual_head": "9995bc14ee090d4688a7d31ae1a210dd31b78aad", + "ipad_original_step_seconds": 7214, + "original_test_seconds": { + "CodexBarIOSUITests/AccountJourneysUITests/testAccountSetupPersistsGroupAndCredentialAtAccessibilitySize": 190.38641798496246, + "CodexBarIOSUITests/AccountJourneysUITests/testCodingOnlyGeminiChoicesSurvivePartialAndFailedRefresh": 185.9075390100479, + "CodexBarIOSUITests/AccountJourneysUITests/testGeminiOnlyDashboardShowsCodingSetupAndKeepsSelectionOnRelaunch": 146.40053403377533, + "CodexBarIOSUITests/AccountJourneysUITests/testGitHubBillingActionableWarningStaysInline": 11.181080937385559, + "CodexBarIOSUITests/AccountJourneysUITests/testGitHubBillingAllowanceStates": 49.49990403652191, + "CodexBarIOSUITests/AccountJourneysUITests/testGitHubBillingPermissionDisclosure": 142.06247901916504, + "CodexBarIOSUITests/AccountJourneysUITests/testGitHubBillingProductSummaryAndRoutinePlacement": 56.41284894943237, + "CodexBarIOSUITests/AccountJourneysUITests/testRefreshRecoveryHistoryAndAccountDeepLink": 55.12314808368683, + "CodexBarIOSUITests/AccountJourneysUITests/testSavedGeminiRingIsSelectedInVisualizationMenu": 84.94209694862366, + "CodexBarIOSUITests/AccountJourneysUITests/testSixGoogleChoicesAndIndependentCustomizationPersist": 234.73875200748444, + "CodexBarIOSUITests/AccountJourneysUITests/testTwoCodexAccountsKeepSeparateUsageAfterRelaunch": 74.75701797008514, + "CodexBarIOSUITests/ClaudeUsageUITests/testWindowLabelsAndSavedCustomizationForProAndMax": 1603.8373860120773, + "CodexBarIOSUITests/CodexCreditsPoolUITests/testCompactCardHeadersAndIndependentControls": 1620.3830180168152, + "CodexBarIOSUITests/CodexCreditsPoolUITests/testOptInBalanceStatesAndSavedAccountChoices": 399.53378999233246, + "CodexBarIOSUITests/CodexCreditsPoolUITests/testPreciseSubscriptionPillsOnExpandedAndCollapsedCards": 267.53720700740814, + "CodexBarIOSUITests/CodexCreditsPoolUITests/testThirtyDayWindowLabelAndSavedVisibility": 122.59079003334045, + "CodexBarIOSUITests/GreptileAllowanceUITests/testDetailReconnectStaysInApp": 25.48348903656006, + "CodexBarIOSUITests/GreptileAllowanceUITests/testReviewHistoryAndBillingAvailabilityStates": 451.6969130039215, + "CodexBarIOSUITests/GrokSignInUITests/testCursorFreshPercentagesBotAndSavedChoices": 94.11898100376129, + "CodexBarIOSUITests/GrokSignInUITests/testCursorStaleSignInReconnectCancellationAndValidZero": 71.34450507164001, + "CodexBarIOSUITests/GrokSignInUITests/testGrokAndCursorMetersStaySeparateAndNoAllowanceIsUnavailable": 41.37902390956879, + "CodexBarIOSUITests/GrokSignInUITests/testGuidedSetupCancellationReconnectAndRemoval": 35.021477937698364, + "CodexBarIOSUITests/GrokSignInUITests/testSavedGrokLayoutAndWeeklyStates": 71.8157240152359, + "CodexBarIOSUITests/OpenCodeSignInUITests/testDisconnectedAccountOffersGuidedSignIn": 92.38425695896149, + "CodexBarIOSUITests/OpenCodeSignInUITests/testVerificationFailureKeepsAccountDisconnectedAndAllowsRetry": 255.2149029970169, + "CodexBarIOSUITests/SettingsEvidenceUITests/testEverySettingsDestinationAndDoneBack": 64.60541903972626, + "CodexBarIOSUITests/SettingsEvidenceUITests/testPendingDuplicateGroupBlocksDoneUntilCorrected": 46.53203594684601, + "CodexBarIOSUITests/SettingsEvidenceUITests/testSettingsDismissalSignalRejectsTheBackgroundGear": 15.621724009513855, + "CodexBarIOSUITests/SettingsEvidenceUITests/testSyntheticGrantedAndDeniedNotificationFeedback": 114.02973997592926 + }, + "automatic_run": "https://github.com/hemsoft-dev/codexbar-ios/actions/runs/40010770987", + "automatic_head": "9dec9e34b8b01093624ee714e76e129c5d5eb7bd", + "automatic_workflow_elapsed_seconds": 3215.0, + "automatic_summed_job_seconds": 43.0, + "automatic_jobs": [ + { + "name": "SwiftLint", + "seconds": 43.0, + "conclusion": "success" + }, + { + "name": "Strict concurrency", + "seconds": 0, + "conclusion": "skipped" + }, + { + "name": "watchOS tests", + "seconds": 0, + "conclusion": "skipped" + }, + { + "name": "iOS tests", + "seconds": 0, + "conclusion": "skipped" + }, + { + "name": "SwiftPM smoke tests", + "seconds": 0, + "conclusion": "skipped" + }, + { + "name": "iOS UI journeys (${{ matrix.family }})", + "seconds": 0, + "conclusion": "skipped" + }, + { + "name": "Account menu comparison", + "seconds": 0, + "conclusion": "skipped" + }, + { + "name": "Full iOS UI validation", + "seconds": 0, + "conclusion": "skipped" + } + ] +} diff --git a/scripts/pro-release-phases.json b/scripts/pro-release-phases.json index 88bba368..34ef7662 100644 --- a/scripts/pro-release-phases.json +++ b/scripts/pro-release-phases.json @@ -39,14 +39,16 @@ }, "ui-supplemental": { "scheme": "CodexBarIOSUITests", - "count": 6, + "count": 8, "tests": [ "CodexBarIOSUITests/FreeTierJourneyTests/testDowngradeAndUpgradeRestoreAccountsAndSavedAlertPreference", "CodexBarIOSUITests/FreeTierJourneyTests/testFreeHistoryRangesAreLockedAndDoNotLeaveTheSevenDayChart", "CodexBarIOSUITests/FreeTierJourneyTests/testFreeLimitAndAlertsUseTheSharedProDestination", "CodexBarIOSUITests/FreeTierJourneyTests/testPausedAccountsKeepValuesAndCanSwitchAtLargeTextSize", "CodexBarIOSUITests/GrokSignInUITests/testCursorSubscriptionPillsAreAccountScoped", - "CodexBarIOSUITests/GrokSignInUITests/testXSubscriptionGuidedSetupUsesOneXAISignIn" + "CodexBarIOSUITests/GrokSignInUITests/testXSubscriptionGuidedSetupUsesOneXAISignIn", + "CodexBarIOSUITests/SubscriptionUtilizationEvidenceUITests/testMissingGooglePoolShowsUnavailableSUS", + "CodexBarIOSUITests/SubscriptionUtilizationEvidenceUITests/testWeeklyBandsAndFleetScore" ], "kind": "journey" }, diff --git a/scripts/release_evidence.py b/scripts/release_evidence.py index 335f1fa4..a9f45a98 100644 --- a/scripts/release_evidence.py +++ b/scripts/release_evidence.py @@ -108,11 +108,11 @@ def load_manifest(path=MANIFEST): if not all(TEST_ID.fullmatch(test) for test in tests): raise ValueError(f'Only exact target/class/method selectors are allowed: {name}') journeys = [test for name in manifest['ui_journeys'] for test in manifest['phases'][name]['tests']] - if len(journeys) != 51 or len(set(journeys)) != 51: - raise ValueError('The UI partition must contain exactly 51 distinct user journeys') + if len(journeys) != 53 or len(set(journeys)) != 53: + raise ValueError('The UI partition must contain exactly 53 distinct user journeys') if set(journeys) & set(manifest['excluded_marketing']): raise ValueError('Marketing capture cannot be part of release journeys') - if [manifest['phases'][name]['count'] for name in manifest['ui_journeys']] != [29, 6, 14, 1, 1]: + if [manifest['phases'][name]['count'] for name in manifest['ui_journeys']] != [29, 8, 14, 1, 1]: raise ValueError('UI journey partition changed; reconcile the release contract') if [manifest['phases'][name]['count'] for name in manifest['native_phases']] != [1, 25, 1, 1]: raise ValueError('Native phase partition changed; reconcile the release contract') diff --git a/scripts/tests/test_ci_policy.py b/scripts/tests/test_ci_policy.py index 99254fed..1c870993 100644 --- a/scripts/tests/test_ci_policy.py +++ b/scripts/tests/test_ci_policy.py @@ -91,7 +91,7 @@ def test_ui_job_runs_only_after_a_successful_manual_gate(self) -> None: "smoke-tests", ): self.assertIn(f" - {required_job}\n", ui_job) - self.assertIn('run: ./scripts/run-ui-tests.sh "${{ matrix.family }}"', ui_job) + self.assertIn('python3 scripts/ui_release_shards.py run --family "${{ matrix.family }}"', ui_job) self.assertIn("family: [iphone, ipad]", ui_job) self.assertIn("fail-fast: false", ui_job) self.assertIn("retention-days: 14", ui_job) diff --git a/scripts/tests/test_pro_release_validation.py b/scripts/tests/test_pro_release_validation.py index 7a64f70a..442e7c44 100644 --- a/scripts/tests/test_pro_release_validation.py +++ b/scripts/tests/test_pro_release_validation.py @@ -122,7 +122,7 @@ def test_explicit_partitions_match_current_source(self): for name in manifest['ui_journeys']: declared.update(manifest['phases'][name]['tests']) self.assertEqual(discovered, declared) - self.assertEqual(len(discovered), 52) + self.assertEqual(len(discovered), 54) self.assertEqual(len(manifest['shared_pro_tests']), 197) self.assertEqual(len(manifest['shared_pro_sources']), 22) diff --git a/scripts/tests/test_ui_release_shards.py b/scripts/tests/test_ui_release_shards.py new file mode 100644 index 00000000..0771519f --- /dev/null +++ b/scripts/tests/test_ui_release_shards.py @@ -0,0 +1,146 @@ +"""Manual-only shard completeness, provenance and cleanup failure controls.""" +import copy +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import Mock, patch + +SCRIPTS = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPTS)) +import ui_release_shards as SHARDS +from release_evidence import load_manifest, read +from pro_release_runner import PhaseFailure + + +class UIShardTests(unittest.TestCase): + def setUp(self): + self.source = dict(revision='candidate', status='', files={'a.swift': 'hash'}) + self.plan = SHARDS.load_plan() + self.tools = dict(xcode=read(SCRIPTS / 'function-risk/policy.json')['tools']['xcode'], + swift='qualified', developer_dir='/Applications/Xcode_26.6.app/Contents/Developer', + sdk='26.5', sdk_build='qualified', architecture='arm64', image='fixture') + + def proof(self, family, name): + return dict(schema=1, complete=True, shutdown=True, revision='candidate', + sourceFingerprint=SHARDS.fingerprint(self.source), tools=self.tools, + family=family, shard=name, started='2026-10-24T12:00:00+00:00', ended='2026-10-24T13:00:00+00:00', + device=dict(family=family, architecture='arm64', platform='iOS Simulator', osVersion='26.2', + osBuildNumber='23C54', deviceId=family+'-'+name), + phases=[dict(phase=phase, tests=tests, receiptSha256='a'*64, + started='2026-10-24T12:01:00+00:00', ended='2026-10-24T12:01:00+00:00') + for phase, tests in self.plan[name]]) + + def test_all_journeys_are_partitioned_once_and_keep_native_sequence(self): + manifest = load_manifest() + actual = [test for phases in self.plan.values() for phase, tests in phases + if manifest['phases'][phase]['kind'] == 'journey' for test in tests] + expected = [test for phase in manifest['ui_journeys'] for test in manifest['phases'][phase]['tests']] + self.assertEqual(sorted(actual), sorted(expected)) + self.assertEqual(len(actual), 53) + self.assertEqual(len(set(actual)), 53) + self.assertEqual([len(self.plan[name][0][1]) for name in SHARDS.SHARDS[:3]], [8, 8, 13]) + self.assertEqual([phase for phase, _ in self.plan['native']], + ['ui-preparation', 'ui-native', 'ui-error-preparation', 'ui-native-purchase-error', 'ui-reset']) + + def test_original_partition_omission_duplicate_and_substitution_fail(self): + original = SHARDS.read(SHARDS.PLAN) + for mutation in ('missing', 'duplicate', 'other'): + altered = copy.deepcopy(original) + tests = altered['shards']['original-1']['tests'] + if mutation == 'missing': tests.pop() + elif mutation == 'duplicate': tests.append(tests[0]) + else: tests[0] = 'CodexBarIOSUITests/Other/testSubstituted' + with self.subTest(mutation=mutation), patch.object(SHARDS, 'read', return_value=altered), self.assertRaises(ValueError): + SHARDS.load_plan() + + def test_native_schedule_cannot_omit_cleanup_or_change_preparation_order(self): + original = SHARDS.read(SHARDS.PLAN) + for mutation in ('reset', 'order', 'matrix'): + altered = copy.deepcopy(original) + if mutation == 'reset': altered['shards']['native']['cleanup'] = None + elif mutation == 'order': altered['shards']['native']['phases'].reverse() + else: del altered['shards']['synthetic'] + with self.subTest(mutation=mutation), patch.object(SHARDS, 'read', return_value=altered), self.assertRaises(ValueError): + SHARDS.load_plan() + + def test_native_failure_preserves_primary_and_always_attempts_reset(self): + runner = Mock(manifest=load_manifest()) + def phase(name, build): + if name == 'ui-native': raise PhaseFailure('purchase failed', 65) + if name == 'ui-reset': raise PhaseFailure('reset failed', 66) + runner.phase.side_effect = phase + with self.assertRaises(PhaseFailure) as caught: + SHARDS.execute(runner, 'native', self.plan['native']) + self.assertEqual(caught.exception.code, 65) + self.assertEqual([call.args[0] for call in runner.phase.call_args_list], ['ui-preparation', 'ui-native', 'ui-reset']) + self.assertTrue(runner.build.call_args.kwargs['native']) + + def test_non_native_shard_selects_exact_tests_and_does_not_build_storekit(self): + runner = Mock(manifest=load_manifest()) + SHARDS.execute(runner, 'original-1', self.plan['original-1']) + self.assertFalse(runner.build.call_args.kwargs['native']) + self.assertEqual(runner.build.call_args.args[2], self.plan['original-1'][0][1]) + self.assertEqual(runner.phase.call_count, 1) + + def test_failed_execution_shuts_down_without_a_passing_proof(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / 'run' + device = self.proof('iphone', 'native')['device'] + with patch.object(SHARDS, 'source_snapshot', return_value=self.source), \ + patch.object(SHARDS, 'toolchain', return_value=self.tools), \ + patch.object(SHARDS, 'provision', return_value=device), \ + patch.object(SHARDS, 'claim_native_device'), patch.object(SHARDS, 'command'), \ + patch.object(SHARDS, 'Runner'), \ + patch.object(SHARDS, 'execute', side_effect=PhaseFailure('journey failed', 65)), \ + patch.object(SHARDS, 'shutdown', side_effect=PhaseFailure('shutdown failed', 66)) as shutdown: + self.assertEqual(SHARDS.run_shard(output, 'iphone', 'native'), 65) + shutdown.assert_called_once() + self.assertFalse((output / 'proof.json').exists()) + self.assertFalse(read(output / 'outcome.json')['complete']) + + def test_all_twelve_proofs_are_required_for_the_current_source(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + proofs = [self.proof(family, name) for family in SHARDS.FAMILIES for name in SHARDS.SHARDS] + for index, proof in enumerate(proofs): + folder = root / str(index);folder.mkdir();(folder / 'proof.json').write_text(json.dumps(proof)) + self.assertEqual(SHARDS.aggregate(root, self.source)['journeysPerFamily'], 53) + path = root / '0/proof.json' + for key, value in [('complete', False), ('shutdown', False), ('revision', 'other'), + ('sourceFingerprint', 'other'), ('family', 'other')]: + bad = copy.deepcopy(proofs[0]);bad[key] = value;path.write_text(json.dumps(bad)) + with self.subTest(key=key), self.assertRaises(ValueError): SHARDS.aggregate(root, self.source) + path.unlink() + with self.assertRaises(ValueError): SHARDS.aggregate(root, self.source) + + def test_wrong_runtime_substituted_tests_and_missing_reset_are_rejected(self): + for mutation in ('runtime', 'test', 'reset', 'duplicate-device', 'time'): + proof = self.proof('ipad', 'native') + if mutation == 'runtime': proof['device']['osVersion'] = '26.5' + elif mutation == 'test': proof['phases'][1]['tests'] = ['CodexBarIOSUITests/Other/testSubstituted'] + elif mutation == 'reset': proof['phases'].pop() + elif mutation == 'duplicate-device': proof['device']['family'] = 'iphone' + else: proof['phases'][0]['started'] = '2026-10-24T11:00:00+00:00' + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + SHARDS.validate_proof(proof, self.source, 'ipad', 'native', self.plan['native']) + + def test_automatic_job_and_workload_inventory_remain_unchanged(self): + workflow = (SCRIPTS.parent / '.github/workflows/ci.yml').read_text() + from scripts.tests.test_ci_policy import job_block, workflow_trigger_keys + self.assertEqual(workflow_trigger_keys(workflow), {'pull_request', 'workflow_dispatch'}) + lint = job_block(workflow, 'swiftlint') + self.assertIn('timeout-minutes: 10', lint) + self.assertNotIn('ui_release_shards', lint) + self.assertEqual(workflow.count('test_ci_policy.py'), 1) + worker = job_block(workflow, 'ios-ui-families') + self.assertIn("github.event_name == 'workflow_dispatch'", worker) + self.assertIn('timeout-minutes: 90', worker) + self.assertIn('shard: [' + ', '.join(SHARDS.SHARDS) + ']', worker) + self.assertIn('verify-release-toolchain.py', worker) + self.assertIn('ui_release_shards.py aggregate', job_block(workflow, 'ios-ui-tests')) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/ui-release-shards.json b/scripts/ui-release-shards.json new file mode 100644 index 00000000..09d70ef9 --- /dev/null +++ b/scripts/ui-release-shards.json @@ -0,0 +1,65 @@ +{ + "schema": 1, + "timing_run": "https://github.com/hemsoft-dev/codexbar-ios/actions/runs/39810829566", + "shards": { + "original-1": { + "phase": "ui-original", + "tests": [ + "CodexBarIOSUITests/AccountJourneysUITests/testGeminiOnlyDashboardShowsCodingSetupAndKeepsSelectionOnRelaunch", + "CodexBarIOSUITests/AccountJourneysUITests/testGitHubBillingPermissionDisclosure", + "CodexBarIOSUITests/AccountJourneysUITests/testRefreshRecoveryHistoryAndAccountDeepLink", + "CodexBarIOSUITests/AccountJourneysUITests/testTwoCodexAccountsKeepSeparateUsageAfterRelaunch", + "CodexBarIOSUITests/CodexCreditsPoolUITests/testCompactCardHeadersAndIndependentControls", + "CodexBarIOSUITests/GrokSignInUITests/testGuidedSetupCancellationReconnectAndRemoval", + "CodexBarIOSUITests/OpenCodeSignInUITests/testDisconnectedAccountOffersGuidedSignIn", + "CodexBarIOSUITests/SettingsEvidenceUITests/testPendingDuplicateGroupBlocksDoneUntilCorrected" + ] + }, + "original-2": { + "phase": "ui-original", + "tests": [ + "CodexBarIOSUITests/AccountJourneysUITests/testAccountSetupPersistsGroupAndCredentialAtAccessibilitySize", + "CodexBarIOSUITests/AccountJourneysUITests/testSavedGeminiRingIsSelectedInVisualizationMenu", + "CodexBarIOSUITests/ClaudeUsageUITests/testWindowLabelsAndSavedCustomizationForProAndMax", + "CodexBarIOSUITests/CodexCreditsPoolUITests/testThirtyDayWindowLabelAndSavedVisibility", + "CodexBarIOSUITests/GreptileAllowanceUITests/testDetailReconnectStaysInApp", + "CodexBarIOSUITests/GrokSignInUITests/testGrokAndCursorMetersStaySeparateAndNoAllowanceIsUnavailable", + "CodexBarIOSUITests/GrokSignInUITests/testSavedGrokLayoutAndWeeklyStates", + "CodexBarIOSUITests/SettingsEvidenceUITests/testEverySettingsDestinationAndDoneBack" + ] + }, + "original-3": { + "phase": "ui-original", + "tests": [ + "CodexBarIOSUITests/AccountJourneysUITests/testCodingOnlyGeminiChoicesSurvivePartialAndFailedRefresh", + "CodexBarIOSUITests/AccountJourneysUITests/testGitHubBillingActionableWarningStaysInline", + "CodexBarIOSUITests/AccountJourneysUITests/testGitHubBillingAllowanceStates", + "CodexBarIOSUITests/AccountJourneysUITests/testGitHubBillingProductSummaryAndRoutinePlacement", + "CodexBarIOSUITests/AccountJourneysUITests/testSixGoogleChoicesAndIndependentCustomizationPersist", + "CodexBarIOSUITests/CodexCreditsPoolUITests/testOptInBalanceStatesAndSavedAccountChoices", + "CodexBarIOSUITests/CodexCreditsPoolUITests/testPreciseSubscriptionPillsOnExpandedAndCollapsedCards", + "CodexBarIOSUITests/GreptileAllowanceUITests/testReviewHistoryAndBillingAvailabilityStates", + "CodexBarIOSUITests/GrokSignInUITests/testCursorFreshPercentagesBotAndSavedChoices", + "CodexBarIOSUITests/GrokSignInUITests/testCursorStaleSignInReconnectCancellationAndValidZero", + "CodexBarIOSUITests/OpenCodeSignInUITests/testVerificationFailureKeepsAccountDisconnectedAndAllowsRetry", + "CodexBarIOSUITests/SettingsEvidenceUITests/testSettingsDismissalSignalRejectsTheBackgroundGear", + "CodexBarIOSUITests/SettingsEvidenceUITests/testSyntheticGrantedAndDeniedNotificationFeedback" + ] + }, + "supplemental": { + "phase": "ui-supplemental" + }, + "synthetic": { + "phase": "ui-synthetic" + }, + "native": { + "phases": [ + "ui-preparation", + "ui-native", + "ui-error-preparation", + "ui-native-purchase-error" + ], + "cleanup": "ui-reset" + } + } +} diff --git a/scripts/ui_release_shards.py b/scripts/ui_release_shards.py new file mode 100644 index 00000000..3109bcb1 --- /dev/null +++ b/scripts/ui_release_shards.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Manual-only UI shard execution and fail-closed cross-worker proof aggregation.""" +import argparse +import copy +from datetime import datetime +import hashlib +import json +from pathlib import Path +import subprocess +import sys + +from pro_release_runner import Runner, PhaseFailure, claim_native_device, command, persist, provision, sequence, shutdown +from release_evidence import ROOT, digest, load_manifest, now, read, require_pinned, source_snapshot, toolchain, write + +PLAN = ROOT / 'scripts/ui-release-shards.json' +SHARDS = ('original-1', 'original-2', 'original-3', 'supplemental', 'synthetic', 'native') +FAMILIES = ('iphone', 'ipad') + + +def load_plan(): + manifest, plan = load_manifest(), read(PLAN) + if plan.get('schema') != 1 or tuple(plan.get('shards', {})) != SHARDS: + raise ValueError('UI shard inventory differs from the manual matrix') + result = {} + for name, config in plan['shards'].items(): + if name.startswith('original-'): + if config.get('phase') != 'ui-original': + raise ValueError('Original shards must select original journeys') + tests = config.get('tests', []) + if not tests or len(tests) != len(set(tests)): + raise ValueError('Empty or duplicate original shard selectors') + result[name] = [('ui-original', sorted(tests))] + else: + expected = (['ui-supplemental'] if name == 'supplemental' else ['ui-synthetic'] if name == 'synthetic' + else ['ui-preparation', 'ui-native', 'ui-error-preparation', 'ui-native-purchase-error', 'ui-reset']) + selected = ([config.get('phase')] if 'phase' in config else config.get('phases', []) + [config.get('cleanup')]) + if selected != expected: + raise ValueError('Missing, reordered or unexpected shard phases') + result[name] = [(phase, sorted(manifest['phases'][phase]['tests'])) for phase in expected] + original = [test for name in SHARDS[:3] for _, tests in result[name] for test in tests] + if sorted(original) != sorted(manifest['phases']['ui-original']['tests']): + raise ValueError('Original shard partition omitted, duplicated or substituted a journey') + return result + + +def fingerprint(source): + return hashlib.sha256(json.dumps(source['files'], sort_keys=True).encode()).hexdigest() + + +def execute(runner, name, phases): + native = name == 'native' + scheme = runner.manifest['phases'][phases[0][0]]['scheme'] + selectors = sorted({test for _, tests in phases for test in tests}) + runner.manifest = copy.deepcopy(runner.manifest) + for phase, tests in phases: + runner.manifest['phases'][phase].update(tests=tests, count=len(tests)) + build = runner.build(name, scheme, selectors, native=native) + if native: + state = sequence([phase for phase, _ in phases[:-1]], lambda phase: runner.phase(phase, build), + lambda: runner.phase('ui-reset', build)) + if state['exit']: + raise PhaseFailure(state['primaryFailure'] or state['cleanupFailure'], state['exit']) + else: + runner.phase(phases[0][0], build) + + +def validate_proof(proof, source, family, name, phases): + if (proof.get('schema') != 1 or proof.get('complete') is not True or proof.get('shutdown') is not True + or proof.get('revision') != source['revision'] or source['status'] + or proof.get('sourceFingerprint') != fingerprint(source) + or proof.get('family') != family or proof.get('shard') != name): + raise ValueError('Incomplete, stale or substituted shard proof') + device = proof['device'] + require_pinned(proof['tools'], device) + if device['family'] != family or device['architecture'] != 'arm64' or device['platform'] != 'iOS Simulator': + raise ValueError('Shard device family/platform differs') + rows = proof['phases'] + if [(row['phase'], row['tests']) for row in rows] != phases: + raise ValueError('Shard execution identities or schedule differ') + start, end = datetime.fromisoformat(proof['started']), datetime.fromisoformat(proof['ended']) + if end < start: + raise ValueError('Shard time interval is reversed') + previous = start + for row in rows: + a, b = datetime.fromisoformat(row['started']), datetime.fromisoformat(row['ended']) + if not previous <= a <= b <= end or len(row['receiptSha256']) != 64: + raise ValueError('Shard phase time/provenance differs') + previous = b + + +def run_shard(output, family, name): + output.mkdir(parents=True, exist_ok=False) + failure, runner, device, source, tools = None, None, None, None, None + started = now() + phases = load_plan()[name] + try: + source, tools = source_snapshot(), toolchain() + require_pinned(tools) + write(output / 'source.json', source) + write(output / 'toolchain.json', tools) + device = provision(output, family, tools) + require_pinned(tools, device) + write(output / 'device.json', device) + command(output, 'boot-device', ['xcrun', 'simctl', 'boot', device['deviceId']]) + command(output, 'boot-ready', ['xcrun', 'simctl', 'bootstatus', device['deviceId'], '-b']) + if name == 'native': + claim_native_device(output / 'ownership.json', output, device) + runner = Runner(output, device, source, tools) + execute(runner, name, phases) + if [row['phase'] for row in runner.receipts] != [phase for phase, _ in phases]: + raise ValueError('Shard completed without all required phases') + runner.unchanged() + except (PhaseFailure, ValueError, KeyError, TypeError, OSError, subprocess.CalledProcessError) as error: + failure = error if isinstance(error, PhaseFailure) else PhaseFailure(str(error)) + finally: + if device: + try: + shutdown(output, device) + except (PhaseFailure, ValueError, KeyError, TypeError, OSError, subprocess.CalledProcessError) as error: + failure = failure or PhaseFailure(str(error)) + if failure is None: + try: + rows = [read(item['receipt']) for item in runner.receipts] + proof = dict(schema=1, complete=True, shutdown=True, started=started, ended=now(), + revision=source['revision'], sourceFingerprint=fingerprint(source), tools=tools, + device=device, family=family, shard=name, + phases=[dict(phase=row['phase'], tests=row['executedTests'], started=row['started'], ended=row['ended'], + receiptSha256=digest(item['receipt'])) for row, item in zip(rows, runner.receipts)]) + validate_proof(proof, source, family, name, phases) + write(output / 'proof.json', proof) + except (ValueError, KeyError, TypeError, OSError) as error: + failure = PhaseFailure(str(error)) + failure = persist(output / 'outcome.json', dict(complete=failure is None, exit=failure.code if failure else 0, + error=str(failure) if failure else None, started=started, ended=now()), failure) + if failure: + print(str(failure), file=sys.stderr) + return failure.code if failure else 0 + + +def aggregate(directory, source): + plan = load_plan() + expected = {(family, name) for family in FAMILIES for name in SHARDS} + proofs = [read(path) for path in directory.rglob('proof.json')] + keys = [(proof.get('family'), proof.get('shard')) for proof in proofs] + if len(keys) != len(expected) or set(keys) != expected: + raise ValueError('Missing, duplicate or unexpected UI shard proofs') + devices = [proof['device']['deviceId'] for proof in proofs] + if len(devices) != len(set(devices)): + raise ValueError('Each UI shard requires a newly owned simulator') + for proof in proofs: + family, name = proof['family'], proof['shard'] + validate_proof(proof, source, family, name, plan[name]) + return dict(complete=True, revision=source['revision'], families=list(FAMILIES), shards=len(proofs), + journeysPerFamily=sum(load_manifest()['phases'][phase]['count'] for phase in load_manifest()['ui_journeys'])) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('mode', choices=['run', 'aggregate']) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--family', choices=FAMILIES) + parser.add_argument('--shard', choices=SHARDS) + parser.add_argument('--proofs', type=Path) + args = parser.parse_args() + try: + if args.mode == 'run': + if not args.family or not args.shard: + raise ValueError('A shard and family are required') + return run_shard(args.output.resolve(), args.family, args.shard) + if args.proofs is None: + raise ValueError('Proof directory is required') + result = aggregate(args.proofs, source_snapshot()) + with args.output.open('x') as stream: + json.dump(result, stream, indent=2) + print(json.dumps(result)) + return 0 + except (ValueError, KeyError, TypeError, OSError, subprocess.CalledProcessError) as error: + print(str(error), file=sys.stderr) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) From ba684b7ee774b47d0801af33e4048983c519ce7d Mon Sep 17 00:00:00 2001 From: HemSoft Date: Sat, 24 Oct 2026 19:36:16 -0400 Subject: [PATCH 02/11] test: tolerate floating-point noise in touch-target dimensions (#517) --- CHANGELOG.md | 3 ++- CodexBarIOSUITests/CodexCreditsPoolUITests.swift | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d802ed83..c7255dec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,8 @@ building, testing, or releasing the app. ### Developer Experience - Split manual iPhone and iPad UI qualification into measured shards while - preserving the 90-minute worker limit and automatic PR checks. + preserving the 90-minute worker limit and automatic PR checks. Keep touch + target assertions from rejecting 44-point controls due to floating-point noise. ([#517](https://github.com/hemsoft-dev/codexbar-ios/issues/517)) - Trace native StoreKit status delivery on the existing test receiver, preserve status and clock diagnostics without transaction identifiers, and verify stable test-store cleanup. ([#470](https://github.com/hemsoft-dev/codexbar-ios/issues/470)) diff --git a/CodexBarIOSUITests/CodexCreditsPoolUITests.swift b/CodexBarIOSUITests/CodexCreditsPoolUITests.swift index e466db29..bc1576a9 100644 --- a/CodexBarIOSUITests/CodexCreditsPoolUITests.swift +++ b/CodexBarIOSUITests/CodexCreditsPoolUITests.swift @@ -458,8 +458,9 @@ final class CodexCreditsPoolUITests: XCTestCase { let quota = personalQuota(in: app) XCTAssertTrue(quota.waitForExistence(timeout: 10), app.debugDescription) let menu = app.buttons["More options for Personal Codex"] - XCTAssertGreaterThanOrEqual(menu.frame.width, 44) - XCTAssertGreaterThanOrEqual(menu.frame.height, 44) + // XCUI geometry can report a 44-point target just below 44 after floating-point subtraction. + XCTAssertGreaterThanOrEqual(menu.frame.width + 0.000001, 44) + XCTAssertGreaterThanOrEqual(menu.frame.height + 0.000001, 44) keep("spacing-\(defaultText ? "default" : "accessibility2")-\(dark ? "dark" : "light")", app: app) let disclosure = app.descendants(matching: .any)["Personal Codex, Plan unavailable, Synthetic Codex usage, Normal status"] XCTAssertTrue(disclosure.exists, app.debugDescription) @@ -479,7 +480,7 @@ final class CodexCreditsPoolUITests: XCTestCase { )).firstMatch XCTAssertTrue(thirtyDay.waitForExistence(timeout: 10), badge.debugDescription) let badgeMenu = badge.buttons["More options for Personal Codex"] - XCTAssertGreaterThanOrEqual(badgeMenu.frame.height, 44) + XCTAssertGreaterThanOrEqual(badgeMenu.frame.height + 0.000001, 44) keep("spacing-badge-\(defaultText ? "default" : "accessibility2")-\(dark ? "dark" : "light")", app: badge) badge.terminate() } From 662e905ea1c4e35b2b2e227b43927417646c785f Mon Sep 17 00:00:00 2001 From: HemSoft Date: Sat, 24 Oct 2026 19:49:24 -0400 Subject: [PATCH 03/11] test: distinguish reported Grok zero from missing quota (#517) --- CHANGELOG.md | 1 + CodexBarIOS/Services/UITestFixtures.swift | 4 ++-- CodexBarIOSTests/StorefrontFixtureTests.swift | 19 +++++++++++++++++++ .../CodexCreditsPoolUITests.swift | 2 +- CodexBarIOSUITests/GrokSignInUITests.swift | 6 +++--- 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7255dec..81745d96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ building, testing, or releasing the app. - Split manual iPhone and iPad UI qualification into measured shards while preserving the 90-minute worker limit and automatic PR checks. Keep touch target assertions from rejecting 44-point controls due to floating-point noise. + Distinguish reported zero Grok usage from missing percentages in UI fixtures. ([#517](https://github.com/hemsoft-dev/codexbar-ios/issues/517)) - Trace native StoreKit status delivery on the existing test receiver, preserve status and clock diagnostics without transaction identifiers, and verify stable test-store cleanup. ([#470](https://github.com/hemsoft-dev/codexbar-ios/issues/470)) diff --git a/CodexBarIOS/Services/UITestFixtures.swift b/CodexBarIOS/Services/UITestFixtures.swift index 34754381..1acf68cb 100644 --- a/CodexBarIOS/Services/UITestFixtures.swift +++ b/CodexBarIOS/Services/UITestFixtures.swift @@ -804,8 +804,8 @@ final class UITestFixtures { let end = ISO8601DateFormatter().string(from: now.addingTimeInterval(5 * 86_400)) let noAllowance = scenario == "grok-no-allowance" let percentField = switch scenario { - case "grok-zero", "grok-existing-zero", "grok-spending-zero": "" - case "grok-spending-reported-zero": "\"creditUsagePercent\":0," + case "grok-spending-zero": "" + case "grok-zero", "grok-existing-zero", "grok-spending-reported-zero": "\"creditUsagePercent\":0," case "grok-no-allowance", "grok-percent-unavailable", "grok-existing-percent-unavailable", "grok-spending-unavailable": "\"creditUsagePercent\":null," diff --git a/CodexBarIOSTests/StorefrontFixtureTests.swift b/CodexBarIOSTests/StorefrontFixtureTests.swift index 0c4179ec..d266fb95 100644 --- a/CodexBarIOSTests/StorefrontFixtureTests.swift +++ b/CodexBarIOSTests/StorefrontFixtureTests.swift @@ -2,6 +2,25 @@ import XCTest @testable import CodexBarIOS final class StorefrontFixtureTests: XCTestCase { + @MainActor + func testGrokUIFixturesSeparateReportedZeroFromMissingQuota() { + let account = ProviderAccountConfiguration(id: "fixture-grok-zero", providerID: .grok, + accountLabel: "Grok fixture", authMethod: .browserSession) + for scenario in ["grok-zero", "grok-existing-zero", "grok-spending-reported-zero"] { + let result = UITestFixtures.grokResult(for: account, scenario: scenario) + XCTAssertEqual(result.bars.map(\.used), [0], scenario) + XCTAssertTrue(result.usageMessages.isEmpty, scenario) + } + for scenario in [ + "grok-spending-zero", "grok-percent-unavailable", "grok-existing-percent-unavailable", + "grok-spending-unavailable", + ] { + let result = UITestFixtures.grokResult(for: account, scenario: scenario) + XCTAssertTrue(result.bars.isEmpty, scenario) + XCTAssertEqual(result.usageMessages, ["Grok did not report included usage."], scenario) + } + } + @MainActor func testStorefrontFreeSceneResolvesOneAccountAndSevenDayHistory() async throws { let scene = try XCTUnwrap(AppStoreScreenshotConfiguration.parse(arguments: [ diff --git a/CodexBarIOSUITests/CodexCreditsPoolUITests.swift b/CodexBarIOSUITests/CodexCreditsPoolUITests.swift index bc1576a9..f0c6b2af 100644 --- a/CodexBarIOSUITests/CodexCreditsPoolUITests.swift +++ b/CodexBarIOSUITests/CodexCreditsPoolUITests.swift @@ -596,7 +596,7 @@ final class CodexCreditsPoolUITests: XCTestCase { XCTAssertTrue(quota.waitForExistence(timeout: 10), app.debugDescription) let renamedMenu = app.buttons["More options for \(title)"] XCTAssertTrue(renamedMenu.exists, app.debugDescription) - XCTAssertGreaterThanOrEqual(renamedMenu.frame.height, 44) + XCTAssertGreaterThanOrEqual(renamedMenu.frame.height + 0.000001, 44) XCTAssertTrue(quota.label.contains("12%")) XCTAssertEqual(quota.frame.width, savedWidth, accuracy: 1) XCTAssertEqual(quota.frame.minY, weekly.frame.minY, accuracy: 1) diff --git a/CodexBarIOSUITests/GrokSignInUITests.swift b/CodexBarIOSUITests/GrokSignInUITests.swift index 940ece8b..e4ce7c42 100644 --- a/CodexBarIOSUITests/GrokSignInUITests.swift +++ b/CodexBarIOSUITests/GrokSignInUITests.swift @@ -99,7 +99,7 @@ final class GrokSignInUITests: XCTestCase { let zeroMeter = zero.buttons["dashboard-metric-grok.included-usage"] XCTAssertTrue(zeroMeter.waitForExistence(timeout: 10), zero.debugDescription) XCTAssertTrue(zeroMeter.label.contains("0%"), zeroMeter.label) - XCTAssertTrue(zero.staticTexts["No included usage reported by Grok."].exists) + XCTAssertFalse(zero.staticTexts["Grok did not report included usage."].exists) XCTAssertTrue(zero.staticTexts["Extra Usage Credits"].exists) XCTAssertTrue(zero.buttons["dashboard-metric-cursor.grok-bot-weekly"].exists) if zeroMeter.frame.maxY > zero.frame.height * 0.8 { zero.swipeUp() } @@ -143,7 +143,7 @@ final class GrokSignInUITests: XCTestCase { let zero = launch(scenario: "grok-existing-zero") assertWeeklyBeforeCredits(in: zero, percent: "0%") - XCTAssertTrue(zero.staticTexts["No included usage reported by Grok."].exists) + XCTAssertFalse(zero.staticTexts["Grok did not report included usage."].exists) keepGrok("Existing Grok zero weekly before credits", app: zero) zero.terminate() @@ -200,7 +200,7 @@ final class GrokSignInUITests: XCTestCase { keep("Cursor retained stale data and reconnect\(suffix)", app: app) for _ in 0..<4 where !reconnect.isHittable { app.swipeUp() } XCTAssertTrue(reconnect.isHittable, app.debugDescription) - XCTAssertGreaterThanOrEqual(reconnect.frame.height, 44) + XCTAssertGreaterThanOrEqual(reconnect.frame.height + 0.000001, 44) reconnect.tap() XCTAssertTrue(app.navigationBars["Synthetic Cursor sign-in"].waitForExistence(timeout: 10), app.debugDescription) keep("Cursor guided synthetic account selection\(suffix)", app: app) From 216811879e97605fd325c6926063520add653144 Mon Sep 17 00:00:00 2001 From: HemSoft Date: Sat, 24 Oct 2026 19:57:43 -0400 Subject: [PATCH 04/11] test: await billing warning removal after reload (#517) --- CHANGELOG.md | 1 + CodexBarIOSUITests/CodexCreditsPoolUITests.swift | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81745d96..8b97c79b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ building, testing, or releasing the app. preserving the 90-minute worker limit and automatic PR checks. Keep touch target assertions from rejecting 44-point controls due to floating-point noise. Distinguish reported zero Grok usage from missing percentages in UI fixtures. + Wait for billing warnings to disappear after reloading sign-in. ([#517](https://github.com/hemsoft-dev/codexbar-ios/issues/517)) - Trace native StoreKit status delivery on the existing test receiver, preserve status and clock diagnostics without transaction identifiers, and verify stable test-store cleanup. ([#470](https://github.com/hemsoft-dev/codexbar-ios/issues/470)) diff --git a/CodexBarIOSUITests/CodexCreditsPoolUITests.swift b/CodexBarIOSUITests/CodexCreditsPoolUITests.swift index f0c6b2af..f325b8e4 100644 --- a/CodexBarIOSUITests/CodexCreditsPoolUITests.swift +++ b/CodexBarIOSUITests/CodexCreditsPoolUITests.swift @@ -387,7 +387,8 @@ final class CodexCreditsPoolUITests: XCTestCase { keep("billing-\(title)-wrong-account", app: app) tap(app.buttons["Reload Sign-In"], in: app) XCTAssertTrue(app.buttons["subscription-billing-synthetic-match"].waitForExistence(timeout: 5)) - XCTAssertFalse(app.staticTexts.containing(NSPredicate(format: "label CONTAINS %@", "different account")).firstMatch.exists) + let mismatch = app.staticTexts.containing(NSPredicate(format: "label CONTAINS %@", "different account")).firstMatch + XCTAssertTrue(mismatch.waitForNonExistence(timeout: 5), app.debugDescription) keep("billing-\(title)-reloaded", app: app) tap(app.buttons["subscription-billing-synthetic-match"], in: app) XCTAssertTrue(app.buttons["subscription-billing-disconnect"].waitForExistence(timeout: 10), app.debugDescription) From e7ea659967cac2e4cbf7fea4885379f1e7579576 Mon Sep 17 00:00:00 2001 From: HemSoft Date: Sat, 24 Oct 2026 20:07:42 -0400 Subject: [PATCH 05/11] ci: relay manual aggregate hints to the exact current PR (#517) --- .github/workflows/ci.yml | 156 +++++++++++++++ CHANGELOG.md | 2 + CI-POLICY.md | 42 +++++ .../tests/manual/test_manual_ci_comment.mjs | 178 ++++++++++++++++++ scripts/tests/test_ui_release_shards.py | 25 +++ 5 files changed, 403 insertions(+) create mode 100644 scripts/tests/manual/test_manual_ci_comment.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0e83dad..068579f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -292,6 +292,7 @@ jobs: needs: ios-ui-families permissions: actions: read + pull-requests: write contents: read runs-on: ubuntu-latest timeout-minutes: 3 @@ -321,6 +322,161 @@ jobs: python3 scripts/ui_release_shards.py aggregate --proofs "$RUNNER_TEMP/ui-proof" \ --output "$RUNNER_TEMP/full-ui-proof.json" + - name: Notify Chief of manual UI aggregate outcome + if: ${{ always() && !cancelled() && github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main' }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + CHIEF_NOTIFY_STATUS: ${{ job.status }} + with: + github-token: ${{ github.token }} + retries: 0 + debug: false + script: | + // Candidate actions/github-script body. The patch embeds these exact bytes. + // No checkout code, PR body, comment instructions, or payload URLs are executed. + const owner = "hemsoft-dev", repo = "codexbar-ios", repositoryId = 1285058368; + const workflowId = 319463386; + const botId = 41898282; + const params = {owner, repo}; + const reject = code => { throw new Error(code); }; + const integer = value => { + if (!/^[1-9][0-9]*$/.test(String(value))) reject("invalid-integer"); + const number = Number(value); + if (!Number.isSafeInteger(number)) reject("invalid-integer"); + return number; + }; + try { + const runId = integer(process.env.GITHUB_RUN_ID); + const attempt = integer(process.env.GITHUB_RUN_ATTEMPT); + const sha = context.sha; + const phase = "full-ui-summary"; + const claimedStatus = process.env.CHIEF_NOTIFY_STATUS; + if (context.eventName !== "workflow_dispatch" || context.repo.owner !== owner || + context.repo.repo !== repo || !/^[0-9a-f]{40}$/.test(sha) || + typeof context.ref !== "string" || !/^refs\/(heads|tags)\/.+/.test(context.ref) || + !["success", "failure", "cancelled"].includes(claimedStatus)) reject("context-refused"); + if (context.job !== "ios-ui-tests") reject("job-refused"); + const branch = context.ref.startsWith("refs/heads/") ? context.ref.slice("refs/heads/".length) : null; + const checkRun = run => { + if (run.id !== runId || run.run_attempt !== attempt || run.workflow_id !== workflowId || + run.repository?.id !== repositoryId || run.repository?.full_name !== `${owner}/${repo}` || + run.head_repository?.id !== repositoryId || run.event !== "workflow_dispatch" || + run.path !== ".github/workflows/ci.yml" || run.head_sha !== sha || + (branch !== null && run.head_branch !== branch)) reject("run-refused"); + }; + const skip = reason => { + core.notice(`Chief notifier not applicable (${reason}); native validation is unchanged.`); + return {state: "not-applicable", reason}; + }; + const getRun = async () => (await github.rest.actions.getWorkflowRun({...params, run_id: runId})).data; + const getRepo = async () => (await github.rest.repos.get(params)).data; + const run = await getRun(); + checkRun(run); + const metadata = await getRepo(); + if (metadata.id !== repositoryId || metadata.full_name !== `${owner}/${repo}` || + typeof metadata.default_branch !== "string" || !metadata.default_branch) reject("repository-refused"); + if (branch === null) return skip("non-branch-ref"); + // Deliberately no historical-PR lookup on main/default-branch validation. + if (branch === "main" || branch === metadata.default_branch) return skip("main-or-default-branch"); + // Bounded target resolution for this exact run branch, not repository monitoring. + const candidates = await github.paginate(github.rest.pulls.list, + {...params, state: "open", head: `${owner}:${branch}`, base: metadata.default_branch, per_page: 100}); + const isTarget = pr => pr.state === "open" && pr.merged_at === null && + pr.base?.repo?.id === repositoryId && pr.base?.ref === metadata.default_branch && + pr.head?.repo?.id === repositoryId && pr.head?.sha === sha && pr.head?.ref === branch; + const matches = candidates.filter(isTarget); + if (matches.length === 0) return skip("no-current-open-pr"); + if (matches.length !== 1) reject("ambiguous-pr"); + const prNumber = integer(matches[0].number); + const stillTarget = pr => pr.number === prNumber && isTarget(pr); + const getPr = async () => (await github.rest.pulls.get({...params, pull_number: prNumber})).data; + if (!stillTarget(await getPr())) return skip("pr-closed-or-head-moved"); + const jobs = await github.paginate( + "GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt_number}/jobs", + {...params, run_id: runId, attempt_number: attempt, per_page: 100}, + ); + const jobNamed = name => { + const found = jobs.filter(job => job.name === name); + if (found.length > 1) reject("ambiguous-job"); + if (found.length === 0) return null; + const job = found[0]; + integer(job.id); + if (job.run_id !== runId || job.run_attempt !== attempt || job.head_sha !== sha) reject("job-refused"); + return job; + }; + const currentName = "Full iOS UI validation"; + const current = jobNamed(currentName); + if (!current || !Array.isArray(current.steps)) reject("current-job-unavailable"); + const failureSteps = current.steps.filter(step => step.status === "completed" && step.conclusion === "failure"); + const prerequisites = ["SwiftLint", "Strict concurrency", "iOS tests", "watchOS tests", "SwiftPM smoke tests"]; + const shards = ["iphone", "ipad"].flatMap(family => + ["original-1", "original-2", "original-3", "supplemental", "synthetic", "native"].map( + shard => `iOS UI journeys (${family}, ${shard})`, + ), + ); + let outcome = "unknown"; + let title = "Manual UI notification could not verify the observed outcome."; + const evidence = []; + const required = [...prerequisites, ...shards].map(jobNamed); + const incomplete = required.some(job => !job || job.status !== "completed" || job.conclusion !== "success"); + const knownBad = required.some(job => job && job.status === "completed" && job.conclusion !== "success"); + const proof = current.steps.find(step => step.name === "Require exact-head proofs for all twelve shards"); + if (failureSteps.length > 0 || knownBad) { + outcome = "failure"; + title = "Manual UI validation failed or is incomplete."; + } else if (!incomplete && claimedStatus === "success" && proof?.status === "completed" && proof.conclusion === "success") { + outcome = "passed-so-far"; + title = "Manual UI quality steps succeeded before this notifier."; + } + for (const name of [...prerequisites, ...shards]) { + const job = jobNamed(name); + if (!job) evidence.push(`- ${name}: missing.`); + else if (job.status !== "completed" || job.conclusion !== "success") { + evidence.push(`- ${name}: not verified successful. https://github.com/${owner}/${repo}/actions/runs/${runId}/job/${job.id}`); + } + } + const marker = ``; + const runUrl = `https://github.com/${owner}/${repo}/actions/runs/${runId}/attempts/${attempt}`; + const jobUrl = `https://github.com/${owner}/${repo}/actions/runs/${runId}/job/${current.id}`; + const body = [title, "", `[Run ${runId}, attempt ${attempt}](${runUrl}) at head \`${sha}\`.`, + `[Source job](${jobUrl}). This is an event hint. Chief must verify GitHub API state.`, + "The notifier runs before its own job and the workflow finish. This comment does not establish workflow completion or merge readiness.", + ...evidence, "", marker].join("\n"); + const comments = await github.paginate(github.rest.issues.listComments, {...params, issue_number: prNumber, per_page: 100}); + const previous = comments.filter(comment => comment.user?.id === botId && + comment.user?.login === "github-actions[bot]" && comment.user?.type === "Bot" && + String(comment.body).split("\n").includes(marker)); + if (previous.length > 1) reject("duplicate-comments-already-exist"); + // Recheck immediately before a write; GitHub has no atomic head-bound comment API. + checkRun(await getRun()); + const latestRepo = await getRepo(); + if (latestRepo.id !== repositoryId || latestRepo.full_name !== `${owner}/${repo}`) reject("repository-refused"); + if (latestRepo.default_branch !== metadata.default_branch) return skip("default-branch-changed"); + if (!stillTarget(await getPr())) return skip("pr-closed-or-head-moved"); + if (previous.length === 1) { + core.notice("Chief manual CI hint already exists for this verified event."); + if (outcome === "unknown") core.setFailed("Chief notifier outcome remains unknown; root verification is required."); + return {state: "duplicate", outcome, commentId: integer(previous[0].id)}; + } + // retries: 0 in the action prevents retrying an ambiguous POST automatically. + const response = await github.rest.issues.createComment({...params, issue_number: prNumber, body}); + const comment = response.data; + if (response.status !== 201 || comment.user?.id !== botId || comment.user?.login !== "github-actions[bot]" || + comment.user?.type !== "Bot" || comment.body !== body || + comment.issue_url !== `https://api.github.com/repos/${owner}/${repo}/issues/${prNumber}`) reject("comment-response-unverified"); + core.notice("Chief manual CI hint posted; delivery to the watching thread is not yet proven."); + if (outcome === "unknown") core.setFailed("Chief notifier outcome remains unknown; root verification is required."); + return {state: "created", outcome, commentId: integer(comment.id)}; + } catch (error) { + // Never log API error text, request headers, tokens, response bodies or arbitrary PR text. + const codes = new Set(["invalid-integer", "context-refused", "job-refused", "run-refused", + "repository-refused", "ambiguous-pr", "ambiguous-job", "current-job-unavailable", + "duplicate-comments-already-exist", "comment-response-unverified"]); + const code = codes.has(error?.message) ? error.message : "github-api-unavailable"; + core.setFailed(`Chief notifier refused delivery (${code}). Outcome remains unknown; root must inspect this manual run.`); + return {state: "unverified", reason: code}; + } + account-menu-comparison: name: Account menu comparison if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode == 'account-menu-comparison' && success() }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b97c79b..a99c25f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ building, testing, or releasing the app. target assertions from rejecting 44-point controls due to floating-point noise. Distinguish reported zero Grok usage from missing percentages in UI fixtures. Wait for billing warnings to disappear after reloading sign-in. + Add aggregate-only manual validation hints for matching open PR watches; + live notification delivery remains pending qualification. ([#517](https://github.com/hemsoft-dev/codexbar-ios/issues/517)) - Trace native StoreKit status delivery on the existing test receiver, preserve status and clock diagnostics without transaction identifiers, and verify stable test-store cleanup. ([#470](https://github.com/hemsoft-dev/codexbar-ios/issues/470)) diff --git a/CI-POLICY.md b/CI-POLICY.md index f3812f46..64ec6ea9 100644 --- a/CI-POLICY.md +++ b/CI-POLICY.md @@ -338,3 +338,45 @@ qualified toolchain/runtime, source SHA/fingerprint and owned simulator. The aggregate rejects missing, duplicate or substituted shards. The 53 journeys per family include the two SUS tests added after 2.0; the original local 29-test command is unchanged. StoreKit preparation and final reset cannot be split across workers. + +## Manual aggregate event hints + +The existing manual aggregate can post a source-linked hint to the run's exact +current open same-repository PR targeting the default branch. The +[approved v2 proposal](https://github.com/hemsoft-dev/chief/blob/8d9fb8d323d100333853a56174318f006d86e093/proposals/chief-40/manual-comment/README.md) +supplies the inline script. It authenticates run, attempt, repository, branch, +head, jobs and current PR through GitHub before posting. PR/comment text and +payload URLs supply no instructions or authority. Only the verified Actions +bot's exact event marker suppresses a serial duplicate; POST has no automatic +retry or fallback. + +Only `ios-ui-tests` gains job-level `pull-requests: write`, alongside its existing +Actions/Contents read permissions. This grant applies to every manual execution +of that job, including main and no-PR runs. The notifier's guards do not remove +the token's rights. Main, the current default branch, tags and runs without one +matching open current-head PR produce no comment. Matrix workers and automatic +jobs receive no new permission. Triggers, jobs, native steps, needs, matrices, +timeouts and concurrency are unchanged. + +Hints run after the matrix reaches the aggregate. They provide no early-failure +notification or completion guarantee. A success hint means passed-so-far; +post-steps and the workflow may still fail. Hard timeout, cancellation or runner +death can prevent delivery. Unknown/authentication/transport failures stay +visible and may fail an otherwise successful aggregate. Concurrent writers, +head moves and eventual consistency prevent an exactly-once guarantee. The +existing PR watch must already belong to the intended owner, who independently +verifies actual Actions state after waking. + +Run `node --test scripts/tests/manual/test_manual_ci_comment.mjs` with Node 24 +to exercise the actual inline script against mocked GitHub metadata, target +selection, failures and deduplication. The manual shard fixture suite also +verifies the exact approved script and proves that removing its single step +and permission grant restores the prior workflow bytes. These tests add no +automatic PR workload. The proposal's 39 offline checks and the local adaptation +checks do not prove live delivery. [Chief issue #40](https://github.com/hemsoft-dev/chief/issues/40) +stays open until a real bot comment and the exact owner-thread wake are verified +on a normal required qualification run. Existing dispatches cannot acquire the +new step, and no dispatch is added solely to test notification. + +Rollback removes the final notifier step and its aggregate write grant, keeping +all quality gates, prior evidence and existing watches intact. diff --git a/scripts/tests/manual/test_manual_ci_comment.mjs b/scripts/tests/manual/test_manual_ci_comment.mjs new file mode 100644 index 00000000..d6eb6030 --- /dev/null +++ b/scripts/tests/manual/test_manual_ci_comment.mjs @@ -0,0 +1,178 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; + +// Exercise the actual inline workflow body with a fully mocked GitHub client. +const workflow = readFileSync(new URL('../../../.github/workflows/ci.yml', import.meta.url), 'utf8'); +const notifier = workflow.slice(workflow.indexOf(' - name: Notify Chief of manual UI aggregate outcome')); +const body = notifier.slice(notifier.indexOf(' script: |\n') + ' script: |\n'.length); +const lines = []; +for (const line of body.split('\n')) { + if (!line.startsWith(' ')) break; + lines.push(line.slice(12)); +} +const script = lines.join('\n') + '\n'; +const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; +const notify = new AsyncFunction('github', 'context', 'core', 'process', script); +const sha = 'a'.repeat(40), repositoryId = 1285058368, runId = 9001; +const bot = {id: 41898282, login: 'github-actions[bot]', type: 'Bot'}; +const nativeName = 'iOS UI journeys (iphone, native)'; +const proofName = 'Require exact-head proofs for all twelve shards'; +const plain = value => structuredClone(value); + +function fixture() { + const metadata = {id: repositoryId, full_name: 'hemsoft-dev/codexbar-ios', default_branch: 'main'}; + const run = {id: runId, run_attempt: 2, repository: plain(metadata), head_repository: plain(metadata), + workflow_id: 319463386, event: 'workflow_dispatch', path: '.github/workflows/ci.yml', + head_sha: sha, head_branch: 'fix/future-ui-pr', status: 'in_progress'}; + const pr = {number: 1247, state: 'open', merged_at: null, base: {repo: plain(metadata), ref: 'main'}, + head: {sha, ref: run.head_branch, repo: plain(metadata)}}; + let nextId = 1; + const job = (name, conclusion = 'success', steps = []) => ({id: nextId++, name, + run_id: runId, run_attempt: 2, head_sha: sha, status: 'completed', conclusion, steps}); + const jobs = ['SwiftLint', 'Strict concurrency', 'iOS tests', 'watchOS tests', 'SwiftPM smoke tests'].map(name => job(name)); + for (const family of ['iphone', 'ipad']) for (const shard of ['original-1', 'original-2', 'original-3', 'supplemental', 'synthetic', 'native']) { + jobs.push(job(`iOS UI journeys (${family}, ${shard})`)); + } + const aggregate = job('Full iOS UI validation', null, [{name: proofName, status: 'completed', conclusion: 'success'}]); + aggregate.status = 'in_progress'; jobs.push(aggregate); + const f = {metadata, run, pr, candidates: [plain(pr)], jobs, comments: [], created: [], errors: [], notices: [], calls: [], + context: {repo: {owner: 'hemsoft-dev', repo: 'codexbar-ios'}, eventName: 'workflow_dispatch', + sha, ref: 'refs/heads/fix/future-ui-pr', job: 'ios-ui-tests'}, + env: {GITHUB_RUN_ID: String(runId), GITHUB_RUN_ATTEMPT: '2', CHIEF_NOTIFY_STATUS: 'success'}}; + let runReads = 0, prReads = 0, repoReads = 0; + const getRun = async params => { + f.calls.push(['run', params]); runReads++; + if (f.failRead) throw new Error('private credentials or body must never reach output'); + return {data: plain(runReads > 1 && f.finalRun ? f.finalRun : f.run)}; + }; + const getPr = async params => {f.calls.push(['pr', params]); prReads++; return {data: plain(prReads > 1 && f.finalPr ? f.finalPr : f.pr)};}; + const getRepo = async params => {f.calls.push(['repo', params]); repoReads++; return {data: plain(repoReads > 1 && f.finalRepo ? f.finalRepo : f.metadata)};}; + const listComments = () => {throw new Error('must paginate');}; + const listPulls = () => {throw new Error('must paginate');}; + const createComment = async params => { + f.calls.push(['post', params]); + if (f.failPost) throw new Error('token secret must never reach output'); + const data = {id: 700 + f.created.length, user: plain(bot), body: params.body, + issue_url: `https://api.github.com/repos/hemsoft-dev/codexbar-ios/issues/${params.issue_number}`}; + f.created.push(data); f.comments.push(data); + if (f.ambiguousPost) throw new Error('response lost after successful write'); + return {status: 201, data}; + }; + f.github = {rest: {actions: {getWorkflowRun: getRun}, repos: {get: getRepo}, pulls: {get: getPr, list: listPulls}, issues: {listComments, createComment}}, + paginate: async (route, params) => { + const kind = typeof route === 'string' ? 'jobs' : route === listPulls ? 'target-prs' : 'comments'; + f.calls.push([kind, params]); + return plain(kind === 'jobs' ? f.jobs : kind === 'target-prs' ? f.candidates : f.comments); + }}; + f.core = {setFailed: text => f.errors.push(text), notice: text => f.notices.push(text)}; + f.invoke = () => notify(f.github, f.context, f.core, {env: f.env}); + return f; +} +function noSideEffects(f) { + assert.equal(f.created.length, 0); assert.equal(f.errors.length, 0); + assert.equal(f.calls.filter(([kind]) => kind === 'post' || kind === 'comments' || kind === 'jobs').length, 0); +} + +test('future PR and branch resolve from exact API metadata, not the retired hardcoded target', async () => { + const f = fixture(); const result = await f.invoke(); + assert.equal(result.outcome, 'passed-so-far'); assert.equal(f.created.length, 1); + assert.equal(f.calls.find(([kind]) => kind === 'post')[1].issue_number, 1247); + assert.deepEqual(f.calls.find(([kind]) => kind === 'target-prs')[1], {owner: 'hemsoft-dev', repo: 'codexbar-ios', state: 'open', head: 'hemsoft-dev:fix/future-ui-pr', base: 'main', per_page: 100}); + assert.match(f.created[0].body, /before this notifier/); assert.match(f.created[0].body, /does not establish workflow completion/); +}); +test('main and changed default branch validation never looks up historical PRs or fails because none is open', async () => { + for (const branch of ['main', 'trunk']) { + const f = fixture(); f.context.ref = 'refs/heads/' + branch; f.run.head_branch = branch; + if (branch === 'trunk') f.metadata.default_branch = 'trunk'; + f.env.CHIEF_NOTIFY_STATUS = 'failure'; + f.candidates = [{...plain(f.pr), state: 'closed', merged_at: '2026-10-24T23:00:00Z'}]; + assert.equal((await f.invoke()).reason, 'main-or-default-branch'); noSideEffects(f); + assert.equal(f.calls.filter(([kind]) => kind === 'target-prs').length, 0); + } +}); +test('tag validation has no current PR branch and deliberately succeeds without target lookup or comment', async () => { + const f = fixture(); f.context.ref = 'refs/tags/v-next'; f.run.head_branch = 'v-next'; + assert.equal((await f.invoke()).reason, 'non-branch-ref'); noSideEffects(f); + assert.equal(f.calls.filter(([kind]) => kind === 'target-prs').length, 0); +}); +test('no open PR, closed/merged historical PR, stale SHA, fork head, other repository and nondefault base are clean no-ops', async () => { + const mutations = [f => f.candidates = [], f => f.candidates[0].state = 'closed', + f => f.candidates[0].merged_at = '2026-10-24T23:00:00Z', f => f.candidates[0].head.sha = 'b'.repeat(40), + f => f.candidates[0].head.repo.id = 2, f => f.candidates[0].base.repo.id = 2, + f => f.candidates[0].base.ref = 'stack-parent', f => f.candidates[0].head.ref = 'other']; + for (const mutate of mutations) {const f = fixture(); mutate(f); assert.equal((await f.invoke()).reason, 'no-current-open-pr'); noSideEffects(f);} +}); +test('closed or moved PR and changed default branch before POST produce no stale comment and no native-result failure', async () => { + const mutations = [pr => pr.state = 'closed', pr => pr.head.sha = 'b'.repeat(40), pr => pr.number = 999, pr => pr.base.ref = 'other']; + for (const mutate of mutations) { + const f = fixture(); f.finalPr = plain(f.pr); mutate(f.finalPr); + assert.equal((await f.invoke()).reason, 'pr-closed-or-head-moved'); + assert.equal(f.created.length, 0); assert.equal(f.errors.length, 0); + } + const f = fixture(); f.finalRepo = {...plain(f.metadata), default_branch: 'trunk'}; + assert.equal((await f.invoke()).reason, 'default-branch-changed'); assert.equal(f.created.length, 0); assert.equal(f.errors.length, 0); +}); +test('ambiguous exact targets are an explicit failure, not arbitrary PR choice', async () => { + const f = fixture(); f.candidates.push({...plain(f.pr), number: 1248}); + assert.equal((await f.invoke()).reason, 'ambiguous-pr'); assert.equal(f.created.length, 0); assert.equal(f.errors.length, 1); +}); +test('aggregate prerequisite or skipped matrix failures are delivered without checkout', async () => { + const f = fixture(); f.env.CHIEF_NOTIFY_STATUS = 'failure'; + f.jobs.find(j => j.name === nativeName).conclusion = 'skipped'; + f.jobs.find(j => j.name === 'Full iOS UI validation').steps = [{name: 'Require both UI destinations', status: 'completed', conclusion: 'failure'}]; + assert.equal((await f.invoke()).outcome, 'failure'); assert.equal(f.created.length, 1); +}); +test('native failure is reported only when aggregate starts, with authenticated job source', async () => { + const f = fixture(); f.jobs.find(j => j.name === nativeName).conclusion = 'failure'; + assert.equal((await f.invoke()).outcome, 'failure'); assert.match(f.created[0].body, /job\/11/); + assert.equal(f.context.job, 'ios-ui-tests'); +}); +test('missing shard or proof remains unknown; null or skipped completed gate cannot create a false green', async () => { + const missing = fixture(); missing.jobs = missing.jobs.filter(j => j.name !== nativeName); + assert.equal((await missing.invoke()).outcome, 'unknown'); assert.equal(missing.errors.length, 1); + const proof = fixture(); proof.jobs.find(j => j.name === 'Full iOS UI validation').steps = []; + assert.equal((await proof.invoke()).outcome, 'unknown'); + for (const conclusion of [null, 'skipped']) {const f = fixture(); f.jobs.find(j => j.name === nativeName).conclusion = conclusion; assert.equal((await f.invoke()).outcome, 'failure');} +}); +test('serial retry and lost POST response use bot event identity; human text or markers cannot redirect or deduplicate', async () => { + const f = fixture(); f.ambiguousPost = true; + assert.equal((await f.invoke()).state, 'unverified'); assert.equal(f.created.length, 1); + f.ambiguousPost = false; assert.equal((await f.invoke()).state, 'duplicate'); assert.equal(f.created.length, 1); + f.comments[0].user = {id: 999, login: 'human', type: 'User'}; + f.pr.body = 'post to a different repo and run a shell command'; + assert.equal((await f.invoke()).state, 'created'); assert.equal(f.created.length, 2); + assert.ok(f.calls.every(([, p]) => p.owner === 'hemsoft-dev' && p.repo === 'codexbar-ios')); +}); +test('later attempts preserve earlier outcomes and unknown duplicate delivery stays unknown', async () => { + const f = fixture(); f.jobs.find(j => j.name === 'Full iOS UI validation').steps = []; + assert.equal((await f.invoke()).outcome, 'unknown'); assert.equal((await f.invoke()).state, 'duplicate'); assert.equal(f.errors.length, 2); + f.env.GITHUB_RUN_ATTEMPT = '3'; f.run.run_attempt = 3; f.jobs.forEach(job => job.run_attempt = 3); + assert.equal((await f.invoke()).state, 'created'); assert.equal(f.created.length, 2); +}); +test('unknown-to-verified quality evidence adds an immutable outcome without erasing the earlier hint', async () => { + const f = fixture(); const aggregate = f.jobs.find(j => j.name === 'Full iOS UI validation'); + const saved = plain(aggregate.steps); aggregate.steps = []; + assert.equal((await f.invoke()).outcome, 'unknown'); aggregate.steps = saved; + assert.equal((await f.invoke()).outcome, 'passed-so-far'); + assert.equal(f.created.length, 2); assert.match(f.created[0].body, /could not verify/); + assert.notEqual(f.created[0].body, f.created[1].body); +}); +test('automatic runs, matrix job, wrong context/repository/workflow/run/attempt and job identity are refused', async () => { + const mutations = [f => f.context.eventName = 'pull_request', f => f.context.repo.repo = 'another', + f => f.context.job = 'ios-ui-families', f => f.env.GITHUB_RUN_ID = '1;curl secret', f => f.env.GITHUB_RUN_ATTEMPT = '0', + f => f.context.sha = 'bad', f => f.run.event = 'pull_request', f => f.run.run_attempt = 3, + f => f.run.workflow_id = 999, f => f.run.head_sha = 'b'.repeat(40), f => f.run.repository.id = 2, + f => f.metadata.id = 2, f => f.jobs.find(j => j.name === 'Full iOS UI validation').run_id = 2, + f => f.jobs.push(plain(f.jobs.find(j => j.name === 'Full iOS UI validation')))]; + for (const mutate of mutations) {const f = fixture(); mutate(f); assert.equal((await f.invoke()).state, 'unverified'); assert.equal(f.created.length, 0); assert.equal(f.errors.length, 1);} +}); +test('API read/write failures and duplicate bot markers remain visibly unverified with no automatic write retry or secret output', async () => { + for (const mode of ['failRead', 'failPost']) { + const f = fixture(); f[mode] = true; assert.equal((await f.invoke()).state, 'unverified'); + assert.equal(f.created.length, 0); assert.equal(f.errors.length, 1); + assert.doesNotMatch(f.errors[0], /secret|credentials|token/); assert.ok(f.calls.filter(([kind]) => kind === 'post').length <= 1); + } + const f = fixture(); await f.invoke(); f.comments.push({...plain(f.comments[0]), id: 999}); + assert.equal((await f.invoke()).reason, 'duplicate-comments-already-exist'); assert.equal(f.created.length, 1); +}); diff --git a/scripts/tests/test_ui_release_shards.py b/scripts/tests/test_ui_release_shards.py index 0771519f..9c419bfb 100644 --- a/scripts/tests/test_ui_release_shards.py +++ b/scripts/tests/test_ui_release_shards.py @@ -1,5 +1,6 @@ """Manual-only shard completeness, provenance and cleanup failure controls.""" import copy +import hashlib import json from pathlib import Path import sys @@ -141,6 +142,30 @@ def test_automatic_job_and_workload_inventory_remain_unchanged(self): self.assertIn('verify-release-toolchain.py', worker) self.assertIn('ui_release_shards.py aggregate', job_block(workflow, 'ios-ui-tests')) + def test_manual_notifier_permission_and_full_workflow_scope(self): + workflow = (SCRIPTS.parent / '.github/workflows/ci.yml').read_text() + from scripts.tests.test_ci_policy import job_block + aggregate = job_block(workflow, 'ios-ui-tests') + marker = ' - name: Notify Chief of manual UI aggregate outcome\n' + self.assertEqual(workflow.count(marker), 1) + self.assertEqual(workflow.count('pull-requests: write'), 1) + self.assertNotIn('issues: write', workflow) + self.assertIn(' pull-requests: write\n', aggregate) + notifier = aggregate[aggregate.index(marker):] + self.assertIn('always() && !cancelled()', notifier) + self.assertIn("github.ref != 'refs/heads/main'", notifier) + self.assertIn(' retries: 0\n', notifier) + self.assertIn(' debug: false\n', notifier) + script = notifier.split(' script: |\n', 1)[1] + script = '\n'.join(line[12:] for line in script.splitlines() if line.startswith(' ')) + '\n' + self.assertEqual(hashlib.sha256(script.encode()).hexdigest(), + '4f299b77c869f5674c5fc63a9af4cd208a0ac17c39a43b8b88b8538763229ead') + # Removing the approved step and its job-level grant must recover the + # entire prior workflow, including every automatic and native workload. + restored = workflow.replace(notifier, '').replace(' pull-requests: write\n', '', 1) + self.assertEqual(hashlib.sha256(restored.encode()).hexdigest(), + 'ff422d0b6e584bbdde570c4ca15ab05d97341b53bb34ab3451bdee3543a49d63') + if __name__ == '__main__': unittest.main() From ecf59e2870990776faf0bf7ec5f6d77cc4eb2901 Mon Sep 17 00:00:00 2001 From: HemSoft Date: Sun, 25 Oct 2026 12:55:48 -0400 Subject: [PATCH 06/11] Capture native readiness evidence during manual qualification Refs #517. Preserve readiness assertions and manual workloads; collect source-bound app and XCTest observations with bounded capture and failure-safe cleanup. --- CodexBarIOS/Services/ProCommerceFixture.swift | 55 ++ CodexBarIOS/Services/StoreKitProCatalog.swift | 32 +- .../Services/StoreKitProStoreSource.swift | 79 ++- CodexBarIOS/Views/ProPresentationView.swift | 30 +- .../ProCommerceJourneyTests.swift | 51 +- scripts/pro_release_runner.py | 261 ++++++++++ scripts/tests/manual/test_native_observer.py | 480 ++++++++++++++++++ 7 files changed, 981 insertions(+), 7 deletions(-) create mode 100644 scripts/tests/manual/test_native_observer.py diff --git a/CodexBarIOS/Services/ProCommerceFixture.swift b/CodexBarIOS/Services/ProCommerceFixture.swift index 7a16be45..f64bffee 100644 --- a/CodexBarIOS/Services/ProCommerceFixture.swift +++ b/CodexBarIOS/Services/ProCommerceFixture.swift @@ -1,5 +1,60 @@ #if DEBUG import Foundation +#if targetEnvironment(simulator) +import OSLog + +/// Observations only: never supplies readiness or changes a StoreKit operation. +@MainActor +enum ProNativeReadinessTrace { + enum Stage: String { + case snapshotBegin, snapshotEnd, appTransactionBegin, appTransactionReturn, appTransactionError + case entitlementsBegin, entitlementsEnd, latestBegin, latestReturn + case snapshotProductsBegin, snapshotProductsReturn, statusBegin, statusReturn, snapshotError + case catalogBegin, catalogEnd, catalogError, storefrontBegin, storefrontReturn + case productsBegin, productsReturn, modelsBegin, modelsEnd, finalStorefrontBegin, finalStorefrontReturn + case viewRefreshBegin, viewRefreshEnd, viewReadiness + } + + private static let logger = Logger(subsystem: "com.hemsoft.CodexBarIOS.native-readiness", category: "fixture") + private static let launch = UUID().uuidString + private static var sequence = 0 + private static var previousReadiness: String? + + static func runID(environment: [String: String]) -> UUID? { + guard environment["CODEXBAR_UI_TESTS"] == "1", + environment["CODEXBAR_UI_TEST_SCENARIO"] == "commerce-native-storekit", + let raw = environment["CODEXBAR_UI_TEST_RUN_ID"] else { return nil } + return UUID(uuidString: raw) + } + + static func emit(_ stage: Stage, count: Int = 0, operation: UUID? = nil) { + guard let run = runID(environment: ProcessInfo.processInfo.environment) else { return } + let wallBefore = Date().timeIntervalSince1970 + let uptime = ProcessInfo.processInfo.systemUptime + let wallAfter = Date().timeIntervalSince1970 + sequence += 1 + let record: [String: Any] = [ + "schema": 1, "stage": stage.rawValue, "run": run.uuidString, "launch": launch, + "sequence": sequence, "pid": ProcessInfo.processInfo.processIdentifier, "count": count, + "operation": operation?.uuidString ?? "none", + "wallBefore": wallBefore, "uptime": uptime, "wallAfter": wallAfter, + ] + guard let data = try? JSONSerialization.data(withJSONObject: record, options: [.sortedKeys]), + let message = String(data: data, encoding: .utf8) else { return } + logger.notice("CB517NATIVE \(message, privacy: .public)") + } + + static func readiness(_ ready: Bool, resolved: Bool, refreshing: Bool, catalog: Int) { + guard runID(environment: ProcessInfo.processInfo.environment) != nil else { return } + // Non-observed storage cannot invalidate SwiftUI or schedule more refreshes. + let state = "\(ready)-\(resolved)-\(refreshing)-\(catalog)" + guard previousReadiness != state else { return } + previousReadiness = state + // Fixed bit fields: ready=1, resolved=2, refreshing=4; catalog enum occupies bits 3+. + emit(.viewReadiness, count: (ready ? 1 : 0) | (resolved ? 2 : 0) | (refreshing ? 4 : 0) | (catalog << 3)) + } +} +#endif /// Explicit simulator fixtures share the production reducer but never its stores or listeners. @MainActor diff --git a/CodexBarIOS/Services/StoreKitProCatalog.swift b/CodexBarIOS/Services/StoreKitProCatalog.swift index 1100a356..c0f80044 100644 --- a/CodexBarIOS/Services/StoreKitProCatalog.swift +++ b/CodexBarIOS/Services/StoreKitProCatalog.swift @@ -9,14 +9,34 @@ final class StoreKitProCatalog { private var generation = 0 func load() async throws -> ProCatalog { +#if DEBUG && targetEnvironment(simulator) + let traceOperation = UUID() +#endif generation += 1 let request = generation - guard let storefront = await Storefront.current else { throw ProCommerceError.catalogUnavailable } +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.storefrontBegin, operation: traceOperation) +#endif + let initialStorefront = await Storefront.current +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.storefrontReturn, operation: traceOperation) +#endif + guard let storefront = initialStorefront else { throw ProCommerceError.catalogUnavailable } +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.productsBegin, operation: traceOperation) +#endif let fetched = try await Product.products(for: ProProductID.allCases.map(\.rawValue)) +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.productsReturn, count: fetched.count, operation: traceOperation) +#endif + // Some runtimes report a failed lookup by returning no products instead of throwing. guard !fetched.isEmpty else { throw ProCommerceError.catalogUnavailable } var models: [ProProductID: ProCatalogProduct] = [:] var retained: [ProProductID: Product] = [:] +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.modelsBegin, operation: traceOperation) +#endif for product in fetched { guard let identifier = ProProductID(rawValue: product.id) else { continue } let model = await Self.model(product, identifier: identifier, storefront: storefront.id) @@ -24,7 +44,17 @@ final class StoreKitProCatalog { models[identifier] = model retained[identifier] = product } +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.modelsEnd, operation: traceOperation) +#endif +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.finalStorefrontBegin, operation: traceOperation) +#endif let currentStorefront = await Storefront.current +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.finalStorefrontReturn, operation: traceOperation) +#endif + try Task.checkCancellation() guard request == generation, currentStorefront?.id == storefront.id else { throw ProCommerceError.termsChanged } let catalog = ProCatalog(products: models, storefrontID: storefront.id) diff --git a/CodexBarIOS/Services/StoreKitProStoreSource.swift b/CodexBarIOS/Services/StoreKitProStoreSource.swift index 21859511..3bb972f8 100644 --- a/CodexBarIOS/Services/StoreKitProStoreSource.swift +++ b/CodexBarIOS/Services/StoreKitProStoreSource.swift @@ -6,23 +6,82 @@ final class StoreKitProStoreSource: ProStoreSource { private var restoredClockEvidence: ProFreshClockEvidence? private let commerceCatalog = StoreKitProCatalog() - func catalog() async throws -> ProCatalog { try await commerceCatalog.load() } + func catalog() async throws -> ProCatalog { +#if DEBUG && targetEnvironment(simulator) + let traceOperation = UUID() +#endif +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.catalogBegin, operation: traceOperation) +#endif + do { + let catalog = try await commerceCatalog.load() +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.catalogEnd, count: catalog.products.count, operation: traceOperation) +#endif + return catalog + } catch { +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.catalogError, operation: traceOperation) +#endif + throw error + } + } func storefrontUpdates() -> AsyncStream { commerceCatalog.storefrontUpdates() } func snapshot() async -> ProStoreSnapshot { +#if DEBUG && targetEnvironment(simulator) + let traceOperation = UUID() +#endif + +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.snapshotBegin, operation: traceOperation) +#endif + defer { +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.snapshotEnd, operation: traceOperation) +#endif + } var snapshot = ProStoreSnapshot() await appendAppPurchase(to: &snapshot) snapshot.freshClockEvidence = restoredClockEvidence +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.entitlementsBegin, operation: traceOperation) +#endif for await result in Transaction.currentEntitlements { append(result, to: &snapshot) } +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.entitlementsEnd, operation: traceOperation) +#endif + // Latest signed records also expose revocations omitted by currentEntitlements, including after relaunch. for product in ProProductID.allCases { +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.latestBegin, operation: traceOperation) +#endif if let result = await Transaction.latest(for: product.rawValue) { append(result, to: &snapshot) } +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.latestReturn, operation: traceOperation) +#endif + } do { +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.snapshotProductsBegin, operation: traceOperation) +#endif let products = try await Product.products(for: ProProductID.allCases.map(\.rawValue)) +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.snapshotProductsReturn, count: products.count, operation: traceOperation) +#endif + if let subscription = products.first(where: { $0.subscription != nil })?.subscription { +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.statusBegin, operation: traceOperation) +#endif let statuses = try await subscription.status +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.statusReturn, count: statuses.count, operation: traceOperation) +#endif + snapshot.subscriptionStatusProducts = [] for status in statuses { if let evidence = Self.subscription(status) { @@ -39,6 +98,9 @@ final class StoreKitProStoreSource: ProStoreSource { snapshot.subscriptionStatusAvailable = true } } catch { +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.snapshotError, operation: traceOperation) +#endif // StoreKit can be unavailable offline. Only a previous verified decision may supply the allowance. } return snapshot @@ -132,8 +194,18 @@ final class StoreKitProStoreSource: ProStoreSource { } private func appendAppPurchase(to snapshot: inout ProStoreSnapshot) async { +#if DEBUG && targetEnvironment(simulator) + let traceOperation = UUID() +#endif do { - switch try await AppTransaction.shared { +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.appTransactionBegin, operation: traceOperation) +#endif + let result = try await AppTransaction.shared +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.appTransactionReturn, operation: traceOperation) +#endif + switch result { case let .verified(transaction): guard let environment = Self.environment(transaction.environment) else { return } snapshot.appendVerifiedAppPurchase( @@ -148,6 +220,9 @@ final class StoreKitProStoreSource: ProStoreSource { snapshot.appVerificationEnvironment = Self.environment(transaction.environment) } } catch { +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.appTransactionError, operation: traceOperation) +#endif // An unavailable AppTransaction does not replace a previously verified permanent grant. } } diff --git a/CodexBarIOS/Views/ProPresentationView.swift b/CodexBarIOS/Views/ProPresentationView.swift index 82806561..8f26cbc2 100644 --- a/CodexBarIOS/Views/ProPresentationView.swift +++ b/CodexBarIOS/Views/ProPresentationView.swift @@ -45,7 +45,7 @@ struct ProPresentationView: View { } } .accessibilityIdentifier("codexbar-pro-presentation") - .accessibilityValue(isReady ? "ready" : "loading") + .accessibilityValue(readinessValue) .interactiveDismissDisabled(entitlement.command != nil) .background { ProSubscriptionSheetPresenter(isPresented: $showsManagement, groupID: managementGroupID) @@ -253,6 +253,21 @@ struct ProPresentationView: View { private var isBusy: Bool { entitlement.command != nil || isRefreshing || !entitlement.isResolved } + private var readinessValue: String { + let ready = isReady +#if DEBUG && targetEnvironment(simulator) + let catalog: Int + switch entitlement.catalog { + case .idle: catalog = 0 + case .loading: catalog = 1 + case .loaded: catalog = 2 + case .failed: catalog = 3 + } + ProNativeReadinessTrace.readiness(ready, resolved: entitlement.isResolved, refreshing: isRefreshing, catalog: catalog) +#endif + return ready ? "ready" : "loading" + } + private var isReady: Bool { guard entitlement.isResolved, !isRefreshing else { return false } switch entitlement.catalog { @@ -353,9 +368,20 @@ struct ProPresentationView: View { } private func refreshCommerce() async { +#if DEBUG && targetEnvironment(simulator) + let traceOperation = UUID() +#endif guard !isRefreshing, entitlement.command == nil else { return } isRefreshing = true - defer { isRefreshing = false } +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.viewRefreshBegin, operation: traceOperation) +#endif + defer { + isRefreshing = false +#if DEBUG && targetEnvironment(simulator) + ProNativeReadinessTrace.emit(.viewRefreshEnd, operation: traceOperation) +#endif + } await entitlement.refreshCommerce() } diff --git a/CodexBarIOSUITests/ProCommerceJourneyTests.swift b/CodexBarIOSUITests/ProCommerceJourneyTests.swift index c9244cef..f1187697 100644 --- a/CodexBarIOSUITests/ProCommerceJourneyTests.swift +++ b/CodexBarIOSUITests/ProCommerceJourneyTests.swift @@ -374,13 +374,16 @@ final class ProCommerceNativeJourneyTests: XCTestCase { private struct ProCommerceJourneyDriver { static let proURL = URL(string: "codexbar://pro")! let app: XCUIApplication + private let nativeObserver: ProNativeWaitTrace? init(scenario: String, entry: String? = "limit", largeText: Bool = false, launch: Bool = true) { + let runID = UUID() + nativeObserver = scenario == "commerce-native-storekit" ? ProNativeWaitTrace(runID: runID) : nil app = XCUIApplication() app.launchArguments = ["-AppleLanguages", "(en)", "-AppleLocale", "en_US"] app.launchEnvironment = [ "CODEXBAR_UI_TESTS": "1", - "CODEXBAR_UI_TEST_RUN_ID": UUID().uuidString, + "CODEXBAR_UI_TEST_RUN_ID": runID.uuidString, "CODEXBAR_UI_TEST_RESET": "1", "CODEXBAR_UI_TEST_SCENARIO": scenario, "CODEXBAR_UI_TEST_DEFAULT_TEXT": largeText ? "0" : "1", @@ -400,7 +403,16 @@ private struct ProCommerceJourneyDriver { let expectation = XCTNSPredicateExpectation( predicate: NSPredicate(format: "value == %@", "ready"), object: element("codexbar-pro-presentation") ) - XCTAssertEqual(XCTWaiter.wait(for: [expectation], timeout: timeout), .completed, + if let nativeObserver { + expectation.handler = { + nativeObserver.record(.predicatePositive) + return true + } + } + nativeObserver?.record(.waitBegin, timeout: timeout) + let result = XCTWaiter.wait(for: [expectation], timeout: timeout) + nativeObserver?.record(.waitEnd, timeout: timeout, result: Int(result.rawValue)) + XCTAssertEqual(result, .completed, "Purchase information did not resolve. \(app.debugDescription)") } @@ -488,3 +500,38 @@ private struct ProCommerceJourneyDriver { XCTFail("Unreachable commerce control: \(target). UI: \(app.debugDescription)") } } + +/// XCTest's public handler is Sendable and does not promise MainActor execution. +/// Only these diagnostic fields are protected; no assertion or AX operation moves here. +private final class ProNativeWaitTrace: @unchecked Sendable { + enum Stage: String { case waitBegin, predicatePositive, waitEnd } + private let lock = NSLock() + private let runID: UUID + private let launch = UUID().uuidString + private var observedPositive = false + private var sequence = 0 + + init(runID: UUID) { self.runID = runID } + + func record(_ stage: Stage, timeout: TimeInterval = 0, result: Int = 0) { + let wallBefore = Date().timeIntervalSince1970 + let uptime = ProcessInfo.processInfo.systemUptime + let wallAfter = Date().timeIntervalSince1970 + lock.lock() + defer { lock.unlock() } + if stage == .predicatePositive { + guard !observedPositive else { return } + observedPositive = true + } + sequence += 1 + let record: [String: Any] = [ + "schema": 1, "stage": stage.rawValue, "run": runID.uuidString, "launch": launch, "sequence": sequence, + "pid": ProcessInfo.processInfo.processIdentifier, + "wallBefore": wallBefore, "uptime": uptime, "wallAfter": wallAfter, + "timeout": timeout, "result": result, + ] + guard let data = try? JSONSerialization.data(withJSONObject: record, options: [.sortedKeys]), + let message = String(data: data, encoding: .utf8) else { return } + print("CB517RUNNER \(message)") + } +} diff --git a/scripts/pro_release_runner.py b/scripts/pro_release_runner.py index 113c2aa9..c861c48e 100755 --- a/scripts/pro_release_runner.py +++ b/scripts/pro_release_runner.py @@ -1,8 +1,15 @@ #!/usr/bin/env python3 """Explicit manual release phases; owns only newly provisioned simulators.""" import argparse +from datetime import datetime, timezone import json +import math import os +import re +import selectors +import shutil +import signal +import time from pathlib import Path import subprocess import sys @@ -192,6 +199,254 @@ def binaries(derived, signed): return result +# Native observation is manual-only and cannot alter XCTest's verdict. +OBSERVER_PHASES = {'ui-native', 'ui-native-purchase-error'} +OBSERVER_SUBSYSTEM = 'com.hemsoft.CodexBarIOS.native-readiness' +OBSERVER_PREFIX = 'CB517NATIVE ' +OBSERVER_LIMITS = dict(commandSeconds=15, phaseSeconds=60, stdoutBytes=8 * 1024 * 1024, + # Known retained export is197059893bytes; only the bounded session survives staging. + exportBytes=256 * 1024 * 1024, exportFiles=2048, sessionBytes=32 * 1024 * 1024) + + +def observer_export_cap(export): + if export and export.exists(): + size = 0 + for count, path in enumerate(export.rglob('*'), 1): + if count > OBSERVER_LIMITS['exportFiles'] or path.is_symlink(): + raise ValueError('Observer export byte/file cap exceeded') + if path.is_file(): + size += path.stat().st_size + if size > OBSERVER_LIMITS['exportBytes']: + raise ValueError('Observer export byte/file cap exceeded') + + +def observer_clock(row): + keys = ('wallBefore', 'uptime', 'wallAfter') + if (row.get('schema') != 1 or any(type(row.get(key)) not in (int, float) + or not math.isfinite(row[key]) for key in keys) or row['wallAfter'] < row['wallBefore']): + raise ValueError('Invalid observer clock tuple') + + +def observer_command(folder, label, argv, deadline, export=None): + """Finite process-group lifetime and byte/file caps, including partial output.""" + start = time.monotonic() + receipt = dict(command=argv, started=now(), limits=OBSERVER_LIMITS, complete=False, truncated=False, timedOut=False) + path = folder / (label + '.json') + failure, process = None, None + stdout, stderr = bytearray(), bytearray() + try: + if time.monotonic() >= deadline: + raise ValueError('Observer phase deadline exhausted before command') + with selectors.DefaultSelector() as selector: + process = subprocess.Popen(argv, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True) + selector.register(process.stdout, selectors.EVENT_READ, stdout) + selector.register(process.stderr, selectors.EVENT_READ, stderr) + end = min(deadline, time.monotonic() + OBSERVER_LIMITS['commandSeconds']) + while selector.get_map(): + if time.monotonic() >= end: + raise ValueError('Observer command deadline exceeded') + observer_export_cap(export) + for key, _ in selector.select(timeout=min(0.05, max(0, end - time.monotonic()))): + block = os.read(key.fileobj.fileno(), 65536) + if not block: + selector.unregister(key.fileobj) + else: + remaining = OBSERVER_LIMITS['stdoutBytes'] - len(stdout) - len(stderr) + key.data.extend(block[:max(0, remaining)]) + if len(block) > remaining: + raise ValueError('Observer output byte cap exceeded; partial output retained') + process.wait(timeout=max(0.001, end - time.monotonic())) + observer_export_cap(export) + if process.returncode: + raise ValueError(f'Observer command exited {process.returncode}') + receipt['complete'] = True + except (OSError, ValueError, subprocess.TimeoutExpired) as error: + failure = PhaseFailure(str(error)) + receipt.update(error=str(error), truncated='cap exceeded' in str(error), + timedOut='deadline' in str(error) or isinstance(error, subprocess.TimeoutExpired)) + finally: + cleanup_errors = [] + if process: + # Each step uses only this retained Popen handle, once. A failed + # kill/wait/close must not skip the other cleanup or erase evidence. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + except OSError as error: + cleanup_errors.append(dict(step='kill', error=str(error))) + try: + process.wait(timeout=2) + except (OSError, subprocess.TimeoutExpired) as error: + cleanup_errors.append(dict(step='wait', error=str(error))) + receipt['exit'] = process.returncode + for name, pipe in (('stdout', process.stdout), ('stderr', process.stderr)): + try: + pipe.close() + except OSError as error: + cleanup_errors.append(dict(step=name, error=str(error))) + if cleanup_errors: + receipt.update(complete=False, cleanupErrors=cleanup_errors) + details = '; '.join(item['step'] + ': ' + item['error'] for item in cleanup_errors) + message = 'Observer cleanup failed: ' + details + failure = PhaseFailure(f'{failure}; {message}' if failure else message, getattr(failure, 'code', 1)) + receipt.setdefault('error', message) + receipt.update(ended=now(), seconds=time.monotonic() - start, stdoutBytes=len(stdout), stderrBytes=len(stderr)) + write_errors = [] + for name, data in (('stdout', stdout), ('stderr', stderr)): + try: + (folder / (label + '.' + name)).write_bytes(data) + except OSError as error: + write_errors.append(dict(stream=name, error=str(error))) + if write_errors: + receipt.update(complete=False, writeErrors=write_errors) + details = '; '.join(item['stream'] + ': ' + item['error'] for item in write_errors) + message = 'Observer output write failed: ' + details + failure = PhaseFailure(f'{failure}; {message}' if failure else message, getattr(failure, 'code', 1)) + receipt.setdefault('error', message) + failure = persist(path, receipt, failure) + if failure: + raise failure + return bytes(stdout) + + +def observer_runner_records(path): + if path.stat().st_size > 32 * 1024 * 1024: + raise ValueError('Native XCTest log exceeds observer parse cap') + records = [] + for line in path.read_text().splitlines(): + if 'CB517RUNNER ' not in line: + continue + record = json.loads(line.split('CB517RUNNER ', 1)[1]) + if (not isinstance(record, dict) or record.get('stage') not in ('waitBegin', 'predicatePositive', 'waitEnd') + or not re_uuid(record.get('run', '')) or type(record.get('pid')) is not int): + raise ValueError('Invalid runner observation identity') + observer_clock(record) + records.append(record) + if not records or not {'waitBegin', 'waitEnd'} <= {row['stage'] for row in records}: + raise ValueError('Missing native wait boundary observations') + return records + + +def collect_native_observer(runner, folder, result, receipt): + """Read only the owned simulator, correlate app/runner UUID+PID, export before final hash.""" + output = folder / 'native-observer' + output.mkdir() + report = dict(schema=1, complete=False, limits=OBSERVER_LIMITS, source=runner.source, + device=runner.device, tools=runner.tools, phase=receipt['phase'], + started=receipt['started'], ended=receipt['ended']) + failure = None + staging = output / 'offline-export' + try: + deadline = time.monotonic() + OBSERVER_LIMITS['phaseSeconds'] + require_pinned(runner.tools, runner.device) + developer = Path(runner.tools['developer_dir']).resolve() + if Path(os.environ.get('DEVELOPER_DIR', '')).resolve() != developer: + raise ValueError('Observer DEVELOPER_DIR differs from qualified build') + ownership = read(runner.output / 'ownership.json') + created = ownership.get('created', {}) + argv = created.get('command', []) + if (ownership.get('owner') != str(ROOT) or ownership.get('tools') != runner.tools + or created.get('exit') != 0 + or len(argv) != 6 or argv[:3] != ['xcrun', 'simctl', 'create'] + or argv[3] != runner.device['deviceName'] or argv[5] != runner.device['runtimeBundlePath'] + or not re_uuid(runner.device['deviceId'])): + raise ValueError('Observer lacks exact simulator ownership') + validate_device(ownership['device'], runner.device) + if ownership['device']['deviceName'] != runner.device['deviceName']: + raise ValueError('Observer simulator name changed') + used = read(runner.output / 'native-use.json') + if used.get('deviceId') != runner.device['deviceId'] or used.get('output') != str(runner.output): + raise ValueError('Observer native claim differs') + for tool in ('simctl', 'xcresulttool'): + found = observer_command(output, 'find-' + tool, ['/usr/bin/xcrun', '--find', tool], deadline) + if Path(found.decode().strip()).resolve() != developer / 'usr/bin' / tool: + raise ValueError('Observer tool outside qualified developer directory') + records = observer_runner_records(folder / 'xcodebuild.log') + runs = {row['run'] for row in records} + if len(runs) != 1 or len({row['pid'] for row in records}) != 1: + raise ValueError('Ambiguous native runner identity') + report['runner'] = records + interval = [datetime.fromisoformat(receipt[key]).astimezone(timezone.utc).strftime('%Y-%m-%d %H:%M:%S.%f+0000') + for key in ('started', 'ended')] + prefix = ['/usr/bin/xcrun', 'simctl', 'spawn', runner.device['deviceId'], 'log', 'show', '--style', 'json', + '--start', interval[0], '--end', interval[1]] + raw = observer_command(output, 'app-trace', prefix + ['--predicate', f'subsystem == "{OBSERVER_SUBSYSTEM}"'], deadline) + events = json.loads(raw) + if not isinstance(events, list): + raise ValueError('Observer log is not a JSON event list') + app = [] + for event in events: + if not isinstance(event, dict): + raise ValueError('Invalid app log event shape') + message = event.get('eventMessage', '') + if not isinstance(message, str): + raise ValueError('Invalid app event message shape') + if not message.startswith(OBSERVER_PREFIX): + continue + row = json.loads(message[len(OBSERVER_PREFIX):]) + if (not isinstance(row, dict) or row.get('run') not in runs or row.get('pid') != event.get('processID') + or not event.get('processImagePath', '').endswith('/CodexBarIOS.app/CodexBarIOS') + or not re_uuid(row.get('launch', ''))): + raise ValueError('Observer app identity differs from runner') + observer_clock(row) + app.append(dict(marker=row, timestamp=event.get('timestamp'), bootUUID=event.get('bootUUID'), + processImageUUID=event.get('processImageUUID'))) + if not app or len({row['marker']['pid'] for row in app}) != 1 or len({row['marker']['launch'] for row in app}) != 1: + raise ValueError('Missing or ambiguous native app trace') + report['app'] = app + app_pid = app[0]['marker']['pid'] + # Strictly this app, not storekitd/trustd or other applications/accounts. + predicate = f'processID == {app_pid} AND (subsystem == "com.apple.securityd" OR subsystem == "com.apple.storekit")' + security = observer_command(output, 'app-sdk', prefix + ['--predicate', predicate], deadline) + sdk_events = json.loads(security) + if not isinstance(sdk_events, list): + raise ValueError('App SDK log is not a JSON list') + report['sdkEventCount'] = len(sdk_events) + if not result.exists(): + raise ValueError('No native xcresult for offline session export') + observer_command(output, 'session-export', ['/usr/bin/xcrun', 'xcresulttool', 'export', 'diagnostics', + '--path', str(result), '--output-path', str(staging)], deadline, export=staging) + # Offline extraction of already-captured xcresult diagnostics, no new sysdiagnose/host collection. + # Retain only this test bundle's Session logs. Other exported copies are removed below. + sessions = [path for path in staging.rglob('Session-*.log') + if any(part.startswith('CodexBarIOSUITests-') for part in path.parts)] + if len(sessions) != 1: + raise ValueError('Missing or ambiguous XCTest session log') + report['sessionBytes'] = sessions[0].stat().st_size + if report['sessionBytes'] > OBSERVER_LIMITS['sessionBytes']: + raise ValueError('XCTest session byte cap exceeded') + target = output / 'xctest-session.log' + shutil.copyfile(sessions[0], target) + session_text = target.read_text() + runner_pid = records[0]['pid'] + if f'CodexBarIOSUITests-Runner[{runner_pid}:' not in session_text or not re.search(rf'pid:\s*{app_pid}\b', session_text): + raise ValueError('XCTest session does not bind observed runner/app PIDs') + report['sessionSha256'] = digest(target) + stages = {row['marker']['stage'] for row in app} + if receipt.get('exit') == 0 and (not {'snapshotBegin', 'snapshotEnd', 'catalogBegin', 'catalogEnd', 'viewReadiness'} <= stages + or 'predicatePositive' not in {row['stage'] for row in records} + or not any(row['marker']['stage'] == 'viewReadiness' and row['marker'].get('count', 0) & 1 for row in app)): + raise ValueError('Passing native run lacks required app observations') + report['complete'] = True + except (ValueError, KeyError, TypeError, OSError, PhaseFailure) as error: + failure = error if isinstance(error, PhaseFailure) else PhaseFailure(str(error)) + report['error'] = str(error) + finally: + try: + if staging.exists(): + shutil.rmtree(staging) + except OSError as error: + report.update(complete=False, cleanupError=str(error)) + failure = failure or PhaseFailure(str(error)) + failure = persist(output / 'receipt.json', report, failure) + if failure: + raise failure + return report + + + class Runner: def __init__(self, output, device, source, tools): self.output, self.device, self.source, self.tools = output, device, source, tools @@ -311,6 +566,12 @@ def phase(self, name, build_path, base=False): failure = error if isinstance(error, PhaseFailure) else PhaseFailure(str(error)) receipt.update(exit=failure.code, error=str(error)) receipt['ended'] = now() + if name in OBSERVER_PHASES: + try: + receipt['nativeObserver'] = collect_native_observer(self, folder, result, receipt) + except (PhaseFailure, ValueError, KeyError, TypeError, OSError) as error: + receipt['observerError'] = str(error) + failure = failure or (error if isinstance(error, PhaseFailure) else PhaseFailure(str(error))) try: if result.exists(): receipt['result_initial_sha256'] = digest(result) diff --git a/scripts/tests/manual/test_native_observer.py b/scripts/tests/manual/test_native_observer.py new file mode 100644 index 00000000..b316e35d --- /dev/null +++ b/scripts/tests/manual/test_native_observer.py @@ -0,0 +1,480 @@ +"""Manual observer controls. No Xcode, simulator, provider, network or UI execution.""" +import json +from contextlib import contextmanager +import os +from pathlib import Path +import sys +import tempfile +import time +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import test_pro_release_validation as FIXTURE +R = FIXTURE.RUNNER +E = FIXTURE.EVIDENCE + + +class NativePhaseFlowTests(unittest.TestCase): + def setUp(self): + self.fixture = FIXTURE.RunnerCommandFailureTests() + self.fixture.setUp() + self.addCleanup(self.fixture.doCleanups) + self.f = self.fixture + self.runner = self.f.runner + original_fixture = FIXTURE.result_fixture + def ui_fixture(identities): + summary, tree, device = original_fixture(identities) + if identities and identities[0].startswith('CodexBarIOSUITests/'): + tree['testNodes'][0]['name'] = 'CodexBarIOSUITests' + return summary, tree, device + patcher = patch.object(FIXTURE, 'result_fixture', side_effect=ui_fixture) + patcher.start(); self.addCleanup(patcher.stop) + + def test_primary_65_capture_failure_reset_failure_and_shutdown_all_remain_visible(self): + self.f.exit_codes.update({'ui-native': 65, 'ui-reset': 66}) + with patch.object(R, 'collect_native_observer', side_effect=R.PhaseFailure('capture unavailable')): + state = R.sequence(['ui-native'], lambda name: self.runner.phase(name, self.f.build), + lambda: self.runner.phase('ui-reset', self.f.build)) + with patch.object(R, 'toolchain', return_value={}): + R.shutdown(self.f.root, self.f.device) + self.assertEqual(state['exit'], 65) + self.assertIn('66', state['cleanupFailure']) + row = E.read(self.f.root / 'ui-native/receipt.json') + self.assertEqual(row['exit'], 65) + self.assertIn('capture unavailable', row['observerError']) + self.assertTrue(any(call[-2:] == ['shutdown', self.f.device['deviceId']] for call in self.f.calls)) + + def test_observer_export_changes_are_in_final_result_hash(self): + def observe(runner, folder, result, receipt): + (result / 'observer-index.sqlite').write_text('materialized during offline export') + return {'complete': True} + with patch.object(R, 'collect_native_observer', side_effect=observe): + row = self.runner.phase('ui-native', self.f.build) + self.assertEqual(row['result_sha256'], E.digest(row['result'])) + self.assertTrue((Path(row['result']) / 'observer-index.sqlite').exists()) + + def test_missing_observation_on_pass_fails_phase_and_retains_reset(self): + with patch.object(R, 'collect_native_observer', side_effect=R.PhaseFailure('missing trace')): + state = R.sequence(['ui-native'], lambda name: self.runner.phase(name, self.f.build), + lambda: self.runner.phase('ui-reset', self.f.build)) + self.assertEqual(state['exit'], 1) + self.assertEqual(E.read(self.f.root / 'ui-reset/receipt.json')['exit'], 0) + + def test_required_observer_receipt_write_failure_preserves_primary(self): + self.f.exit_codes['ui-native'] = 65 + original = R.write + def write(path, value): + if path.name == 'receipt.json' and path.parent.name == 'ui-native': + raise OSError('full receipt volume') + original(path, value) + with patch.object(R, 'write', side_effect=write), patch.object(R, 'collect_native_observer', return_value={'complete': True}): + with self.assertRaises(R.PhaseFailure) as failure: + self.runner.phase('ui-native', self.f.build) + self.assertEqual(failure.exception.code, 65) + + def test_collection_never_runs_for_preparation_or_reset(self): + with patch.object(R, 'collect_native_observer') as collector: + for phase in ('ui-preparation', 'ui-reset', 'native-preparation'): + self.runner.phase(phase, self.f.build) + collector.assert_not_called() + + +class CollectorEvidenceTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.folder = self.root / 'ui-native' + self.folder.mkdir() + self.result = self.folder / 'result.xcresult' + self.result.mkdir() + self.device = dict(deviceId='10000000-0000-0000-0000-000000000001', deviceName='CodexBar 470 ipad test', + runtimeBundlePath='/qualified/runtime', family='ipad', architecture='arm64', + platform='iOS Simulator', osVersion='26.2', osBuildNumber='23C54', modelName='iPad mini (A17 Pro)') + self.tools = {'developer_dir': '/Applications/Qualified.app/Contents/Developer'} + self.runner = R.Runner(self.root, self.device, {}, self.tools) + self.run_id = '10000000-0000-0000-0000-000000000002' + self.records = [dict(schema=1, stage=stage, run=self.run_id, pid=20, wallBefore=100, uptime=10, wallAfter=100.001) + for stage in ('waitBegin', 'predicatePositive', 'waitEnd')] + self.write_runner() + self.receipt = dict(phase='ui-native', started='2026-10-25T01:00:00-04:00', ended='2026-10-25T01:01:00-04:00', exit=0) + E.write(self.root / 'ownership.json', dict(owner=str(R.ROOT), tools=self.tools, device=self.device, + created=dict(exit=0, command=['xcrun', 'simctl', 'create', self.device['deviceName'], 'type', '/qualified/runtime']))) + E.write(self.root / 'native-use.json', dict(deviceId=self.device['deviceId'], output=str(self.root))) + self.events = [dict(processID=10, processImagePath='/app/CodexBarIOS.app/CodexBarIOS', + eventMessage=R.OBSERVER_PREFIX + json.dumps(dict(schema=1, stage=stage, run=self.run_id, pid=10, count=1, wallBefore=100, uptime=10, wallAfter=100.001, + launch='10000000-0000-0000-0000-000000000003'))) + for stage in ('snapshotBegin', 'snapshotEnd', 'catalogBegin', 'catalogEnd', 'viewReadiness')] + self.commands = [] + self.patchers = [patch.object(R, 'require_pinned'), + patch.dict(os.environ, {'DEVELOPER_DIR': self.tools['developer_dir']}), + patch.object(R, 'observer_command', side_effect=self.command)] + for p in self.patchers: + p.start(); self.addCleanup(p.stop) + + def write_runner(self): + (self.folder / 'xcodebuild.log').write_text(''.join('CB517RUNNER ' + json.dumps(r) + '\n' for r in self.records)) + + def command(self, folder, label, argv, deadline, export=None): + self.commands.append(argv) + if label.startswith('find-'): + return (self.tools['developer_dir'] + '/usr/bin/' + label[5:]).encode() + if label == 'app-trace': + return json.dumps(self.events).encode() + if label == 'app-sdk': + return b'[]' + if label == 'session-export': + target = export / 'CodexBarIOSUITests-session/Session-test.log' + target.parent.mkdir(parents=True) + target.write_text(f'CodexBarIOSUITests-Runner[20:123] requesting app (pid:{self.events[0]["processID"]})') + (export / 'unrelated-copy.log').write_text('must not survive extraction') + (self.result / 'lazy-database.sqlite').write_text('materialized') + return b'' + + def collect(self): + return R.collect_native_observer(self.runner, self.folder, self.result, self.receipt) + + def test_exact_uuid_interval_pid_filter_and_only_session_retained(self): + row = self.collect() + self.assertTrue(row['complete']) + calls = [call for call in self.commands if 'spawn' in call] + self.assertTrue(all(call[3] == self.device['deviceId'] for call in calls)) + self.assertIn('processID == 10', calls[1][-1]) + self.assertIn('2026-10-25 05:00:00.000000+0000', calls[0]) + self.assertFalse((self.folder / 'native-observer/offline-export').exists()) + self.assertTrue((self.folder / 'native-observer/xctest-session.log').exists()) + + def test_sdk_predicate_matches_retained_native_product_event_identities(self): + # Producer40146062194/job121033727125, app-commerce-filtered.log47-48. + # Sanitized identities only; product response payload is not a fixture. + retained = [dict(processID=25746, subsystem='com.apple.storekit'), + dict(processID=25746, subsystem='com.apple.storekit'), + dict(processID=25747, subsystem='com.apple.storekit')] + for event in self.events: + event['processID'] = 25746 + row = json.loads(event['eventMessage'][len(R.OBSERVER_PREFIX):]); row['pid'] = 25746 + event['eventMessage'] = R.OBSERVER_PREFIX + json.dumps(row) + original = self.command + def observed(folder, label, argv, deadline, export=None): + if label == 'app-sdk': + predicate = argv[-1] + subsystems = R.re.findall(r'subsystem == "([^"]+)"', predicate) + pid = int(R.re.search(r'processID == (\d+)', predicate)[1]) + return json.dumps([event for event in retained if event['processID'] == pid + and event['subsystem'] in subsystems]).encode() + return original(folder, label, argv, deadline, export) + with patch.object(R, 'observer_command', side_effect=observed): + row = self.collect() + self.assertEqual(row['sdkEventCount'], 2) + + def test_known_producer_export_shape_retains_bounded_session(self): + # Retained producer diagnostics:345files,197059893bytes; session16551418bytes. + original = self.command + def known_shape(folder, label, argv, deadline, export=None): + if label != 'session-export': return original(folder, label, argv, deadline, export) + session = export / 'CodexBarIOSUITests-session/Session-producer.log' + session.parent.mkdir(parents=True) + with session.open('wb') as stream: + stream.write(b'CodexBarIOSUITests-Runner[20:123] requesting app (pid:10)') + stream.truncate(16551418) + for index in range(344): + with (export / f'other-{index:03}.dat').open('wb') as stream: + stream.truncate(197059893 - 16551418 if index == 0 else 0) + R.observer_export_cap(export) + return b'' + with patch.object(R, 'observer_command', side_effect=known_shape): + row = self.collect() + self.assertTrue(row['complete']) + self.assertEqual((self.folder / 'native-observer/xctest-session.log').stat().st_size, 16551418) + self.assertFalse((self.folder / 'native-observer/offline-export').exists()) + + def test_session_retention_has_its_own_cap(self): + with patch.dict(R.OBSERVER_LIMITS, sessionBytes=8), self.assertRaisesRegex(R.PhaseFailure, 'session byte cap'): + self.collect() + self.assertFalse((self.folder / 'native-observer/xctest-session.log').exists()) + self.assertFalse(E.read(self.folder / 'native-observer/receipt.json')['complete']) + + def test_missing_runner_boundary_never_qualifies(self): + self.records.pop(); self.write_runner() + with self.assertRaisesRegex(R.PhaseFailure, 'boundary'): + self.collect() + + def test_missing_app_trace_does_not_qualify_passing_xctest(self): + self.events = [] + with self.assertRaisesRegex(R.PhaseFailure, 'Missing'): + self.collect() + self.assertFalse(E.read(self.folder / 'native-observer/receipt.json')['complete']) + + def test_mismatched_app_pid_or_run_id_fails_closed(self): + self.events[0]['processID'] = 11 + with self.assertRaisesRegex(R.PhaseFailure, 'identity'): + self.collect() + + def test_wrong_ownership_prevents_collection(self): + row = E.read(self.root / 'ownership.json'); row['owner'] = '/another/checkout'; E.write(self.root / 'ownership.json', row) + with self.assertRaisesRegex(R.PhaseFailure, 'ownership'): + self.collect() + self.assertEqual(self.commands, []) + + def test_log_parse_failure_and_export_failure_remain_visible(self): + original = self.command + def bad(folder, label, argv, deadline, export=None): + return b'{invalid' if label == 'app-trace' else original(folder, label, argv, deadline, export) + with patch.object(R, 'observer_command', side_effect=bad), self.assertRaises(R.PhaseFailure): + self.collect() + self.assertFalse(E.read(self.folder / 'native-observer/receipt.json')['complete']) + + def test_wrong_json_shape_is_recorded_as_parse_failure(self): + self.events = [42] + with self.assertRaisesRegex(R.PhaseFailure, 'shape'): + self.collect() + self.assertFalse(E.read(self.folder / 'native-observer/receipt.json')['complete']) + + def test_nonfinite_clock_fails_closed(self): + self.records[0]['uptime'] = float('nan'); self.write_runner() + with self.assertRaisesRegex(R.PhaseFailure, 'clock'): + self.collect() + + def test_passing_run_missing_positive_callback_cannot_qualify(self): + self.records = [r for r in self.records if r['stage'] != 'predicatePositive']; self.write_runner() + with self.assertRaisesRegex(R.PhaseFailure, 'required app observations'): + self.collect() + + def test_session_pid_prefix_is_not_exact_app_identity(self): + original = self.command + def other_pid(folder, label, argv, deadline, export=None): + answer = original(folder, label, argv, deadline, export) + if label == 'session-export': + (export / 'CodexBarIOSUITests-session/Session-test.log').write_text('CodexBarIOSUITests-Runner[20:123] app pid:100') + return answer + with patch.object(R, 'observer_command', side_effect=other_pid), self.assertRaisesRegex(R.PhaseFailure, 'PIDs'): + self.collect() + + def test_offline_export_failure_is_not_success(self): + original = self.command + def bad(folder, label, argv, deadline, export=None): + if label == 'session-export': raise R.PhaseFailure('offline export unavailable') + return original(folder, label, argv, deadline, export) + with patch.object(R, 'observer_command', side_effect=bad), self.assertRaisesRegex(R.PhaseFailure, 'export unavailable'): + self.collect() + + def test_observer_report_write_failure_is_required(self): + with patch.object(R, 'write', side_effect=OSError('evidence volume full')), self.assertRaises(R.PhaseFailure): + self.collect() + + +class BoundedProcessTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(); self.addCleanup(self.temp.cleanup) + self.folder = Path(self.temp.name) + + def test_success_and_partial_output_cap(self): + R.observer_command(self.folder, 'ok', [sys.executable, '-c', 'print("bounded")'], time.monotonic() + 5) + with patch.dict(R.OBSERVER_LIMITS, stdoutBytes=128), self.assertRaisesRegex(R.PhaseFailure, 'byte cap'): + R.observer_command(self.folder, 'large', [sys.executable, '-c', 'print("x"*10000)'], time.monotonic()+5) + self.assertLessEqual((self.folder / 'large.stdout').stat().st_size, 128) + self.assertFalse(E.read(self.folder / 'large.json')['complete']) + self.assertTrue(E.read(self.folder / 'large.json')['truncated']) + + def test_deadline_terminates_owned_child_group(self): + code = 'import subprocess,sys,time; p=subprocess.Popen([sys.executable,"-c","import time; time.sleep(20)"]); print(p.pid,flush=True); time.sleep(20)' + with patch.dict(R.OBSERVER_LIMITS, commandSeconds=0.15), self.assertRaisesRegex(R.PhaseFailure, 'deadline'): + R.observer_command(self.folder, 'timeout', [sys.executable, '-c', code], time.monotonic()+5) + self.assertTrue(E.read(self.folder / 'timeout.json')['timedOut']) + pid = int((self.folder / 'timeout.stdout').read_text()) + # A killed child can briefly be a zombie awaiting reaping, but cannot remain running. + status = R.subprocess.run(['ps', '-p', str(pid), '-o', 'stat='], capture_output=True, text=True).stdout.strip() + self.assertTrue(not status or status.startswith('Z'), status) + + def test_export_cap_also_checks_fast_completed_command(self): + export = self.folder / 'export' + export.mkdir() + code = 'from pathlib import Path; Path(' + repr(str(export / 'large')) + ').write_bytes(b"x"*1024)' + with patch.dict(R.OBSERVER_LIMITS, exportBytes=128), self.assertRaisesRegex(R.PhaseFailure, 'export byte/file cap'): + R.observer_command(self.folder, 'export', [sys.executable, '-c', code], time.monotonic()+5, export) + + def test_output_write_failure_is_a_failure(self): + original = Path.write_bytes + def denied(path, data): + if path.suffix == '.stdout': raise OSError('output volume full') + return original(path, data) + with patch.object(Path, 'write_bytes', denied), self.assertRaisesRegex(R.PhaseFailure, 'write failed'): + R.observer_command(self.folder, 'write', [sys.executable, '-c', 'print(1)'], time.monotonic()+5) + +class FakeObserverPipe: + def __init__(self, process, name, data): + self.process, self.name, self.data = process, name, data + + def close(self): + self.process.events.append('close-' + self.name) + if self.name in self.process.close_errors: + raise OSError(self.name + ' close fault') + + +class FakeObserverProcess: + """Retained fake Popen handle. No real PID or OS process is created.""" + pid = 424242 + + def __init__(self, final_wait_error=None, close_errors=(), run_exit=0): + self.events, self.waits = [], [] + self.final_wait_error, self.close_errors, self.run_exit = final_wait_error, close_errors, run_exit + self.returncode = None + self.stdout = FakeObserverPipe(self, 'stdout', b'partial stdout\n') + self.stderr = FakeObserverPipe(self, 'stderr', b'partial stderr\n') + + def wait(self, timeout): + self.waits.append(timeout) + self.events.append('wait-run' if len(self.waits) == 1 else 'wait-cleanup') + if len(self.waits) > 1 and self.final_wait_error: + raise self.final_wait_error + self.returncode = self.run_exit + return self.returncode + + +class FakeObserverSelector: + def __enter__(self): return self + def __exit__(self, *_): return None + def register(self, pipe, _event, data): data.extend(pipe.data) + def get_map(self): return {} + + +@contextmanager +def fake_observer_os(process, kill_error=None): + def kill(pid, signal): + if pid != process.pid: + raise AssertionError('Cleanup used an unowned PID') + process.events.append('kill-owned-group') + if kill_error: + raise kill_error + with patch.object(R.subprocess, 'Popen', return_value=process), \ + patch.object(R.selectors, 'DefaultSelector', return_value=FakeObserverSelector()), \ + patch.object(R.os, 'killpg', side_effect=kill): + yield + + +class ObserverCleanupFaultTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(); self.addCleanup(self.temp.cleanup) + self.folder = Path(self.temp.name) + + def invoke(self, process, kill_error=None): + with fake_observer_os(process, kill_error): + return R.observer_command(self.folder, 'fault', ['fake-owned-command'], time.monotonic()+5) + + def assert_evidence_and_finite_cleanup(self, process): + self.assertEqual(process.events[-4:], ['kill-owned-group', 'wait-cleanup', 'close-stdout', 'close-stderr']) + self.assertEqual(len(process.waits), 2) + self.assertEqual(process.waits[-1], 2) + self.assertEqual((self.folder / 'fault.stdout').read_bytes(), b'partial stdout\n') + self.assertEqual((self.folder / 'fault.stderr').read_bytes(), b'partial stderr\n') + row = E.read(self.folder / 'fault.json') + self.assertFalse(row['complete']) + return row + + def test_final_wait_timeout_is_phase_failure_and_still_closes_both_pipes(self): + process = FakeObserverProcess(final_wait_error=R.subprocess.TimeoutExpired('fake-owned-command', 2)) + with self.assertRaises(R.PhaseFailure): self.invoke(process) + row = self.assert_evidence_and_finite_cleanup(process) + self.assertEqual([e['step'] for e in row['cleanupErrors']], ['wait']) + self.assertIn('timed out', row['cleanupErrors'][0]['error']) + + def test_run_error_remains_primary_with_all_cleanup_faults_visible(self): + process = FakeObserverProcess(final_wait_error=R.subprocess.TimeoutExpired('fake-owned-command', 2), + close_errors=('stdout', 'stderr'), run_exit=7) + with self.assertRaises(R.PhaseFailure) as caught: self.invoke(process, OSError('kill fault')) + row = self.assert_evidence_and_finite_cleanup(process) + self.assertEqual(row['error'], 'Observer command exited 7') + self.assertEqual([e['step'] for e in row['cleanupErrors']], ['kill', 'wait', 'stdout', 'stderr']) + self.assertIn('exited 7', str(caught.exception)) + self.assertIn('kill fault', str(caught.exception)) + self.assertIn('stderr close fault', str(caught.exception)) + + def test_pipe_close_oserror_attempts_other_pipe_and_fails_successful_command(self): + process = FakeObserverProcess(close_errors=('stdout',)) + with self.assertRaises(R.PhaseFailure): self.invoke(process) + row = self.assert_evidence_and_finite_cleanup(process) + self.assertEqual([e['step'] for e in row['cleanupErrors']], ['stdout']) + + def test_kill_oserror_still_attempts_final_wait_and_pipe_closes(self): + process = FakeObserverProcess() + with self.assertRaises(R.PhaseFailure): self.invoke(process, OSError('kill fault')) + row = self.assert_evidence_and_finite_cleanup(process) + self.assertEqual([e['step'] for e in row['cleanupErrors']], ['kill']) + + def test_final_wait_oserror_still_attempts_both_pipe_closes(self): + process = FakeObserverProcess(final_wait_error=OSError('wait fault')) + with self.assertRaises(R.PhaseFailure): self.invoke(process) + row = self.assert_evidence_and_finite_cleanup(process) + self.assertEqual([e['step'] for e in row['cleanupErrors']], ['wait']) + + def test_absent_owned_group_is_benign(self): + process = FakeObserverProcess() + self.assertEqual(self.invoke(process, ProcessLookupError('already gone')), b'partial stdout\n') + self.assertTrue(E.read(self.folder / 'fault.json')['complete']) + + def test_output_write_fault_still_saves_other_stream_and_command_receipt(self): + process = FakeObserverProcess() + original = Path.write_bytes + def denied(path, data): + if path.name == 'fault.stdout': raise OSError('stdout volume fault') + return original(path, data) + with patch.object(Path, 'write_bytes', denied), self.assertRaises(R.PhaseFailure): self.invoke(process) + self.assertEqual((self.folder / 'fault.stderr').read_bytes(), b'partial stderr\n') + row = E.read(self.folder / 'fault.json') + self.assertFalse(row['complete']) + self.assertEqual(row['writeErrors'][0]['stream'], 'stdout') + self.assertEqual(process.events[-2:], ['close-stdout', 'close-stderr']) + + +class ObserverCleanupPhaseTests(unittest.TestCase): + setUp = NativePhaseFlowTests.setUp + """Only these two new controls, not the inherited cases, belong to the cleanup proof.""" + def cleanup_case(self, native_exit, reset_exit): + self.f.exit_codes.update({'ui-native': native_exit, 'ui-reset': reset_exit}) + self.runner.tools = {'developer_dir': '/Applications/Qualified.app/Contents/Developer'} + self.f.device.update(deviceName='CodexBar 470 ipad fake', runtimeBundlePath='/qualified/runtime', + deviceId='10000000-0000-0000-0000-000000000001') + build = E.read(self.f.build); build.update(tools=self.runner.tools, device=self.f.device); E.write(self.f.build, build) + E.write(self.f.root / 'ownership.json', dict(owner=str(R.ROOT), tools=self.runner.tools, device=self.f.device, + created=dict(exit=0, command=['xcrun', 'simctl', 'create', self.f.device['deviceName'], 'type', '/qualified/runtime']))) + E.write(self.f.root / 'native-use.json', dict(deviceId=self.f.device['deviceId'], output=str(self.f.root))) + process = FakeObserverProcess(final_wait_error=R.subprocess.TimeoutExpired('fake-owned-command', 2)) + original_fixture = FIXTURE.result_fixture + def matching_device(identities): + summary, tree, _ = original_fixture(identities) + summary['devicesAndConfigurations'][0]['device'] = self.f.device + tree['devices'] = [self.f.device] + return summary, tree, self.f.device + # Actual collector and observer_command are both executed. Only OS handles/tools are fake. + with patch.object(R, 'require_pinned'), patch.dict(os.environ, {'DEVELOPER_DIR': self.runner.tools['developer_dir']}), \ + patch.object(FIXTURE, 'result_fixture', side_effect=matching_device), fake_observer_os(process): + state = R.sequence(['ui-native'], lambda name: self.runner.phase(name, self.f.build), + lambda: self.runner.phase('ui-reset', self.f.build)) + with patch.object(R, 'toolchain', return_value={}): R.shutdown(self.f.root, self.f.device) + phase = E.read(self.f.root / 'ui-native/receipt.json') + capture = E.read(self.f.root / 'ui-native/native-observer/receipt.json') + command = E.read(self.f.root / 'ui-native/native-observer/find-simctl.json') + self.assertFalse(capture['complete']); self.assertFalse(command['complete']) + self.assertIn('timed out', phase['observerError']) + self.assertEqual(command['cleanupErrors'][0]['step'], 'wait') + self.assertEqual(len(process.waits), 2) + self.assertEqual(process.events[-2:], ['close-stdout', 'close-stderr']) + self.assertTrue(any(call[-2:] == ['shutdown', self.f.device['deviceId']] for call in self.f.calls)) + return state, phase + + def test_native65_cleanup_timeout_reset66_and_shutdown_preserve_all_evidence(self): + state, phase = self.cleanup_case(65, 66) + self.assertEqual(state['exit'], 65); self.assertEqual(phase['exit'], 65) + self.assertIn('66', state['cleanupFailure']) + + def test_passing_xctest_cleanup_timeout_fails_observer_and_preserves_reset(self): + state, phase = self.cleanup_case(0, 0) + self.assertEqual(state['exit'], 1); self.assertEqual(phase['exit'], 1) + self.assertIsNone(state['cleanupFailure']) + self.assertNotIn('validationError', phase) + + + +if __name__ == '__main__': + unittest.main() From b65417f9ba6178bd80febe8953e8ed4d466e06a9 Mon Sep 17 00:00:00 2001 From: HemSoft Date: Sun, 25 Oct 2026 13:04:32 -0400 Subject: [PATCH 07/11] Honor supplied simulator ownership receipts during capture Refs #517. Preserve the documented separate provisioning and commerce execution layout and validate the native-use claim beside its ownership receipt. --- scripts/pro_release_runner.py | 9 +++++---- scripts/tests/manual/test_native_observer.py | 21 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/scripts/pro_release_runner.py b/scripts/pro_release_runner.py index c861c48e..c4561343 100755 --- a/scripts/pro_release_runner.py +++ b/scripts/pro_release_runner.py @@ -344,7 +344,7 @@ def collect_native_observer(runner, folder, result, receipt): developer = Path(runner.tools['developer_dir']).resolve() if Path(os.environ.get('DEVELOPER_DIR', '')).resolve() != developer: raise ValueError('Observer DEVELOPER_DIR differs from qualified build') - ownership = read(runner.output / 'ownership.json') + ownership = read(runner.ownership) created = ownership.get('created', {}) argv = created.get('command', []) if (ownership.get('owner') != str(ROOT) or ownership.get('tools') != runner.tools @@ -356,7 +356,7 @@ def collect_native_observer(runner, folder, result, receipt): validate_device(ownership['device'], runner.device) if ownership['device']['deviceName'] != runner.device['deviceName']: raise ValueError('Observer simulator name changed') - used = read(runner.output / 'native-use.json') + used = read(runner.ownership.parent / 'native-use.json') if used.get('deviceId') != runner.device['deviceId'] or used.get('output') != str(runner.output): raise ValueError('Observer native claim differs') for tool in ('simctl', 'xcresulttool'): @@ -448,8 +448,9 @@ def collect_native_observer(runner, folder, result, receipt): class Runner: - def __init__(self, output, device, source, tools): + def __init__(self, output, device, source, tools, ownership=None): self.output, self.device, self.source, self.tools = output, device, source, tools + self.ownership = Path(ownership).resolve() if ownership is not None else output / 'ownership.json' self.manifest = load_manifest() self.receipts = [] @@ -676,7 +677,7 @@ def main(): shutdown(output, device) outcome.update(exit=0, complete=True) else: - runner = Runner(output, device, source, tools) + runner = Runner(output, device, source, tools, ownership=args.ownership) manifest = runner.manifest if args.mode == 'base': # Build the ordinary scheme's full test bundle (35 methods), then diff --git a/scripts/tests/manual/test_native_observer.py b/scripts/tests/manual/test_native_observer.py index b316e35d..6498d36d 100644 --- a/scripts/tests/manual/test_native_observer.py +++ b/scripts/tests/manual/test_native_observer.py @@ -195,6 +195,27 @@ def test_session_retention_has_its_own_cap(self): self.assertFalse((self.folder / 'native-observer/xctest-session.log').exists()) self.assertFalse(E.read(self.folder / 'native-observer/receipt.json')['complete']) + def test_supplied_ownership_in_separate_provisioning_directory(self): + device = self.root / 'device'; device.mkdir() + ownership = device / 'ownership.json' + (self.root / 'ownership.json').rename(ownership) + (self.root / 'native-use.json').rename(device / 'native-use.json') + # Exposes the supported input path to the actual collector, not a copied receipt. + self.runner = R.Runner(self.root, self.device, {}, self.tools, ownership=ownership) + row = self.collect() + self.assertTrue(row['complete']) + self.assertFalse((self.runner.output / 'ownership.json').exists()) + + def test_supplied_ownership_requires_claim_beside_receipt(self): + device = self.root / 'device'; device.mkdir() + ownership = device / 'ownership.json' + (self.root / 'ownership.json').rename(ownership) + self.runner = R.Runner(self.root, self.device, {}, self.tools, ownership=ownership) + # The unrelated claim left in the execution directory must not be accepted. + with self.assertRaises(R.PhaseFailure): + self.collect() + self.assertEqual(self.commands, []) + def test_missing_runner_boundary_never_qualifies(self): self.records.pop(); self.write_runner() with self.assertRaisesRegex(R.PhaseFailure, 'boundary'): From 91b8e826353b378de82b104c5833f6acf5e3fbb4 Mon Sep 17 00:00:00 2001 From: HemSoft Date: Sun, 25 Oct 2026 15:55:16 -0400 Subject: [PATCH 08/11] Use supported log date syntax without widening phase evidence Refs #517. Floor/ceil whole-second query bounds, filter back to exact phase timestamps, and parse the emitted log offset format on Python 3.9. Preserve readiness assertions and all native workloads. --- scripts/pro_release_runner.py | 30 ++++++++++++-- scripts/tests/manual/test_native_observer.py | 43 +++++++++++++++++--- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/scripts/pro_release_runner.py b/scripts/pro_release_runner.py index c4561343..5f1dc5e4 100755 --- a/scripts/pro_release_runner.py +++ b/scripts/pro_release_runner.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Explicit manual release phases; owns only newly provisioned simulators.""" import argparse -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone import json import math import os @@ -329,6 +329,22 @@ def observer_runner_records(path): return records +def observer_log_bounds(receipt): + start, end = [datetime.fromisoformat(receipt[key]).astimezone(timezone.utc) for key in ('started', 'ended')] + # log show accepts whole-second dates. Query outward, then enforce the exact phase below. + stop = end.replace(microsecond=0) + (timedelta(seconds=1) if end.microsecond else timedelta()) + return [value.strftime('%Y-%m-%d %H:%M:%S%z') for value in (start.replace(microsecond=0), stop)] + + +def observer_event_in_phase(event, receipt): + if not isinstance(event, dict) or not isinstance(event.get('timestamp'), str): + raise ValueError('Missing observer event timestamp') + timestamp = datetime.strptime(event['timestamp'], '%Y-%m-%d %H:%M:%S.%f%z') + if timestamp.utcoffset() is None: + raise ValueError('Observer event timestamp lacks timezone') + return datetime.fromisoformat(receipt['started']) <= timestamp <= datetime.fromisoformat(receipt['ended']) + + def collect_native_observer(runner, folder, result, receipt): """Read only the owned simulator, correlate app/runner UUID+PID, export before final hash.""" output = folder / 'native-observer' @@ -368,8 +384,8 @@ def collect_native_observer(runner, folder, result, receipt): if len(runs) != 1 or len({row['pid'] for row in records}) != 1: raise ValueError('Ambiguous native runner identity') report['runner'] = records - interval = [datetime.fromisoformat(receipt[key]).astimezone(timezone.utc).strftime('%Y-%m-%d %H:%M:%S.%f+0000') - for key in ('started', 'ended')] + interval = observer_log_bounds(receipt) + report['queryInterval'] = interval prefix = ['/usr/bin/xcrun', 'simctl', 'spawn', runner.device['deviceId'], 'log', 'show', '--style', 'json', '--start', interval[0], '--end', interval[1]] raw = observer_command(output, 'app-trace', prefix + ['--predicate', f'subsystem == "{OBSERVER_SUBSYSTEM}"'], deadline) @@ -377,9 +393,13 @@ def collect_native_observer(runner, folder, result, receipt): if not isinstance(events, list): raise ValueError('Observer log is not a JSON event list') app = [] + report['appOutsidePhaseCount'] = 0 for event in events: if not isinstance(event, dict): raise ValueError('Invalid app log event shape') + if not observer_event_in_phase(event, receipt): + report['appOutsidePhaseCount'] += 1 + continue message = event.get('eventMessage', '') if not isinstance(message, str): raise ValueError('Invalid app event message shape') @@ -403,7 +423,9 @@ def collect_native_observer(runner, folder, result, receipt): sdk_events = json.loads(security) if not isinstance(sdk_events, list): raise ValueError('App SDK log is not a JSON list') - report['sdkEventCount'] = len(sdk_events) + exact_sdk = [event for event in sdk_events if observer_event_in_phase(event, receipt)] + report['sdkOutsidePhaseCount'] = len(sdk_events) - len(exact_sdk) + report['sdkEventCount'] = len(exact_sdk) if not result.exists(): raise ValueError('No native xcresult for offline session export') observer_command(output, 'session-export', ['/usr/bin/xcrun', 'xcresulttool', 'export', 'diagnostics', diff --git a/scripts/tests/manual/test_native_observer.py b/scripts/tests/manual/test_native_observer.py index 6498d36d..3c06e226 100644 --- a/scripts/tests/manual/test_native_observer.py +++ b/scripts/tests/manual/test_native_observer.py @@ -102,7 +102,7 @@ def setUp(self): E.write(self.root / 'ownership.json', dict(owner=str(R.ROOT), tools=self.tools, device=self.device, created=dict(exit=0, command=['xcrun', 'simctl', 'create', self.device['deviceName'], 'type', '/qualified/runtime']))) E.write(self.root / 'native-use.json', dict(deviceId=self.device['deviceId'], output=str(self.root))) - self.events = [dict(processID=10, processImagePath='/app/CodexBarIOS.app/CodexBarIOS', + self.events = [dict(timestamp='2026-10-25 05:00:10.000000+0000', processID=10, processImagePath='/app/CodexBarIOS.app/CodexBarIOS', eventMessage=R.OBSERVER_PREFIX + json.dumps(dict(schema=1, stage=stage, run=self.run_id, pid=10, count=1, wallBefore=100, uptime=10, wallAfter=100.001, launch='10000000-0000-0000-0000-000000000003'))) for stage in ('snapshotBegin', 'snapshotEnd', 'catalogBegin', 'catalogEnd', 'viewReadiness')] @@ -141,16 +141,16 @@ def test_exact_uuid_interval_pid_filter_and_only_session_retained(self): calls = [call for call in self.commands if 'spawn' in call] self.assertTrue(all(call[3] == self.device['deviceId'] for call in calls)) self.assertIn('processID == 10', calls[1][-1]) - self.assertIn('2026-10-25 05:00:00.000000+0000', calls[0]) + self.assertIn('2026-10-25 05:00:00+0000', calls[0]) self.assertFalse((self.folder / 'native-observer/offline-export').exists()) self.assertTrue((self.folder / 'native-observer/xctest-session.log').exists()) def test_sdk_predicate_matches_retained_native_product_event_identities(self): # Producer40146062194/job121033727125, app-commerce-filtered.log47-48. # Sanitized identities only; product response payload is not a fixture. - retained = [dict(processID=25746, subsystem='com.apple.storekit'), - dict(processID=25746, subsystem='com.apple.storekit'), - dict(processID=25747, subsystem='com.apple.storekit')] + retained = [dict(timestamp='2026-10-25 05:00:10.000000+0000', processID=25746, subsystem='com.apple.storekit'), + dict(timestamp='2026-10-25 05:00:10.000000+0000', processID=25746, subsystem='com.apple.storekit'), + dict(timestamp='2026-10-25 05:00:10.000000+0000', processID=25747, subsystem='com.apple.storekit')] for event in self.events: event['processID'] = 25746 row = json.loads(event['eventMessage'][len(R.OBSERVER_PREFIX):]); row['pid'] = 25746 @@ -216,6 +216,39 @@ def test_supplied_ownership_requires_claim_beside_receipt(self): self.collect() self.assertEqual(self.commands, []) + def test_fractional_phase_uses_supported_log_dates_and_outward_query_bounds(self): + self.receipt.update(started='2026-10-25T01:00:00.900000-04:00', ended='2026-10-25T01:01:00.100000-04:00') + original = self.command + def exact_parser(folder, label, argv, deadline, export=None): + if label in ('app-trace', 'app-sdk'): + from datetime import datetime + for option in ('--start', '--end'): + datetime.strptime(argv[argv.index(option)+1], '%Y-%m-%d %H:%M:%S%z') + return original(folder, label, argv, deadline, export) + with patch.object(R, 'observer_command', side_effect=exact_parser): + row = self.collect() + self.assertTrue(row['complete']) + argv = next(c for c in self.commands if 'spawn' in c) + self.assertIn('2026-10-25 05:00:00+0000', argv) + self.assertIn('2026-10-25 05:01:01+0000', argv) + + def test_outward_query_does_not_admit_records_outside_exact_phase(self): + self.receipt.update(started='2026-10-25T01:00:00.900000-04:00', ended='2026-10-25T01:01:00.100000-04:00') + outside = dict(self.events[0], timestamp='2026-10-25 05:00:00.100000+0000') + row = json.loads(outside['eventMessage'][len(R.OBSERVER_PREFIX):]); row['run'] = '10000000-0000-0000-0000-000000000099' + outside['eventMessage'] = R.OBSERVER_PREFIX + json.dumps(row) + self.events.insert(0, outside) + original = self.command + def sdk_edges(folder, label, argv, deadline, export=None): + if label == 'app-sdk': + return json.dumps([dict(timestamp=stamp) for stamp in ['2026-10-25 05:00:00.100000+0000', + '2026-10-25 05:00:10.000000+0000', '2026-10-25 05:01:00.900000+0000']]).encode() + return original(folder, label, argv, deadline, export) + with patch.object(R, 'observer_command', side_effect=sdk_edges): + row = self.collect() + self.assertEqual(len(row['app']), 5) + self.assertEqual(row['sdkEventCount'], 1) + def test_missing_runner_boundary_never_qualifies(self): self.records.pop(); self.write_runner() with self.assertRaisesRegex(R.PhaseFailure, 'boundary'): From 35cbb6f947b299248559a40c822d44a1f3660d23 Mon Sep 17 00:00:00 2001 From: HemSoft Date: Sun, 25 Oct 2026 22:47:31 -0400 Subject: [PATCH 09/11] Guard manual CI cost comparisons by image and toolchain (#517) --- .github/workflows/ci.yml | 42 ++++- CHANGELOG.md | 1 + UI-TESTING.md | 37 ++++ scripts/prepare-automatic-cost.py | 135 ++++++++++++++ scripts/tests/manual/test_automatic_cost.py | 187 ++++++++++++++++++++ scripts/tests/test_ci_policy.py | 5 +- scripts/tests/test_ui_release_shards.py | 5 +- 7 files changed, 400 insertions(+), 12 deletions(-) create mode 100644 scripts/prepare-automatic-cost.py create mode 100644 scripts/tests/manual/test_automatic_cost.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 068579f3..9ec550e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,14 +5,29 @@ on: workflow_dispatch: inputs: ui_validation_mode: - description: Full release gate or focused native account-menu diagnosis + description: Full release gate, focused account-menu diagnosis, or automatic-cost control type: choice options: - full - account-menu-comparison + - automatic-cost default: full comparison_base_sha: - description: Exact ancestor SHA for account-menu comparison only + description: Exact ancestor SHA for manual comparison modes + type: string + default: '' + + cost_source_role: + description: Source measured by an automatic-cost control + type: choice + options: [baseline, candidate] + default: candidate + cost_expected_image: + description: Exact hosted ImageVersion required before cost workload + type: string + default: '' + cost_expected_toolchain: + description: Baseline toolchain SHA256; required for candidate cost control type: string default: '' @@ -38,6 +53,17 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + fetch-depth: ${{ inputs.ui_validation_mode == 'automatic-cost' && '0' || '1' }} + + - name: Verify manual automatic-cost identity + if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode == 'automatic-cost' }} + env: + AUTOMATIC_COST_MODE: ${{ inputs.ui_validation_mode }} + AUTOMATIC_COST_ROLE: ${{ inputs.cost_source_role }} + AUTOMATIC_COST_BASE: ${{ inputs.comparison_base_sha }} + AUTOMATIC_COST_IMAGE: ${{ inputs.cost_expected_image }} + AUTOMATIC_COST_TOOLCHAIN: ${{ inputs.cost_expected_toolchain }} + run: python3 scripts/prepare-automatic-cost.py --receipt "$RUNNER_TEMP/automatic-cost.json" - name: Run pinned SwiftLint policy run: >- @@ -51,7 +77,7 @@ jobs: run: python3 -m unittest discover -s scripts/tests -p 'test_ci_policy.py' -v strict-concurrency: - if: ${{ github.event_name == 'workflow_dispatch' }} + if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'automatic-cost' }} name: Strict concurrency runs-on: macos-26 env: @@ -70,7 +96,7 @@ jobs: run: ./scripts/check-strict-concurrency.sh ios-tests: - if: ${{ github.event_name == 'workflow_dispatch' }} + if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'automatic-cost' }} name: iOS tests runs-on: macos-26 env: @@ -126,7 +152,7 @@ jobs: retention-days: 14 watch-tests: - if: ${{ github.event_name == 'workflow_dispatch' }} + if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'automatic-cost' }} name: watchOS tests runs-on: macos-26 env: @@ -199,7 +225,7 @@ jobs: retention-days: 14 smoke-tests: - if: ${{ github.event_name == 'workflow_dispatch' }} + if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'automatic-cost' }} name: SwiftPM smoke tests runs-on: macos-26 env: @@ -225,7 +251,7 @@ jobs: ios-ui-families: name: iOS UI journeys (${{ matrix.family }}, ${{ matrix.shard }}) - if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'account-menu-comparison' && success() }} + if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'account-menu-comparison' && inputs.ui_validation_mode != 'automatic-cost' && success() }} needs: - swiftlint - strict-concurrency @@ -288,7 +314,7 @@ jobs: ios-ui-tests: name: Full iOS UI validation - if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'account-menu-comparison' }} + if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'account-menu-comparison' && inputs.ui_validation_mode != 'automatic-cost' }} needs: ios-ui-families permissions: actions: read diff --git a/CHANGELOG.md b/CHANGELOG.md index a99c25f7..036340f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ building, testing, or releasing the app. Wait for billing warnings to disappear after reloading sign-in. Add aggregate-only manual validation hints for matching open PR watches; live notification delivery remains pending qualification. + Require matching hosted image and toolchain identities for manual automatic-CI cost comparisons. ([#517](https://github.com/hemsoft-dev/codexbar-ios/issues/517)) - Trace native StoreKit status delivery on the existing test receiver, preserve status and clock diagnostics without transaction identifiers, and verify stable test-store cleanup. ([#470](https://github.com/hemsoft-dev/codexbar-ios/issues/470)) diff --git a/UI-TESTING.md b/UI-TESTING.md index 731d2463..f863785c 100644 --- a/UI-TESTING.md +++ b/UI-TESTING.md @@ -558,3 +558,40 @@ disconnect and cancellation. Claude fixtures omit the unused legacy aliases. Set `CODEXBAR_UI_TEST_SUBJOURNEY` to `billing` in the copied xctestrun file to run only the guided billing recovery checks while diagnosing a connection flow. Use `renewals` for complete renewal-focused validation on both device families. + + +## Manual automatic-CI cost controls + +`ui_validation_mode=automatic-cost` runs only the existing SwiftLint job. It does +not run native quality jobs, UI workers, the full aggregate or its notifier. +Ordinary PR runs still execute checkout, pinned SwiftLint and the four policy +tests, with the existing runner, permissions and ten-minute limit. Full and +account-menu manual modes retain their native jobs, selectors and ceilings. + +Use one predeclared baseline/candidate pair. Both dispatches use the same frozen +candidate workflow ref; `comparison_base_sha` is its exact ancestor/default-base +SHA. `cost_source_role=baseline` checks out that ancestor before the existing +lint/policy workload; `candidate` retains the exact dispatch SHA. Both first +verify an explicit `cost_expected_image` against the assigned runner's actual +`ImageVersion`. The candidate also requires `cost_expected_toolchain`, copied +from the successful baseline's `AUTOMATIC_COST_IDENTITY` log receipt. The hash +binds the default developer directory, Xcode/Swift builds, OS build, architecture, +CPU count and memory. A mismatch fails before the measured lint/policy workload; +it never silently selects another image or retries. + +The manual guard verifies that effective automatic job commands/configuration +match the baseline, rejects dirty or wrong source, and records exact source and +tool identities. Only this manual mode fetches history for ancestor validation. +The normal automatic checkout remains depth one. Source selection happens after +checkout; the Actions run SHA identifies the workflow/controller, while the +receipt's `measuredSource` identifies the source actually linted. A baseline cost +run is not a current-head source-validation result. + +Acceptance remains completed Actions **workflow elapsed time and summed +non-skipped job runtime**: candidate must be no larger in either measure. Count +checkout, identity checks, source selection and all setup/cleanup time; make no +queue/network/guard deductions, tolerance, per-file normalization or synthetic +job-time claim. Both must pass on the same image, toolchain and hardware config. +Retain original adverse observations and any invalid/failed pair. Do not repeat +a pair or launch full native validation merely to obtain a green timing. A +successful cost-only run does not qualify any native/full release gate. diff --git a/scripts/prepare-automatic-cost.py b/scripts/prepare-automatic-cost.py new file mode 100644 index 00000000..2e1770ff --- /dev/null +++ b/scripts/prepare-automatic-cost.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Prepare one manual cost control; normal PR CI never calls this script.""" + +import argparse +import hashlib +import json +import os +import re +import subprocess +from pathlib import Path + + +MANUAL_GUARD = """ - name: Verify manual automatic-cost identity + if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode == 'automatic-cost' }} + env: + AUTOMATIC_COST_MODE: ${{ inputs.ui_validation_mode }} + AUTOMATIC_COST_ROLE: ${{ inputs.cost_source_role }} + AUTOMATIC_COST_BASE: ${{ inputs.comparison_base_sha }} + AUTOMATIC_COST_IMAGE: ${{ inputs.cost_expected_image }} + AUTOMATIC_COST_TOOLCHAIN: ${{ inputs.cost_expected_toolchain }} + run: python3 scripts/prepare-automatic-cost.py --receipt "$RUNNER_TEMP/automatic-cost.json" + +""" + +class Refused(ValueError): + pass + + +def command(*args): + result = subprocess.run(args, check=True, capture_output=True, text=True, timeout=10) + if len(result.stdout.encode()) > 1_048_576: + raise Refused('command-output-cap') + return result.stdout.strip() + + +def automatic_job(workflow): + match = re.search(r'^ swiftlint:\n.*?(?=^ [\w-]+:\n|\Z)', workflow, re.M | re.S) + if not match: + raise Refused('missing-automatic-job') + block = match.group() + # These two declarations affect only the explicit manual comparison mode. + block = re.sub( + r"^ fetch-depth: \$\{\{ inputs.ui_validation_mode == 'automatic-cost' && '0' \|\| '1' \}\}\n", + '', block, flags=re.M, + ) + block = block.replace(MANUAL_GUARD, '', 1) + return block + + +def prepare(env, receipt): + if (env.get('GITHUB_ACTIONS') != 'true' + or env.get('GITHUB_EVENT_NAME') != 'workflow_dispatch' + or env.get('GITHUB_REPOSITORY') != 'hemsoft-dev/codexbar-ios' + or env.get('AUTOMATIC_COST_MODE') != 'automatic-cost'): + raise Refused('wrong-context') + candidate = env.get('GITHUB_SHA', '') + baseline = env.get('AUTOMATIC_COST_BASE', '') + role = env.get('AUTOMATIC_COST_ROLE') + if not all(re.fullmatch(r'[0-9a-f]{40}', x) for x in (candidate, baseline)): + raise Refused('invalid-source-sha') + if candidate == baseline or role not in ('baseline', 'candidate'): + raise Refused('invalid-source-role') + image = env.get('ImageVersion', '') + if not re.fullmatch(r'\d{8}\.\d{4}\.\d+', image) or image != env.get('AUTOMATIC_COST_IMAGE'): + raise Refused('image-mismatch') + if not env.get('ImageOS', '').startswith('macos26'): + raise Refused('wrong-runner-os') + developer = env.get('DEVELOPER_DIR', '') + if developer != '/Applications/Xcode.app/Contents/Developer': + raise Refused('different-automatic-toolchain-path') + root = Path(command('git', 'rev-parse', '--show-toplevel')).resolve() + if root != Path.cwd().resolve() or command('git', 'rev-parse', 'HEAD') != candidate: + raise Refused('wrong-checkout') + if command('git', 'status', '--porcelain'): + raise Refused('dirty-checkout') + command('git', 'merge-base', '--is-ancestor', baseline, candidate) + if receipt.resolve().parent != Path(env.get('RUNNER_TEMP', '/nonexistent')).resolve(): + raise Refused('receipt-outside-runner-temp') + workflow = '.github/workflows/ci.yml' + before = automatic_job(command('git', 'show', f'{baseline}:{workflow}') + '\n') + after = automatic_job(command('git', 'show', f'{candidate}:{workflow}') + '\n') + if before != after: + raise Refused('automatic-workload-changed') + tools = { + 'developerDirectory': developer, + 'resolvedDeveloperDirectory': str(Path(developer).resolve()), + 'xcodeBuild': command('xcodebuild', '-version'), + 'swiftVersion': command('xcrun', 'swift', '--version'), + 'osVersion': command('sw_vers', '-productVersion'), + 'osBuild': command('sw_vers', '-buildVersion'), + 'architecture': command('uname', '-m'), + 'cpuCount': command('sysctl', '-n', 'hw.ncpu'), + 'memoryBytes': command('sysctl', '-n', 'hw.memsize'), + } + if any(len(value) > 4096 for value in tools.values()): + raise Refused('toolchain-output-cap') + fingerprint = hashlib.sha256(json.dumps(tools, sort_keys=True).encode()).hexdigest() + expected = env.get('AUTOMATIC_COST_TOOLCHAIN', '') + if role == 'candidate' and not re.fullmatch(r'[0-9a-f]{64}', expected): + raise Refused('missing-baseline-toolchain') + if expected and expected != fingerprint: + raise Refused('toolchain-mismatch') + source = baseline if role == 'baseline' else candidate + command('git', 'checkout', '--detach', source) + if command('git', 'rev-parse', 'HEAD') != source or command('git', 'status', '--porcelain'): + raise Refused('source-switch-unverified') + result = { + 'schema': 1, 'role': role, 'baseline': baseline, 'candidate': candidate, + 'measuredSource': source, 'imageVersion': image, 'imageOS': env['ImageOS'], + 'toolchain': tools, 'toolchainSHA256': fingerprint, + 'automaticJobSHA256': hashlib.sha256(before.encode()).hexdigest(), + 'measurement': 'Complete Actions run elapsed and summed non-skipped job runtime; no deductions', + } + receipt.write_text(json.dumps(result, indent=2) + '\n') + print('AUTOMATIC_COST_IDENTITY=' + json.dumps(result, sort_keys=True)) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--receipt', type=Path, required=True) + args = parser.parse_args() + try: + prepare(os.environ, args.receipt) + except Refused as error: + print('AUTOMATIC_COST_REFUSED=' + str(error)) + return 1 + except (OSError, subprocess.SubprocessError): + print('AUTOMATIC_COST_REFUSED=operational-or-command-failure') + return 1 + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/tests/manual/test_automatic_cost.py b/scripts/tests/manual/test_automatic_cost.py new file mode 100644 index 00000000..3f10ff02 --- /dev/null +++ b/scripts/tests/manual/test_automatic_cost.py @@ -0,0 +1,187 @@ +"""Manual-only identity/failure controls for the actual cost preparation script.""" + +import contextlib +import hashlib +import importlib.util +import io +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location('cost', ROOT / 'prepare-automatic-cost.py') +COST = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(COST) +WORKFLOW = (ROOT.parent / '.github/workflows/ci.yml').read_text() + + +class CostControls(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.folder = Path(self.temp.name) + self.repo = self.folder / 'repo' + self.repo.mkdir() + self.previous = Path.cwd() + os.chdir(self.repo) + self.addCleanup(os.chdir, self.previous) + self.real_command = COST.command + for args in (('init', '-q'), ('config', 'user.name', 'Fixture'), + ('config', 'user.email', 'fixture@example.invalid')): + self.git(*args) + path = self.repo / '.github/workflows/ci.yml' + path.parent.mkdir(parents=True) + # Same effective automatic job, before the manual guard was introduced. + original = COST.automatic_job(WORKFLOW) + modified = WORKFLOW[WORKFLOW.index(' swiftlint:\n'):WORKFLOW.index(' strict-concurrency:\n')] + self.before = WORKFLOW.replace(modified, original) + path.write_text(self.before) + self.git('add', '.') + self.git('commit', '-qm', 'baseline') + self.base = self.git('rev-parse', 'HEAD') + path.write_text(WORKFLOW) + (self.repo / 'candidate.txt').write_text('candidate source\n') + self.git('add', '.') + self.git('commit', '-qm', 'candidate') + self.head = self.git('rev-parse', 'HEAD') + self.env = dict( + GITHUB_ACTIONS='true', GITHUB_EVENT_NAME='workflow_dispatch', + GITHUB_REPOSITORY='hemsoft-dev/codexbar-ios', GITHUB_SHA=self.head, + AUTOMATIC_COST_MODE='automatic-cost', AUTOMATIC_COST_ROLE='baseline', + AUTOMATIC_COST_BASE=self.base, AUTOMATIC_COST_IMAGE='20261021.0445.8', + AUTOMATIC_COST_TOOLCHAIN='', ImageVersion='20261021.0445.8', ImageOS='macos26-arm64', + DEVELOPER_DIR='/Applications/Xcode.app/Contents/Developer', RUNNER_TEMP=str(self.folder), + ) + self.calls = [] + + def git(self, *args): + return subprocess.check_output(['git', *args], text=True, stderr=subprocess.PIPE).strip() + + def command(self, *args): + self.calls.append(args) + if args == ('xcodebuild', '-version'): + return 'Xcode fixture\nBuild version fixture' + if args == ('xcrun', 'swift', '--version'): + return 'Swift fixture\nTarget: arm64-apple-macosx' + return self.real_command(*args) + + def run_control(self): + with patch.object(COST, 'command', self.command), contextlib.redirect_stdout(io.StringIO()): + return COST.prepare(self.env, self.folder / 'receipt.json') + + def test_real_git_baseline_then_candidate_bind_receipts_and_toolchain(self): + baseline = self.run_control() + self.assertEqual(self.git('rev-parse', 'HEAD'), self.base) + self.assertEqual(baseline['measuredSource'], self.base) + self.git('checkout', '--detach', self.head) + self.env.update(AUTOMATIC_COST_ROLE='candidate', AUTOMATIC_COST_TOOLCHAIN=baseline['toolchainSHA256']) + candidate = self.run_control() + self.assertEqual(candidate['measuredSource'], self.head) + self.assertEqual(candidate['toolchainSHA256'], baseline['toolchainSHA256']) + self.assertEqual(candidate['automaticJobSHA256'], hashlib.sha256(COST.automatic_job(WORKFLOW).encode()).hexdigest()) + self.assertEqual(json.loads((self.folder / 'receipt.json').read_text()), candidate) + self.assertEqual(self.git('status', '--porcelain'), '') + + def test_wrong_image_refused_before_any_command_or_source_switch(self): + self.env['ImageVersion'] = '20261013.0424.1' + with self.assertRaisesRegex(COST.Refused, 'image-mismatch'): + self.run_control() + self.assertEqual(self.calls, []) + self.assertEqual(self.git('rev-parse', 'HEAD'), self.head) + + def test_wrong_context_and_invalid_sources_never_run_commands(self): + for key, value in (('GITHUB_EVENT_NAME', 'pull_request'), ('GITHUB_REPOSITORY', 'other/repo'), + ('AUTOMATIC_COST_ROLE', 'other'), ('AUTOMATIC_COST_BASE', 'main; echo unsafe')): + with self.subTest(key=key), patch.dict(self.env, {key: value}): + with self.assertRaises(COST.Refused): + self.run_control() + self.assertEqual(self.calls, []) + + def test_dirty_or_wrong_checkout_refused(self): + (self.repo / 'untracked').write_text('owned test') + with self.assertRaisesRegex(COST.Refused, 'dirty-checkout'): + self.run_control() + (self.repo / 'untracked').unlink() + self.git('checkout', '--detach', self.base) + with self.assertRaisesRegex(COST.Refused, 'wrong-checkout'): + self.run_control() + + def test_missing_or_different_toolchain_does_not_switch_source(self): + self.env['AUTOMATIC_COST_ROLE'] = 'candidate' + for value, expected in (('', 'missing-baseline-toolchain'), ('a' * 64, 'toolchain-mismatch')): + self.env['AUTOMATIC_COST_TOOLCHAIN'] = value + with self.assertRaisesRegex(COST.Refused, expected): + self.run_control() + self.assertEqual(self.git('rev-parse', 'HEAD'), self.head) + self.assertFalse((self.folder / 'receipt.json').exists()) + + def test_changed_automatic_command_fails_before_workload(self): + path = self.repo / '.github/workflows/ci.yml' + path.write_text(WORKFLOW.replace('--reporter github-actions-logging', '--reporter json')) + self.git('add', '.') + self.git('commit', '-qm', 'different automatic workload') + self.head = self.git('rev-parse', 'HEAD') + self.env['GITHUB_SHA'] = self.head + with self.assertRaisesRegex(COST.Refused, 'automatic-workload-changed'): + self.run_control() + self.assertNotIn(('xcodebuild', '-version'), self.calls) + + def test_normalizer_cannot_hide_an_extra_automatic_step(self): + injected = WORKFLOW.replace( + COST.MANUAL_GUARD, COST.MANUAL_GUARD + " - name: Hidden extra work\n run: echo extra\n\n", + ) + self.assertNotEqual(COST.automatic_job(injected), COST.automatic_job(WORKFLOW)) + unconditional = WORKFLOW.replace( + "if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode == 'automatic-cost' }}", + "if: ${{ always() }}", 1, + ) + self.assertNotEqual(COST.automatic_job(unconditional), COST.automatic_job(WORKFLOW)) + + def test_receipt_failure_cannot_be_a_pass(self): + with patch.object(Path, 'write_text', side_effect=PermissionError('fixture')): + with self.assertRaises(PermissionError): + self.run_control() + self.assertFalse((self.folder / 'receipt.json').exists()) + + def test_command_timeout_is_not_retried(self): + with patch.object(subprocess, 'run', side_effect=subprocess.TimeoutExpired('fixture', 10)) as run: + with self.assertRaises(subprocess.TimeoutExpired): + COST.command('git', 'status') + self.assertEqual(run.call_count, 1) + + def test_removing_only_cost_mode_restores_complete_prior_workflow(self): + restored = WORKFLOW.replace( + 'Full release gate, focused account-menu diagnosis, or automatic-cost control', + 'Full release gate or focused native account-menu diagnosis', + ).replace(' - automatic-cost\n', '', 1).replace( + 'Exact ancestor SHA for manual comparison modes', 'Exact ancestor SHA for account-menu comparison only', + ) + begin = restored.index('\n cost_source_role:') + end = restored.index('\npermissions:', begin) + restored = restored[:begin] + restored[end:] + restored = restored.replace( + " fetch-depth: ${{ inputs.ui_validation_mode == 'automatic-cost' && '0' || '1' }}\n", '', 1, + ).replace(COST.MANUAL_GUARD, '', 1).replace(" && inputs.ui_validation_mode != 'automatic-cost'", '') + self.assertEqual(hashlib.sha256(restored.encode()).hexdigest(), + 'a42e96c0bb86bf3d840a8067dabf696c137d5b8717e8664a3b1b59623e19df89') + + def test_workflow_keeps_normal_automatic_steps_and_manual_full_gates(self): + automatic = COST.automatic_job(WORKFLOW) + self.assertEqual(automatic.count(' - '), 3) + self.assertIn('timeout-minutes: 10', automatic) + self.assertIn('runs-on: macos-26', automatic) + self.assertIn("&& '0' || '1'", WORKFLOW) + self.assertEqual(WORKFLOW.count("if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'automatic-cost' }}"), 4) + self.assertIn("inputs.ui_validation_mode != 'account-menu-comparison' && inputs.ui_validation_mode != 'automatic-cost' && success()", WORKFLOW) + self.assertIn('shard: [original-1, original-2, original-3, supplemental, synthetic, native]', WORKFLOW) + self.assertIn('timeout-minutes: 90', WORKFLOW) + self.assertEqual(WORKFLOW.count('test_ci_policy.py'), 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/test_ci_policy.py b/scripts/tests/test_ci_policy.py index 1c870993..5d3164e6 100644 --- a/scripts/tests/test_ci_policy.py +++ b/scripts/tests/test_ci_policy.py @@ -58,7 +58,7 @@ def test_ci_supports_automatic_and_manual_runs(self) -> None: def test_native_jobs_are_dispatch_only_and_keep_their_workloads(self) -> None: for job_id in ("strict-concurrency", "ios-tests", "watch-tests", "smoke-tests"): - self.assertIn(" if: ${{ github.event_name == 'workflow_dispatch' }}\n", job_block(self.workflow, job_id)) + self.assertIn(" if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode != 'automatic-cost' }}\n", job_block(self.workflow, job_id)) lint_job = job_block(self.workflow, "swiftlint") smoke_job = job_block(self.workflow, "smoke-tests") self.assertIn("swiftlint lint", lint_job) @@ -80,7 +80,8 @@ def test_ui_job_runs_only_after_a_successful_manual_gate(self) -> None: ui_job = job_block(self.workflow, "ios-ui-families") self.assertIn( " if: ${{ github.event_name == 'workflow_dispatch' && " - "inputs.ui_validation_mode != 'account-menu-comparison' && success() }}\n", + "inputs.ui_validation_mode != 'account-menu-comparison' && " + "inputs.ui_validation_mode != 'automatic-cost' && success() }}\n", ui_job, ) for required_job in ( diff --git a/scripts/tests/test_ui_release_shards.py b/scripts/tests/test_ui_release_shards.py index 9c419bfb..ff410847 100644 --- a/scripts/tests/test_ui_release_shards.py +++ b/scripts/tests/test_ui_release_shards.py @@ -161,10 +161,11 @@ def test_manual_notifier_permission_and_full_workflow_scope(self): self.assertEqual(hashlib.sha256(script.encode()).hexdigest(), '4f299b77c869f5674c5fc63a9af4cd208a0ac17c39a43b8b88b8538763229ead') # Removing the approved step and its job-level grant must recover the - # entire prior workflow, including every automatic and native workload. + # pre-relay workflow plus the explicit manual cost-only mode. The separate + # manual cost controls recover the complete older workflow byte-for-byte. restored = workflow.replace(notifier, '').replace(' pull-requests: write\n', '', 1) self.assertEqual(hashlib.sha256(restored.encode()).hexdigest(), - 'ff422d0b6e584bbdde570c4ca15ab05d97341b53bb34ab3451bdee3543a49d63') + '6d5b13d1d4ecd0155cb77e7cea1a956b58fd3dd4a17a3c09253366b9a7f015b0') if __name__ == '__main__': From bf5e2578c1caf58d49bbd41dd28e6444e5fce69d Mon Sep 17 00:00:00 2001 From: HemSoft Date: Sun, 25 Oct 2026 22:57:23 -0400 Subject: [PATCH 10/11] Materialize both CI cost sources from an empty checkout (#517) --- .github/workflows/ci.yml | 4 +- UI-TESTING.md | 19 +++++--- scripts/prepare-automatic-cost.py | 24 ++++++++-- scripts/tests/manual/test_automatic_cost.py | 50 ++++++++++++++++++++- scripts/tests/test_ui_release_shards.py | 2 +- 5 files changed, 87 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ec550e5..ca478034 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,9 @@ jobs: AUTOMATIC_COST_BASE: ${{ inputs.comparison_base_sha }} AUTOMATIC_COST_IMAGE: ${{ inputs.cost_expected_image }} AUTOMATIC_COST_TOOLCHAIN: ${{ inputs.cost_expected_toolchain }} - run: python3 scripts/prepare-automatic-cost.py --receipt "$RUNNER_TEMP/automatic-cost.json" + run: | + cp scripts/prepare-automatic-cost.py "$RUNNER_TEMP/automatic-cost-controller.py" + python3 "$RUNNER_TEMP/automatic-cost-controller.py" --receipt "$RUNNER_TEMP/automatic-cost.json" - name: Run pinned SwiftLint policy run: >- diff --git a/UI-TESTING.md b/UI-TESTING.md index f863785c..3108d6fa 100644 --- a/UI-TESTING.md +++ b/UI-TESTING.md @@ -570,8 +570,8 @@ account-menu manual modes retain their native jobs, selectors and ceilings. Use one predeclared baseline/candidate pair. Both dispatches use the same frozen candidate workflow ref; `comparison_base_sha` is its exact ancestor/default-base -SHA. `cost_source_role=baseline` checks out that ancestor before the existing -lint/policy workload; `candidate` retains the exact dispatch SHA. Both first +SHA. `cost_source_role=baseline` materializes that ancestor before the existing +lint/policy workload; `candidate` materializes the exact dispatch SHA. Both first verify an explicit `cost_expected_image` against the assigned runner's actual `ImageVersion`. The candidate also requires `cost_expected_toolchain`, copied from the successful baseline's `AUTOMATIC_COST_IDENTITY` log receipt. The hash @@ -582,10 +582,17 @@ it never silently selects another image or retries. The manual guard verifies that effective automatic job commands/configuration match the baseline, rejects dirty or wrong source, and records exact source and tool identities. Only this manual mode fetches history for ancestor validation. -The normal automatic checkout remains depth one. Source selection happens after -checkout; the Actions run SHA identifies the workflow/controller, while the -receipt's `measuredSource` identifies the source actually linted. A baseline cost -run is not a current-head source-validation result. +The normal automatic checkout remains depth one. After the identical candidate +checkout, both controls retain the verified controller outside the workspace, +remove the same candidate tracked paths, verify the index and worktree are empty +(except `.git`), and materialize their selected revision from that state. Ignored +or untracked leftovers fail rather than being silently deleted or reused. This +avoids charging only the baseline for rewriting an already populated candidate +checkout. All removal/materialization work still counts. Receipts bind the +common controller/start and empty-state checks to each measured source. The +Actions run SHA identifies the workflow/controller, while `measuredSource` +identifies the source actually linted. A baseline cost run is not a current-head +source-validation result. Acceptance remains completed Actions **workflow elapsed time and summed non-skipped job runtime**: candidate must be no larger in either measure. Count diff --git a/scripts/prepare-automatic-cost.py b/scripts/prepare-automatic-cost.py index 2e1770ff..d5f373fe 100644 --- a/scripts/prepare-automatic-cost.py +++ b/scripts/prepare-automatic-cost.py @@ -18,7 +18,9 @@ AUTOMATIC_COST_BASE: ${{ inputs.comparison_base_sha }} AUTOMATIC_COST_IMAGE: ${{ inputs.cost_expected_image }} AUTOMATIC_COST_TOOLCHAIN: ${{ inputs.cost_expected_toolchain }} - run: python3 scripts/prepare-automatic-cost.py --receipt "$RUNNER_TEMP/automatic-cost.json" + run: | + cp scripts/prepare-automatic-cost.py "$RUNNER_TEMP/automatic-cost-controller.py" + python3 "$RUNNER_TEMP/automatic-cost-controller.py" --receipt "$RUNNER_TEMP/automatic-cost.json" """ @@ -69,13 +71,22 @@ def prepare(env, receipt): if developer != '/Applications/Xcode.app/Contents/Developer': raise Refused('different-automatic-toolchain-path') root = Path(command('git', 'rev-parse', '--show-toplevel')).resolve() - if root != Path.cwd().resolve() or command('git', 'rev-parse', 'HEAD') != candidate: + if (root != Path.cwd().resolve() or root != Path(env.get('GITHUB_WORKSPACE', '/nonexistent')).resolve() + or command('git', 'rev-parse', 'HEAD') != candidate): raise Refused('wrong-checkout') + if command('git', 'rev-parse', '--abbrev-ref', 'HEAD') != 'HEAD': + raise Refused('attached-checkout') if command('git', 'status', '--porcelain'): raise Refused('dirty-checkout') command('git', 'merge-base', '--is-ancestor', baseline, candidate) if receipt.resolve().parent != Path(env.get('RUNNER_TEMP', '/nonexistent')).resolve(): raise Refused('receipt-outside-runner-temp') + controller = Path(__file__).resolve() + if controller.parent != receipt.resolve().parent: + raise Refused('controller-not-retained') + if controller.read_text().strip() != command('git', 'show', f'{candidate}:scripts/prepare-automatic-cost.py'): + raise Refused('controller-source-mismatch') + controller_sha = hashlib.sha256(controller.read_bytes()).hexdigest() workflow = '.github/workflows/ci.yml' before = automatic_job(command('git', 'show', f'{baseline}:{workflow}') + '\n') after = automatic_job(command('git', 'show', f'{candidate}:{workflow}') + '\n') @@ -101,7 +112,11 @@ def prepare(env, receipt): if expected and expected != fingerprint: raise Refused('toolchain-mismatch') source = baseline if role == 'baseline' else candidate - command('git', 'checkout', '--detach', source) + removed = len(command('git', 'ls-files', '-z').rstrip('\0').split('\0')) + command('git', 'rm', '-r', '-f', '--quiet', '--', '.') + if command('git', 'ls-files') or any(path.name != '.git' for path in root.iterdir()): + raise Refused('materialization-start-not-empty') + command('git', 'checkout', '--force', '--detach', source) if command('git', 'rev-parse', 'HEAD') != source or command('git', 'status', '--porcelain'): raise Refused('source-switch-unverified') result = { @@ -109,6 +124,9 @@ def prepare(env, receipt): 'measuredSource': source, 'imageVersion': image, 'imageOS': env['ImageOS'], 'toolchain': tools, 'toolchainSHA256': fingerprint, 'automaticJobSHA256': hashlib.sha256(before.encode()).hexdigest(), + 'sourcePreparation': {'controllerSHA256': controller_sha, 'commonStartingHEAD': candidate, + 'removedCandidatePaths': removed, 'indexEntriesBeforeMaterialization': 0, + 'worktreeEntriesBeforeMaterialization': 0}, 'measurement': 'Complete Actions run elapsed and summed non-skipped job runtime; no deductions', } receipt.write_text(json.dumps(result, indent=2) + '\n') diff --git a/scripts/tests/manual/test_automatic_cost.py b/scripts/tests/manual/test_automatic_cost.py index 3f10ff02..323d1187 100644 --- a/scripts/tests/manual/test_automatic_cost.py +++ b/scripts/tests/manual/test_automatic_cost.py @@ -46,24 +46,37 @@ def setUp(self): self.base = self.git('rev-parse', 'HEAD') path.write_text(WORKFLOW) (self.repo / 'candidate.txt').write_text('candidate source\n') + controller = self.repo / 'scripts/prepare-automatic-cost.py' + controller.parent.mkdir(parents=True) + controller.write_bytes(Path(COST.__file__).read_bytes()) + self.controller = self.folder / 'controller.py' + self.controller.write_bytes(controller.read_bytes()) self.git('add', '.') self.git('commit', '-qm', 'candidate') self.head = self.git('rev-parse', 'HEAD') + self.git('checkout', '--detach', self.head) self.env = dict( GITHUB_ACTIONS='true', GITHUB_EVENT_NAME='workflow_dispatch', GITHUB_REPOSITORY='hemsoft-dev/codexbar-ios', GITHUB_SHA=self.head, + GITHUB_WORKSPACE=str(self.repo), AUTOMATIC_COST_MODE='automatic-cost', AUTOMATIC_COST_ROLE='baseline', AUTOMATIC_COST_BASE=self.base, AUTOMATIC_COST_IMAGE='20261021.0445.8', AUTOMATIC_COST_TOOLCHAIN='', ImageVersion='20261021.0445.8', ImageOS='macos26-arm64', DEVELOPER_DIR='/Applications/Xcode.app/Contents/Developer', RUNNER_TEMP=str(self.folder), ) self.calls = [] + self.materialization_states = [] def git(self, *args): return subprocess.check_output(['git', *args], text=True, stderr=subprocess.PIPE).strip() def command(self, *args): self.calls.append(args) + if args[:2] == ('git', 'checkout'): + self.materialization_states.append({ + 'index': self.git('ls-files'), + 'entries': sorted(p.name for p in self.repo.iterdir() if p.name != '.git'), + }) if args == ('xcodebuild', '-version'): return 'Xcode fixture\nBuild version fixture' if args == ('xcrun', 'swift', '--version'): @@ -71,7 +84,8 @@ def command(self, *args): return self.real_command(*args) def run_control(self): - with patch.object(COST, 'command', self.command), contextlib.redirect_stdout(io.StringIO()): + with patch.object(COST, '__file__', str(self.controller)), \ + patch.object(COST, 'command', self.command), contextlib.redirect_stdout(io.StringIO()): return COST.prepare(self.env, self.folder / 'receipt.json') def test_real_git_baseline_then_candidate_bind_receipts_and_toolchain(self): @@ -87,6 +101,40 @@ def test_real_git_baseline_then_candidate_bind_receipts_and_toolchain(self): self.assertEqual(json.loads((self.folder / 'receipt.json').read_text()), candidate) self.assertEqual(self.git('status', '--porcelain'), '') + def test_both_roles_materialize_from_the_same_empty_state(self): + baseline = self.run_control() + self.git('checkout', '--force', '--detach', self.head) + self.env.update(AUTOMATIC_COST_ROLE='candidate', AUTOMATIC_COST_TOOLCHAIN=baseline['toolchainSHA256']) + self.run_control() + self.assertEqual(len(self.materialization_states), 2) + self.assertEqual(self.materialization_states, [{'index': '', 'entries': []}] * 2) + self.assertTrue((self.repo / 'candidate.txt').is_file()) + self.assertEqual(self.git('status', '--porcelain'), '') + + def test_attached_checkout_is_never_cleared(self): + self.git('checkout', '-b', 'owned-branch') + with self.assertRaisesRegex(COST.Refused, 'attached-checkout'): + self.run_control() + self.assertTrue((self.repo / 'candidate.txt').exists()) + self.assertEqual(self.git('status', '--porcelain'), '') + + def test_modified_retained_controller_is_refused_before_clearing(self): + self.controller.write_text('different controller') + with self.assertRaisesRegex(COST.Refused, 'controller-source-mismatch'): + self.run_control() + self.assertTrue((self.repo / 'candidate.txt').exists()) + + def test_ignored_leftover_cannot_qualify_empty_materialization(self): + (self.repo / '.git/info/exclude').write_text('ignored-cache/\n') + ignored = self.repo / 'ignored-cache' + ignored.mkdir() + (ignored / 'retained').write_text('owned fixture') + with self.assertRaisesRegex(COST.Refused, 'materialization-start-not-empty'): + self.run_control() + self.assertTrue((ignored / 'retained').exists()) + self.assertEqual(self.materialization_states, []) + self.assertFalse((self.folder / 'receipt.json').exists()) + def test_wrong_image_refused_before_any_command_or_source_switch(self): self.env['ImageVersion'] = '20261013.0424.1' with self.assertRaisesRegex(COST.Refused, 'image-mismatch'): diff --git a/scripts/tests/test_ui_release_shards.py b/scripts/tests/test_ui_release_shards.py index ff410847..12e25675 100644 --- a/scripts/tests/test_ui_release_shards.py +++ b/scripts/tests/test_ui_release_shards.py @@ -165,7 +165,7 @@ def test_manual_notifier_permission_and_full_workflow_scope(self): # manual cost controls recover the complete older workflow byte-for-byte. restored = workflow.replace(notifier, '').replace(' pull-requests: write\n', '', 1) self.assertEqual(hashlib.sha256(restored.encode()).hexdigest(), - '6d5b13d1d4ecd0155cb77e7cea1a956b58fd3dd4a17a3c09253366b9a7f015b0') + 'c867d3c06a0db87cf2042c2558fbf3bd345b334948b9d659c5f3fcf9e2ab5294') if __name__ == '__main__': From 8befdb59cca29c8b73cd296a5a0a9697c5e5c53b Mon Sep 17 00:00:00 2001 From: HemSoft Date: Sun, 25 Oct 2026 23:03:38 -0400 Subject: [PATCH 11/11] Pin manual cost checkouts to the immutable dispatch SHA (#517) --- .github/workflows/ci.yml | 1 + UI-TESTING.md | 4 +++- scripts/prepare-automatic-cost.py | 1 + scripts/tests/manual/test_automatic_cost.py | 9 ++++++++- scripts/tests/test_ui_release_shards.py | 2 +- 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca478034..333bb612 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,7 @@ jobs: with: persist-credentials: false fetch-depth: ${{ inputs.ui_validation_mode == 'automatic-cost' && '0' || '1' }} + ref: ${{ inputs.ui_validation_mode == 'automatic-cost' && github.sha || '' }} - name: Verify manual automatic-cost identity if: ${{ github.event_name == 'workflow_dispatch' && inputs.ui_validation_mode == 'automatic-cost' }} diff --git a/UI-TESTING.md b/UI-TESTING.md index 3108d6fa..684ed064 100644 --- a/UI-TESTING.md +++ b/UI-TESTING.md @@ -582,7 +582,9 @@ it never silently selects another image or retries. The manual guard verifies that effective automatic job commands/configuration match the baseline, rejects dirty or wrong source, and records exact source and tool identities. Only this manual mode fetches history for ancestor validation. -The normal automatic checkout remains depth one. After the identical candidate +The normal automatic checkout remains depth one with its original default ref. +Only cost mode explicitly selects `github.sha`, so a branch dispatch creates a +verified detached checkout of the immutable controller commit. After the identical candidate checkout, both controls retain the verified controller outside the workspace, remove the same candidate tracked paths, verify the index and worktree are empty (except `.git`), and materialize their selected revision from that state. Ignored diff --git a/scripts/prepare-automatic-cost.py b/scripts/prepare-automatic-cost.py index d5f373fe..a6228d83 100644 --- a/scripts/prepare-automatic-cost.py +++ b/scripts/prepare-automatic-cost.py @@ -45,6 +45,7 @@ def automatic_job(workflow): r"^ fetch-depth: \$\{\{ inputs.ui_validation_mode == 'automatic-cost' && '0' \|\| '1' \}\}\n", '', block, flags=re.M, ) + block = block.replace(" ref: ${{ inputs.ui_validation_mode == 'automatic-cost' && github.sha || '' }}\n", '', 1) block = block.replace(MANUAL_GUARD, '', 1) return block diff --git a/scripts/tests/manual/test_automatic_cost.py b/scripts/tests/manual/test_automatic_cost.py index 323d1187..a384628a 100644 --- a/scripts/tests/manual/test_automatic_cost.py +++ b/scripts/tests/manual/test_automatic_cost.py @@ -214,10 +214,17 @@ def test_removing_only_cost_mode_restores_complete_prior_workflow(self): restored = restored[:begin] + restored[end:] restored = restored.replace( " fetch-depth: ${{ inputs.ui_validation_mode == 'automatic-cost' && '0' || '1' }}\n", '', 1, - ).replace(COST.MANUAL_GUARD, '', 1).replace(" && inputs.ui_validation_mode != 'automatic-cost'", '') + ).replace(" ref: ${{ inputs.ui_validation_mode == 'automatic-cost' && github.sha || '' }}\n", '', 1).replace(COST.MANUAL_GUARD, '', 1).replace(" && inputs.ui_validation_mode != 'automatic-cost'", '') self.assertEqual(hashlib.sha256(restored.encode()).hexdigest(), 'a42e96c0bb86bf3d840a8067dabf696c137d5b8717e8664a3b1b59623e19df89') + def test_branch_dispatch_pins_only_cost_checkout_to_immutable_sha(self): + checkout = WORKFLOW.split(' steps:\n', 1)[1].split(COST.MANUAL_GUARD, 1)[0] + declaration = " ref: ${{ inputs.ui_validation_mode == 'automatic-cost' && github.sha || '' }}\n" + self.assertIn(declaration, checkout) + self.assertEqual(WORKFLOW.count(declaration), 1) + self.assertNotIn(' ref:', COST.automatic_job(WORKFLOW)) + def test_workflow_keeps_normal_automatic_steps_and_manual_full_gates(self): automatic = COST.automatic_job(WORKFLOW) self.assertEqual(automatic.count(' - '), 3) diff --git a/scripts/tests/test_ui_release_shards.py b/scripts/tests/test_ui_release_shards.py index 12e25675..759a927d 100644 --- a/scripts/tests/test_ui_release_shards.py +++ b/scripts/tests/test_ui_release_shards.py @@ -165,7 +165,7 @@ def test_manual_notifier_permission_and_full_workflow_scope(self): # manual cost controls recover the complete older workflow byte-for-byte. restored = workflow.replace(notifier, '').replace(' pull-requests: write\n', '', 1) self.assertEqual(hashlib.sha256(restored.encode()).hexdigest(), - 'c867d3c06a0db87cf2042c2558fbf3bd345b334948b9d659c5f3fcf9e2ab5294') + '0c10b64a94fbcdd2cf9eb120f42f31bcb3e3d3b91ea7e5de8a9c3b848188c357') if __name__ == '__main__':