From d6a9403348ad693d9bc258c7b63f0aa7a5a63b9f Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sat, 17 Oct 2026 23:06:38 +0700 Subject: [PATCH] ci: parse xcresult bundles in one place --- .claude/rules/ci.md | 7 +- .../references/verification-apple.md | 4 +- .../fix-issue/scripts/lib/platform-apple.sh | 132 +------ .github/workflows/repo-hygiene.yml | 1 + scripts/ci/summarize-xcresult.sh | 145 +------- scripts/lib/test_xcresult_cases.py | 331 ++++++++++++++++++ scripts/lib/xcresult-cases.py | 270 ++++++++++++++ 7 files changed, 626 insertions(+), 264 deletions(-) create mode 100644 scripts/lib/test_xcresult_cases.py create mode 100755 scripts/lib/xcresult-cases.py diff --git a/.claude/rules/ci.md b/.claude/rules/ci.md index ae849a2d6d..f272d1feaf 100644 --- a/.claude/rules/ci.md +++ b/.claude/rules/ci.md @@ -50,9 +50,10 @@ The required check names are in `AGENTS.md` (CI). Reading a red check: `.claude/ ## Summaries (`scripts/ci/summarize-xcresult.sh`) +- Runs `scripts/lib/xcresult-cases.py summary`. `verify.sh` reads its verdicts through the same parser, and `scripts/lib/test_xcresult_cases.py` tests it in Repo Hygiene. - Reads `.xcresult` bundles and `swift test` xunit files, and skips a missing input. -- Writes build errors with file and line, per-bundle counts and minutes, up to 50 failed cases and the 20 slowest to the step summary. -- Exits 1 when a bundle or a `swift test` run executed nothing, because `xcodebuild` and `swift test` both call that success. +- Writes build errors with file and 1-based line, runner failures ("System Failures"), per-bundle counts and minutes, up to 50 failed cases and the 20 slowest to the step summary. +- Exits 1 when a bundle or a `swift test` run executed nothing, because `xcodebuild` and `swift test` both call that success. A runner failure is not an executed test. - For `X.xml` it also reads `X-swift-testing.xml`. - It never fails on a failed case; the test step already did. @@ -73,7 +74,7 @@ The required check names are in `AGENTS.md` (CI). Reading a red check: `.claude/ - Steps, in order: - actionlint, which also shellchecks every inline `run:` block; - `shellcheck --severity=warning` on every `*.sh` under `scripts` and `.claude/skills`; - - the tooling tests: `scripts/ci/test_*.py`, `scripts/test_localization.py`, `.github/scripts/test_update_registry.py` and two shell tests; + - the tooling tests: `scripts/ci/test_*.py`, `scripts/lib/test_*.py`, `scripts/test_localization.py`, `.github/scripts/test_update_registry.py` and two shell tests; - `check-lib-sourcing.py`, `check-plugin-manifest.py`, `check-deployment-target.sh`, `check-ios-shared-isolation.py`; - `localization.py plugins` and `plugins --target ios`; - `check-string-catalogs.py`, `check-license-inventory.py`, `check-entitlements.py`, `check-test-plans.py`; diff --git a/.claude/skills/fix-issue/references/verification-apple.md b/.claude/skills/fix-issue/references/verification-apple.md index e63300c7be..c97869cd76 100644 --- a/.claude/skills/fix-issue/references/verification-apple.md +++ b/.claude/skills/fix-issue/references/verification-apple.md @@ -33,7 +33,7 @@ Read this when a verdict, a test name or a red CI check needs explaining. The co ## Verdicts -- A test verdict comes from the result bundle or the xunit file, never the console banner. +- A test verdict comes from the result bundle or the xunit file, never the console banner. `scripts/lib/xcresult-cases.py` reads both, as it does for the CI summary. - The result-bundle rules, first match wins: 1. nothing executed and the bundle holds build errors: FAIL, "the test build failed"; 2. nothing executed and `xcodebuild` exited nonzero: FAIL; @@ -99,7 +99,7 @@ Read this when a verdict, a test name or a red CI check needs explaining. The co ## Reading CI - Artifacts exist only for a failed job and are kept 7 days. `macos-tests-` holds every `.xcresult` plus `app-diagnostics` (TablePro crash reports and 2 hours of its unified log); `ios-tests-` holds `iOS.xcresult`. -- The step summary lists build errors, failed cases and the slowest cases (`.claude/rules/ci.md`, Summaries). A case that passed only on the UI plan's retry shows as a `Passed only on retry` warning annotation while the job stays green. +- The step summary lists build errors, runner failures, failed cases and the slowest cases (`.claude/rules/ci.md`, Summaries). A case that passed only on the UI plan's retry shows as a `Passed only on retry` warning annotation while the job stays green. - `xcbeautify` can hide a compiler error behind "exit code 65". Read the check-run annotations, which also work while the run is going: `gh api repos/TableProApp/TablePro/check-runs//annotations`. Or open `Build.xcresult` from the artifact. - UI evidence: `gh run download -n macos-tests-`, then `xcrun xcresulttool get test-results activities --test-id --path UI.xcresult` and `xcrun xcresulttool export attachments --path UI.xcresult --output-path `. The `element-tree-` attachment shows which window held focus; the recording and the Synthesized Event plists show where each event landed. - A long silent stretch in the build log is a target's emit-module job, not a stall. diff --git a/.claude/skills/fix-issue/scripts/lib/platform-apple.sh b/.claude/skills/fix-issue/scripts/lib/platform-apple.sh index ab6c93d2a9..581b1e4160 100644 --- a/.claude/skills/fix-issue/scripts/lib/platform-apple.sh +++ b/.claude/skills/fix-issue/scripts/lib/platform-apple.sh @@ -39,7 +39,7 @@ diagnose_environment() { found=0 fi if grep -q 'Unable to open base configuration reference file' "$log" 2> /dev/null; then - note "cause: Secrets.xcconfig is missing from this checkout. A fresh worktree needs it symlinked from the main checkout." + note "cause: the generated project names an xcconfig this checkout lacks. Rerun scripts/generate-project.sh (verify.sh generate)." found=0 fi if grep -q 'The test runner hung before establishing connection' "$log" 2> /dev/null; then @@ -207,111 +207,23 @@ test_regex() { # ---------------------------------------------------------------------------- test results -# Every test case in a result bundle as "//retried?". A runner -# that crashed or never started is reported as a case under "System Failures"; it is not a test. -xcresult_cases() { - xcrun xcresulttool get test-results tests --path "$1" --compact 2> /dev/null | jq -r ' - [.. | objects | select((.nodeType // "") | endswith("test bundle"))][] | .name as $target - | .children[]? | (.nodeType == "Test Suite" and .name == "System Failures") as $system - | .. | objects | select(.nodeType == "Test Case") - | [ "\($target)/\(.nodeIdentifier // .name)", (if $system then "System Failure" else .result end), - (if .result == "Passed" and any(.children[]?; .nodeType == "Repetition" and .result == "Failed") - then "retried" else "" end) ] | @tsv' -} - -# Every test case in swift test's xunit files as "./". -xunit_cases() { - local file - for file in "$@"; do - [ -f "$file" ] || continue - xmllint --format "$file" 2> /dev/null | awk ' - function attr(line, name, v) { - if (!match(line, " " name "=\"[^\"]*\"")) return "" - v = substr(line, RSTART + length(name) + 3, RLENGTH - length(name) - 4) - gsub(/"/, "\"", v); gsub(/'/, "\047", v) - gsub(/</, "<", v); gsub(/>/, ">", v); gsub(/&/, "\\&", v) - return v - } - function flush() { if (open) print key "\t" state; open = 0 } - /]/ { - flush() - # A nested Swift Testing suite comes as Target.Outer.Inner; its filters use Outer/Inner. - cls = attr($0, "classname"); dot = index(cls, ".") - if (dot) { rest = substr(cls, dot + 1); gsub(/\./, "/", rest); cls = substr(cls, 1, dot) rest } - key = cls "/" attr($0, "name"); state = "Passed"; open = 1 - if ($0 ~ /\/>[[:space:]]*$/) flush() - next - } - open && /\/]/ { state = "Failed" } - /<\/testcase>/ { flush() } - END { flush() }' - done -} - -# Tallies a case list against the run's filters ("