ci: parse xcresult bundles in one place

.claude/rules/ci.md

@@ -50,9 +50,10 @@ The required check names are in `AGENTS.md` (CI). Reading a red check: `.claude/
 
 ## Summaries (`scripts/ci/summarize-xcresult.sh`)
 
+- Runs `scripts/lib/xcresult-cases.py summary`. `verify.sh` reads its verdicts through the same parser, and `scripts/lib/test_xcresult_cases.py` tests it in Repo Hygiene.
 - Reads `.xcresult` bundles and `swift test` xunit files, and skips a missing input.
-- Writes build errors with file and line, per-bundle counts and minutes, up to 50 failed cases and the 20 slowest to the step summary.
-- Exits 1 when a bundle or a `swift test` run executed nothing, because `xcodebuild` and `swift test` both call that success.
+- Writes build errors with file and 1-based line, runner failures ("System Failures"), per-bundle counts and minutes, up to 50 failed cases and the 20 slowest to the step summary.
+- Exits 1 when a bundle or a `swift test` run executed nothing, because `xcodebuild` and `swift test` both call that success. A runner failure is not an executed test.
 - For `X.xml` it also reads `X-swift-testing.xml`.
 - It never fails on a failed case; the test step already did.
 
@@ -73,7 +74,7 @@ The required check names are in `AGENTS.md` (CI). Reading a red check: `.claude/
 - Steps, in order:
   - actionlint, which also shellchecks every inline `run:` block;
   - `shellcheck --severity=warning` on every `*.sh` under `scripts` and `.claude/skills`;
-  - the tooling tests: `scripts/ci/test_*.py`, `scripts/test_localization.py`, `.github/scripts/test_update_registry.py` and two shell tests;
+  - the tooling tests: `scripts/ci/test_*.py`, `scripts/lib/test_*.py`, `scripts/test_localization.py`, `.github/scripts/test_update_registry.py` and two shell tests;
   - `check-lib-sourcing.py`, `check-plugin-manifest.py`, `check-deployment-target.sh`, `check-ios-shared-isolation.py`;
   - `localization.py plugins` and `plugins --target ios`;
   - `check-string-catalogs.py`, `check-license-inventory.py`, `check-entitlements.py`, `check-test-plans.py`;

.claude/skills/fix-issue/references/verification-apple.md

@@ -33,7 +33,7 @@ Read this when a verdict, a test name or a red CI check needs explaining. The co
 
 ## Verdicts
 
-- A test verdict comes from the result bundle or the xunit file, never the console banner.
+- A test verdict comes from the result bundle or the xunit file, never the console banner. `scripts/lib/xcresult-cases.py` reads both, as it does for the CI summary.
 - The result-bundle rules, first match wins:
   1. nothing executed and the bundle holds build errors: FAIL, "the test build failed";
   2. nothing executed and `xcodebuild` exited nonzero: FAIL;
@@ -99,7 +99,7 @@ Read this when a verdict, a test name or a red CI check needs explaining. The co
 ## Reading CI
 
 - Artifacts exist only for a failed job and are kept 7 days. `macos-tests-<attempt>` holds every `.xcresult` plus `app-diagnostics` (TablePro crash reports and 2 hours of its unified log); `ios-tests-<attempt>` holds `iOS.xcresult`.
-- The step summary lists build errors, failed cases and the slowest cases (`.claude/rules/ci.md`, Summaries). A case that passed only on the UI plan's retry shows as a `Passed only on retry` warning annotation while the job stays green.
+- The step summary lists build errors, runner failures, failed cases and the slowest cases (`.claude/rules/ci.md`, Summaries). A case that passed only on the UI plan's retry shows as a `Passed only on retry` warning annotation while the job stays green.
 - `xcbeautify` can hide a compiler error behind "exit code 65". Read the check-run annotations, which also work while the run is going: `gh api repos/TableProApp/TablePro/check-runs/<job-id>/annotations`. Or open `Build.xcresult` from the artifact.
 - UI evidence: `gh run download <run-id> -n macos-tests-<attempt>`, then `xcrun xcresulttool get test-results activities --test-id <id> --path UI.xcresult` and `xcrun xcresulttool export attachments --path UI.xcresult --output-path <dir>`. The `element-tree-<n>` attachment shows which window held focus; the recording and the Synthesized Event plists show where each event landed.
 - A long silent stretch in the build log is a target's emit-module job, not a stall.

.claude/skills/fix-issue/scripts/lib/platform-apple.sh

@@ -39,7 +39,7 @@ diagnose_environment() {
         found=0
     fi
     if grep -q 'Unable to open base configuration reference file' "$log" 2> /dev/null; then
-        note "cause: Secrets.xcconfig is missing from this checkout. A fresh worktree needs it symlinked from the main checkout."
+        note "cause: the generated project names an xcconfig this checkout lacks. Rerun scripts/generate-project.sh (verify.sh generate)."
         found=0
     fi
     if grep -q 'The test runner hung before establishing connection' "$log" 2> /dev/null; then
@@ -207,127 +207,32 @@ test_regex() {
 
 # ---------------------------------------------------------------------------- test results
 
-# Every test case in a result bundle as "<target>/<suite>/<case><TAB><result><TAB>retried?". A runner
-# that crashed or never started is reported as a case under "System Failures"; it is not a test.
-xcresult_cases() {
-    xcrun xcresulttool get test-results tests --path "$1" --compact 2> /dev/null | jq -r '
-        [.. | objects | select((.nodeType // "") | endswith("test bundle"))][] | .name as $target
-        | .children[]? | (.nodeType == "Test Suite" and .name == "System Failures") as $system
-        | .. | objects | select(.nodeType == "Test Case")
-        | [ "\($target)/\(.nodeIdentifier // .name)", (if $system then "System Failure" else .result end),
-            (if .result == "Passed" and any(.children[]?; .nodeType == "Repetition" and .result == "Failed")
-             then "retried" else "" end) ] | @tsv'
-}
-
-# Every test case in swift test's xunit files as "<target>.<suite>/<case><TAB><result>".
-xunit_cases() {
-    local file
-    for file in "$@"; do
-        [ -f "$file" ] || continue
-        xmllint --format "$file" 2> /dev/null | awk '
-            function attr(line, name,    v) {
-                if (!match(line, " " name "=\"[^\"]*\"")) return ""
-                v = substr(line, RSTART + length(name) + 3, RLENGTH - length(name) - 4)
-                gsub(/&quot;/, "\"", v); gsub(/&apos;/, "\047", v)
-                gsub(/&lt;/, "<", v); gsub(/&gt;/, ">", v); gsub(/&amp;/, "\\&", v)
-                return v
-            }
-            function flush() { if (open) print key "\t" state; open = 0 }
-            /<testcase[ >]/ {
-                flush()
-                # A nested Swift Testing suite comes as Target.Outer.Inner; its filters use Outer/Inner.
-                cls = attr($0, "classname"); dot = index(cls, ".")
-                if (dot) { rest = substr(cls, dot + 1); gsub(/\./, "/", rest); cls = substr(cls, 1, dot) rest }
-                key = cls "/" attr($0, "name"); state = "Passed"; open = 1
-                if ($0 ~ /\/>[[:space:]]*$/) flush()
-                next
-            }
-            open && /<skipped/ { state = "Skipped" }
-            open && /<(failure|error)[ >\/]/ { state = "Failed" }
-            /<\/testcase>/ { flush() }
-            END { flush() }'
-    done
-}
-
-# Tallies a case list against the run's filters ("<label><TAB><regex>[<TAB>...]"). Prints the notes,
-# then a last line "<executed> <failed> <filters that ran nothing> <runner failures>".
-tally_cases() {
-    local cases="$1" filters="$2" sep="$3"
-    awk -F'\t' -v sep="$sep" '
-        FILENAME == ARGV[1] { if ($0 != "") { n++; label[n] = $1; re[n] = $2 }; next }
-        $0 == "" { next }
-        {
-            target = $1
-            cut = index(target, sep)
-            if (cut) target = substr(target, 1, cut - 1)
-            if ($2 == "System Failure") { runner++; next }
-            if (!(target in seen)) { seen[target] = 1; order[++targets] = target }
-            if ($2 == "Skipped") { skipped++; tskipped[target]++; next }
-            executed++; tran[target]++
-            if ($2 == "Passed") passed++
-            else if ($2 == "Failed") failed++
-            else expected++
-            if ($3 == "retried") retried[++nretried] = $1
-            for (i = 1; i <= n; i++) if ($1 ~ re[i]) hits[i]++
-        }
-        END {
-            line = "tests: " executed + 0 " executed, " passed + 0 " passed, " failed + 0 " failed, " skipped + 0 " skipped"
-            if (expected) line = line ", " expected " expected failures"
-            if (runner) line = line ", " runner " runner failures"
-            print line
-            if (targets) {
-                line = "targets:"
-                for (i = 1; i <= targets; i++) {
-                    t = order[i]
-                    line = line (i > 1 ? "," : "") " " t " " tran[t] + 0
-                    if (tskipped[t]) line = line " (" tskipped[t] " skipped)"
-                }
-                print line
-            }
-            zero = 0
-            for (i = 1; i <= n; i++) if (!hits[i]) { zero++; print "ran nothing: " label[i] }
-            if (nretried) {
-                print "passed only on retry, a flake to fix:"
-                for (i = 1; i <= nretried; i++) print "  " retried[i]
-            }
-            print executed + 0, failed + 0, zero, runner + 0
-        }' "$filters" "$cases"
-}
+# Beside this script, not under --root, so the parser always matches the tally format read here.
+XCRESULT_CASES="$SCRIPT_DIR/../../../../scripts/lib/xcresult-cases.py"
 
-# Sets RUN_EXECUTED, RUN_FAILED, RUN_ZERO and RUN_SYSTEM from a case list, and notes the tallies.
+# Sets RUN_EXECUTED, RUN_FAILED, RUN_ZERO and RUN_SYSTEM from the cases in a result bundle or xunit
+# file, counted against the run's filters ("<label><TAB><regex>[<TAB>...]"), and notes the tallies.
 apply_tally() {
-    local out counts
-    out="$(tally_cases "$@")"
-    counts="$(tail -1 <<< "$out")"
-    read -r RUN_EXECUTED RUN_FAILED RUN_ZERO RUN_SYSTEM <<< "$counts"
+    local filters="$1" input="$2" out
+    out="$(python3 "$XCRESULT_CASES" tally --filters "$filters" "$input")" \
+        || out="cause: scripts/lib/xcresult-cases.py failed, so no case was read."$'\n'"0 0 0 0"
+    read -r RUN_EXECUTED RUN_FAILED RUN_ZERO RUN_SYSTEM <<< "$(tail -1 <<< "$out")"
     note "$(sed '$d' <<< "$out")"
 }
 
 # Build errors with their file and line, which the result bundle keeps even when the log interleaves them.
 report_build_errors() {
     local errors
-    errors="$(xcrun xcresulttool get build-results --path "$1" --compact 2> /dev/null | jq -r '
-        .errors[]? | select(.message | startswith("Testing cancelled") | not)
-        | if .sourceURL then
-            (.sourceURL | capture("^file://(?<path>[^#]*)#.*StartingLineNumber=(?<line>[0-9]+)")) as $at
-            | "\($at.path | split("/") | last):\($at.line | tonumber + 1): \(.message)"
-          else .message end' | head -12)"
+    errors="$(python3 "$XCRESULT_CASES" errors "$1" | head -12)"
     [ -n "$errors" ] || return 1
     note "errors:"
     note "$(sed 's/^/  /' <<< "$errors")"
 }
 
 # The result bundle is the verdict. The log only explains a run that left no tests in it.
 report_xcresult() {
-    local bundle="$1" code="$2" filters="$3" log="$4" cases
-    cases="${bundle%.xcresult}.cases"
-    xcresult_cases "$bundle" > "$cases"
-    apply_tally "$cases" "$filters" /
-    rm -f "$cases"
-    if [ "$RUN_FAILED" -gt 0 ] || [ "$RUN_SYSTEM" -gt 0 ]; then
-        note "$(xcrun xcresulttool get test-results summary --path "$bundle" --compact 2> /dev/null | jq -r '
-            .testFailures[:10][] | "  \(.targetName)/\(.testIdentifierString): \(.failureText | split("\n")[0])"')"
-    fi
+    local bundle="$1" code="$2" filters="$3" log="$4"
+    apply_tally "$filters" "$bundle"
     if [ "$RUN_EXECUTED" -eq 0 ] && [ "$RUN_FAILED" -eq 0 ] && [ "$RUN_SYSTEM" -eq 0 ] && report_build_errors "$bundle"; then
         RUN_STATUS=FAIL
         note "cause: the test build failed, so no test ran."
@@ -353,12 +258,9 @@ report_xcresult() {
 }
 
 report_xunit() {
-    local log="$1" code="$2" filters="$3" xunit="${1%.log}.xml" cases
+    local log="$1" code="$2" filters="$3" xunit="${1%.log}.xml"
     note "results: $xunit"
-    cases="${log%.log}.cases"
-    xunit_cases "$xunit" "${xunit%.xml}-swift-testing.xml" > "$cases"
-    apply_tally "$cases" "$filters" .
-    rm -f "$cases"
+    apply_tally "$filters" "$xunit"
     if [ "$code" -ne 0 ] || [ "$RUN_FAILED" -gt 0 ]; then
         RUN_STATUS=FAIL
         note "$(report_errors "$log")"
@@ -533,6 +435,7 @@ platform_step() {
             [ "$STEP" = "uitest" ] && plan=UI other=test
             require_tool jq
             require_tool git
+            require_tool python3
             [ -f "$REPO_ROOT/TestPlans/$plan.xctestplan" ] || { echo "no TestPlans/$plan.xctestplan in $REPO_ROOT" >&2; exit 3; }
             TEST_TARGETS="$(test_targets_table)"
             plan_tests="" swift_tests="" packages=""
@@ -578,7 +481,6 @@ platform_step() {
             fi
             for package in $packages; do
                 require_tool swift
-                require_tool xmllint
                 # The first run writes the step's own log; each later one gets its own beside it.
                 if [ -z "$run_log" ]; then
                     run_log="$log"
@@ -643,7 +545,7 @@ platform_step() {
             STEP_DETAIL="$1${2:+ --filter $2}"
             setup_toolchain
             require_tool swift
-            require_tool xmllint
+            require_tool python3
             log="$(new_log "package-$1")"
             : > "${log%.log}.filters"
             [ $# -ge 2 ] && printf '%s\t%s\n' "$2" "$2" > "${log%.log}.filters"
@@ -657,7 +559,7 @@ platform_step() {
             STEP_DETAIL="TableProMobileTests: $*"
             setup_toolchain
             require_tool xcodebuild
-            require_tool jq
+            require_tool python3
             wait_for_free_toolchain
             name="$(printf '%s' "$1" | tr -c 'A-Za-z0-9_-' '_')"
             log="$(new_log "ios-$name")"

.github/workflows/repo-hygiene.yml

@@ -47,6 +47,7 @@ jobs:
         run: |
           set -euo pipefail
           python3 -m unittest discover -s scripts/ci -p 'test_*.py'
+          python3 -m unittest discover -s scripts/lib -p 'test_*.py'
           python3 -m unittest discover -s scripts -p 'test_localization.py'
           python3 .github/scripts/test_update_registry.py
           bash scripts/ci/test_create_dmg_args.sh

scripts/ci/summarize-xcresult.sh

@@ -1,149 +1,6 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Writes build errors, test counts and the slowest cases to $GITHUB_STEP_SUMMARY, or stdout.
 # Usage: summarize-xcresult.sh <bundle.xcresult | swift-test-xunit.xml>...
-#
-# Exits 1 when a test bundle, or a swift test run, executed no test: xcodebuild and swift test both
-# report that as success. A missing input is skipped, since its step did not run. For X.xml it also
-# reads X-swift-testing.xml, the file swift test writes for Swift Testing beside the XCTest one.
-# A case that passed only on a retry becomes a warning annotation.
 
-exec python3 - "$@" <<'PY'
-import json
-import os
-import subprocess
-import sys
-import urllib.parse
-import xml.etree.ElementTree as ET
-from pathlib import Path
-
-summary = open(os.environ.get("GITHUB_STEP_SUMMARY") or "/dev/stdout", "a", encoding="utf-8")
-status = 0
-
-
-def emit(text=""):
-    print(text, file=summary)
-
-
-def xcresulttool(bundle, *arguments):
-    result = subprocess.run(
-        ["xcrun", "xcresulttool", "get", *arguments, "--path", bundle, "--format", "json"],
-        capture_output=True, text=True)
-    return json.loads(result.stdout) if result.returncode == 0 else None
-
-
-def heading(name, cases, failed):
-    minutes = sum(case[2] for case in cases) / 60
-    emit(f"### {name}: {len(cases)} cases, {minutes:.1f} min"
-         + (f", {len(failed)} not passing" if failed else "") + "\n")
-    if failed:
-        for label in failed[:50]:
-            emit(f"- failed: `{label}`")
-        emit()
-
-
-def slowest(cases):
-    emit("<details><summary>Slowest 20 cases</summary>\n\n| case | suite | seconds |\n| --- | --- | ---: |")
-    for suite, name, seconds in sorted(cases, key=lambda case: -case[2])[:20]:
-        emit(f"| {name} | {suite} | {seconds:.1f} |")
-    emit("\n</details>\n")
-
-
-def build_errors(bundle, name):
-    build = xcresulttool(bundle, "build-results") or {}
-    errors = build.get("errors") or []
-    if not errors:
-        return
-    emit(f"### {name}: {len(errors)} build error{'' if len(errors) == 1 else 's'}\n")
-    for issue in errors[:50]:
-        url = urllib.parse.urlparse(issue.get("sourceURL") or "")
-        line = urllib.parse.parse_qs(url.fragment).get("StartingLineNumber", ["?"])[0]
-        location = f"`{url.path}:{line}` " if url.path else ""
-        emit(f"- {location}{issue.get('message', '')}")
-    emit()
-
-
-def test_cases(node, suite, found):
-    kind = node.get("nodeType")
-    if kind == "Test Case":
-        found.append((suite, node))
-        return
-    for child in node.get("children") or []:
-        test_cases(child, node.get("name", "") if kind == "Test Suite" else suite, found)
-
-
-def xcresult(bundle, name):
-    global status
-    build_errors(bundle, name)
-    content = xcresulttool(bundle, "content-availability") or {}
-    if not content.get("hasTestResults"):
-        return
-    tree = xcresulttool(bundle, "test-results", "tests")
-    if tree is None:
-        return
-    bundles = []
-    for plan in tree.get("testNodes") or []:
-        children = [child for child in plan.get("children") or [] if child.get("nodeType", "").endswith("test bundle")]
-        bundles.extend(children or [plan])
-    everything, failed, retried = [], [], []
-    for target in bundles:
-        found = []
-        test_cases(target, "", found)
-        executed = [(suite, case) for suite, case in found if case.get("result") != "Skipped"]
-        if not executed:
-            emit(f"### {name}: {target.get('name', '?')} executed no test\n")
-            print(f"::error::{name}: {target.get('name', '?')} executed no test")
-            status = 1
-        for suite, case in executed:
-            label = f"{target.get('name', '?')}/{suite + '/' if suite else ''}{case.get('name', '')}"
-            everything.append((suite, case.get("name", ""), case.get("durationInSeconds") or 0.0))
-            if case.get("result") not in ("Passed", "Expected Failure"):
-                failed.append(label)
-            elif any(child.get("nodeType") == "Repetition" and child.get("result") == "Failed"
-                     for child in case.get("children") or []):
-                retried.append(label)
-    if not everything:
-        return
-    heading(name, everything, failed)
-    if retried:
-        emit("Passed only on retry, a flake to fix:\n")
-        for label in retried:
-            emit(f"- `{label}`")
-            print(f"::warning title=Passed only on retry::{label}")
-        emit()
-    slowest(everything)
-
-
-def xunit(path, name):
-    global status
-    sources = [path, path.with_name(f"{path.stem}-swift-testing.xml")]
-    present = [source for source in sources if source.is_file()]
-    if not present:
-        return
-    everything, failed = [], []
-    for source in present:
-        for case in ET.parse(source).getroot().iter("testcase"):
-            if case.find("skipped") is not None:
-                continue
-            label = f"{case.get('classname', '')}/{case.get('name', '')}"
-            everything.append((case.get("classname", ""), case.get("name", ""), float(case.get("time") or 0)))
-            if case.find("failure") is not None or case.find("error") is not None:
-                failed.append(label)
-    if not everything:
-        emit(f"### {name}: no test executed\n")
-        print(f"::error::{name}: swift test executed no test")
-        status = 1
-        return
-    heading(name, everything, failed)
-    slowest(everything)
-
-
-for argument in sys.argv[1:]:
-    path = Path(argument)
-    if path.suffix == ".xml":
-        xunit(path, path.stem)
-    elif path.is_dir():
-        xcresult(argument, path.stem)
-sys.exit(status)
-PY
+exec python3 "$(dirname "$0")/../lib/xcresult-cases.py" summary "$@"