From b9a627f9681ff05b01b5b1c323c4b3ffaec6ef67 Mon Sep 17 00:00:00 2001 From: Martin Kourim Date: Wed, 12 Aug 2026 22:34:04 +0200 Subject: [PATCH] fix(ci): make AI failure analysis reliable The nightly AI failure-analysis step produced no analysis: the prompts instructed the agent to run compound shell snippets (pipes, command substitution) for counting test results, but the CI allowlist permits only simple command prefixes. Every attempt was denied and the agent burned all 40 turns on workarounds without writing the analysis file. Counting and failed/broken test enumeration: - Add standalone scripts/count_test_results.py, allowlist it in both reusable workflows and point the regression and upgrade prompts at it. It groups allure result files per test (by historyId) and takes the newest real result of each test as authoritative. The per-file grep arithmetic used before could produce nonsense (Total lower than Passed, negative Skipped): pytest evaluates skipif markers before plain skip markers, so skipif-decorated tests are registered by the --skipall pass under their own skip reason and get a second result file from the real run, and the --skipall early return in conftest skips dynamic marker additions, so MARKEXPR can select different test sets in the two passes (the conftest issues will be addressed separately). Verified on a real results dir: grep arithmetic gave Total 815/Skipped -9, the script gives the true Total 948/ Skipped 124. - Unreadable result files produce partial counts, an explicit warning and a non-zero exit code. Missing and empty directories are reported and skipped. Tests that have only the skipall registration and no real result are called out - they indicate an interrupted testrun. - The failed/broken enumeration is capped at 40 lines with a '+N more' tail, so a mass failure cannot flood the agent tool output (which gets truncated around 30kB). Guarantee at least a partial analysis: - Double max_turns to 80 and instruct the agent (CI constraints prompt) to write a first draft of failure_analysis.md as soon as counts and failure list are known, so a cut-off run still leaves output on disk. - Add timeout-minutes to the analyze step as insurance against action hangs - a job-level timeout would cancel artifact upload and mail. Make failed analysis debuggable, never fail the testrun job: - Copy the Claude execution log into run_workdir (created if the testing step died early) and include it in the testrun-files artifact; warn when no execution log is available. - Mark the analysis as possibly incomplete in the mail, step summary and log when the analyze step outcome is not success (a partial early draft would otherwise read as a complete analysis). - Escalate the missing-analysis annotation from warning to error and include the analyze step outcome in it and in the mail sentinel. All annotations are non-failing; the analyze step keeps continue-on-error, so artifact uploads and the failure mail run regardless of how the analysis ends. --- .github/workflows/regression_reusable.yaml | 40 ++- .github/workflows/upgrade_reusable.yaml | 40 ++- agent_docs/ci_analysis_prompt.md | 1 + agent_docs/failure_analysis_prompt.md | 19 +- agent_docs/upgrade_failure_analysis_prompt.md | 15 +- scripts/count_test_results.py | 257 ++++++++++++++++++ 6 files changed, 342 insertions(+), 30 deletions(-) create mode 100755 scripts/count_test_results.py diff --git a/.github/workflows/regression_reusable.yaml b/.github/workflows/regression_reusable.yaml index df772cafa..59931f7c4 100644 --- a/.github/workflows/regression_reusable.yaml +++ b/.github/workflows/regression_reusable.yaml @@ -149,20 +149,37 @@ jobs: id: analyze-failures if: (success() || failure()) && steps.testing-step.outcome != 'success' && env.HAS_OAUTH_TOKEN == 'true' continue-on-error: true + # The action has its own internal timeout, this is insurance against + # the action itself hanging - without it a stuck step would run into + # the job timeout, and a job timeout is a cancellation that skips the + # artifact upload and mail steps. + timeout-minutes: 30 uses: anthropics/claude-code-base-action@beta with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} model: claude-sonnet-5 - max_turns: "40" - allowed_tools: "Read,Write,Glob,Grep,Bash(ls:*),Bash(wc:*),Bash(head:*),Bash(tail:*),Bash(grep:*),Bash(find:*),Bash(cut:*),Bash(sort:*),Bash(uniq:*),Bash(awk:*),Bash(jq:*),Bash(sqlite3:*)" + max_turns: "80" + allowed_tools: "Read,Write,Glob,Grep,Bash(scripts/count_test_results.py:*),Bash(ls:*),Bash(wc:*),Bash(head:*),Bash(tail:*),Bash(grep:*),Bash(find:*),Bash(cut:*),Bash(sort:*),Bash(uniq:*),Bash(awk:*),Bash(jq:*),Bash(sqlite3:*)" prompt: ${{ env.ANALYSIS_PROMPT }} - name: Read failure analysis into env id: read-analysis if: (success() || failure()) && steps.testing-step.outcome != 'success' && env.HAS_OAUTH_TOKEN == 'true' + env: + EXECUTION_FILE: ${{ steps.analyze-failures.outputs.execution_file }} + ANALYZE_OUTCOME: ${{ steps.analyze-failures.outcome }} run: | + # When the analyze step did not finish cleanly (e.g. ran out of + # turns mid-write), the analysis file may be a partial draft - mark + # it as such everywhere it is surfaced. + partial_note="" + if [ "$ANALYZE_OUTCOME" != "success" ]; then + partial_note="**Note: the analyze step did not finish cleanly (outcome: ${ANALYZE_OUTCOME}) - the analysis below may be incomplete.**" + fi + if [ -s run_workdir/failure_analysis.md ]; then { echo 'FAILURE_ANALYSIS<<__EOF_ANALYSIS42__' + if [ -n "$partial_note" ]; then echo "$partial_note"; echo; fi cat run_workdir/failure_analysis.md # ensure delimiter is on its own line even if the file lacks a trailing newline printf '\n__EOF_ANALYSIS42__\n' @@ -171,14 +188,28 @@ jobs: # Surface the analysis in the workflow run UI itself: full content # in a foldable log group, and a copy in the run summary so it's # reachable without downloading the testrun-files artifact. + if [ -n "$partial_note" ]; then + echo "::warning::Analyze step outcome: ${ANALYZE_OUTCOME} - failure_analysis.md may be partial." + { echo "$partial_note"; echo; } >> "$GITHUB_STEP_SUMMARY" + fi echo "::group::Preliminary failure analysis" cat run_workdir/failure_analysis.md echo echo "::endgroup::" cat run_workdir/failure_analysis.md >> "$GITHUB_STEP_SUMMARY" else - echo "FAILURE_ANALYSIS=(no analysis produced)" >> "$GITHUB_ENV" - echo "::warning::No run_workdir/failure_analysis.md produced by the analyze step." + echo "FAILURE_ANALYSIS=(no analysis produced; analyze step outcome: ${ANALYZE_OUTCOME})" >> "$GITHUB_ENV" + echo "::error::No run_workdir/failure_analysis.md produced by the analyze step (outcome: ${ANALYZE_OUTCOME})." + fi + + # Keep the Claude execution log with the testrun files so failed or + # incomplete analysis runs can be debugged from the artifact alone. + # The testing step can die before it creates run_workdir. + if [ -f "${EXECUTION_FILE:-}" ]; then + mkdir -p run_workdir + cp "$EXECUTION_FILE" run_workdir/claude-execution-output.json + else + echo "::warning::No Claude execution log available (execution_file: '${EXECUTION_FILE:-}')." fi - name: Report test results if: (success() || failure()) && inputs.testrun_name @@ -225,6 +256,7 @@ jobs: run_workdir/requirements_coverage.json run_workdir/monitor.log run_workdir/failure_analysis.md + run_workdir/claude-execution-output.json - name: ↟ Upload CLI coverage uses: actions/upload-artifact@v7 if: success() || failure() diff --git a/.github/workflows/upgrade_reusable.yaml b/.github/workflows/upgrade_reusable.yaml index fc47eaea1..38d03fc3a 100644 --- a/.github/workflows/upgrade_reusable.yaml +++ b/.github/workflows/upgrade_reusable.yaml @@ -97,20 +97,37 @@ jobs: id: analyze-failures if: (success() || failure()) && steps.testing-step.outcome != 'success' && env.HAS_OAUTH_TOKEN == 'true' continue-on-error: true + # The action has its own internal timeout, this is insurance against + # the action itself hanging - without it a stuck step would run into + # the job timeout, and a job timeout is a cancellation that skips the + # artifact upload and mail steps. + timeout-minutes: 30 uses: anthropics/claude-code-base-action@beta with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} model: claude-sonnet-5 - max_turns: "40" - allowed_tools: "Read,Write,Glob,Grep,Bash(ls:*),Bash(wc:*),Bash(head:*),Bash(tail:*),Bash(grep:*),Bash(find:*),Bash(cut:*),Bash(sort:*),Bash(uniq:*),Bash(awk:*),Bash(jq:*),Bash(sqlite3:*)" + max_turns: "80" + allowed_tools: "Read,Write,Glob,Grep,Bash(scripts/count_test_results.py:*),Bash(ls:*),Bash(wc:*),Bash(head:*),Bash(tail:*),Bash(grep:*),Bash(find:*),Bash(cut:*),Bash(sort:*),Bash(uniq:*),Bash(awk:*),Bash(jq:*),Bash(sqlite3:*)" prompt: ${{ env.ANALYSIS_PROMPT }} - name: Read failure analysis into env id: read-analysis if: (success() || failure()) && steps.testing-step.outcome != 'success' && env.HAS_OAUTH_TOKEN == 'true' + env: + EXECUTION_FILE: ${{ steps.analyze-failures.outputs.execution_file }} + ANALYZE_OUTCOME: ${{ steps.analyze-failures.outcome }} run: | + # When the analyze step did not finish cleanly (e.g. ran out of + # turns mid-write), the analysis file may be a partial draft - mark + # it as such everywhere it is surfaced. + partial_note="" + if [ "$ANALYZE_OUTCOME" != "success" ]; then + partial_note="**Note: the analyze step did not finish cleanly (outcome: ${ANALYZE_OUTCOME}) - the analysis below may be incomplete.**" + fi + if [ -s run_workdir/failure_analysis.md ]; then { echo 'FAILURE_ANALYSIS<<__EOF_ANALYSIS42__' + if [ -n "$partial_note" ]; then echo "$partial_note"; echo; fi cat run_workdir/failure_analysis.md # ensure delimiter is on its own line even if the file lacks a trailing newline printf '\n__EOF_ANALYSIS42__\n' @@ -119,14 +136,28 @@ jobs: # Surface the analysis in the workflow run UI itself: full content # in a foldable log group, and a copy in the run summary so it's # reachable without downloading the testrun-files artifact. + if [ -n "$partial_note" ]; then + echo "::warning::Analyze step outcome: ${ANALYZE_OUTCOME} - failure_analysis.md may be partial." + { echo "$partial_note"; echo; } >> "$GITHUB_STEP_SUMMARY" + fi echo "::group::Preliminary failure analysis" cat run_workdir/failure_analysis.md echo echo "::endgroup::" cat run_workdir/failure_analysis.md >> "$GITHUB_STEP_SUMMARY" else - echo "FAILURE_ANALYSIS=(no analysis produced)" >> "$GITHUB_ENV" - echo "::warning::No run_workdir/failure_analysis.md produced by the analyze step." + echo "FAILURE_ANALYSIS=(no analysis produced; analyze step outcome: ${ANALYZE_OUTCOME})" >> "$GITHUB_ENV" + echo "::error::No run_workdir/failure_analysis.md produced by the analyze step (outcome: ${ANALYZE_OUTCOME})." + fi + + # Keep the Claude execution log with the testrun files so failed or + # incomplete analysis runs can be debugged from the artifact alone. + # The testing step can die before it creates run_workdir. + if [ -f "${EXECUTION_FILE:-}" ]; then + mkdir -p run_workdir + cp "$EXECUTION_FILE" run_workdir/claude-execution-output.json + else + echo "::warning::No Claude execution log available (execution_file: '${EXECUTION_FILE:-}')." fi - name: ↟ Upload testing artifacts on failure uses: actions/upload-artifact@v7 @@ -172,6 +203,7 @@ jobs: run_workdir/scheduling.log run_workdir/errors_all.log run_workdir/failure_analysis.md + run_workdir/claude-execution-output.json - name: ↟ Upload CLI coverage uses: actions/upload-artifact@v7 if: success() || failure() diff --git a/agent_docs/ci_analysis_prompt.md b/agent_docs/ci_analysis_prompt.md index 53ee8586d..2114b224b 100644 --- a/agent_docs/ci_analysis_prompt.md +++ b/agent_docs/ci_analysis_prompt.md @@ -3,6 +3,7 @@ Additional constraints for automated CI runs: - Output a single markdown file at: `{RUN_DIR}/failure_analysis.md`. +- Write a first rough draft of that file as soon as you have the test counts and the list of failed/broken tests, then refine it as you dig deeper. You have a limited number of turns and may be cut off at any point - a partial analysis must already be on disk when that happens. - Keep it under ~300 lines. Start now. diff --git a/agent_docs/failure_analysis_prompt.md b/agent_docs/failure_analysis_prompt.md index 8c5acfb4c..5df817617 100644 --- a/agent_docs/failure_analysis_prompt.md +++ b/agent_docs/failure_analysis_prompt.md @@ -16,26 +16,17 @@ Inputs available under `{RUN_DIR}/` (use only what exists): Counting tests (IMPORTANT): -Run following code snippet to get the correct counts: +Run following command to get the correct counts and the list of failed/broken tests: ```sh -passed=$(grep -l '"status": "passed"' {RUN_DIR}/allure-results/*result.json | wc -l) -failed=$(grep -l '"status": "failed"' {RUN_DIR}/allure-results/*result.json | wc -l) -broken=$(grep -l '"status": "broken"' {RUN_DIR}/allure-results/*result.json | wc -l) -total=$(grep -l '"message": "Skipped: collected, not run"' {RUN_DIR}/allure-results/*result.json | wc -l) -if [ "$total" -gt 0 ]; then - skipped=$(( total - passed - failed - broken )) -else - skipped=$(grep -l '"status": "skipped"' {RUN_DIR}/allure-results/*result.json | wc -l) - total=$(( passed + failed + broken + skipped )) -fi -echo "Total: $total, Passed: $passed, Failed: $failed, Broken: $broken, Skipped: $skipped" +scripts/count_test_results.py {RUN_DIR}/allure-results ``` +Run it exactly as shown, as a single plain command - do not wrap it in pipes, command substitution or other compound shell constructs, those are rejected by the CI command allowlist. + Steps: -1. Enumerate failed/broken tests: - `grep -E '"status": "(failed|broken)"' {RUN_DIR}/allure-results/*result.json | cut -c1-200`. +1. Enumerate failed/broken tests from the `count_test_results.py` output above. (`broken` = pytest error in setup/teardown; `failed` = assertion failure.) 2. Group failures by likely root cause (same exception class + message head, same node crash, same infra symptom). Treat one node crash that flunks many tests as a single group. 3. For each group: list affected tests (truncate to ~10 with a "+N more" tail), give the most informative 1–3 lines of error context, and classify as one of `node-bug | test-bug | infra-flake | env-issue | unknown` with a short justification. diff --git a/agent_docs/upgrade_failure_analysis_prompt.md b/agent_docs/upgrade_failure_analysis_prompt.md index d888c3c0c..e0338e548 100644 --- a/agent_docs/upgrade_failure_analysis_prompt.md +++ b/agent_docs/upgrade_failure_analysis_prompt.md @@ -21,15 +21,14 @@ Inputs available under `{RUN_DIR}/` (use only what exists): Steps: -1. Enumerate failed/broken tests per step: - `for s in step1 step2 step3; do grep -l -E '"status": "(failed|broken)"' {RUN_DIR}/allure-results-$s/*result.json 2>/dev/null | sed "s|^|$s: |"; done` +1. Get per-step counts and failed/broken test lists (with `statusDetails.message` heads): + `scripts/count_test_results.py {RUN_DIR}/allure-results-step1 {RUN_DIR}/allure-results-step2 {RUN_DIR}/allure-results-step3` + Run it exactly as shown, as a single plain command - do not wrap it in pipes, command substitution or other compound shell constructs, those are rejected by the CI command allowlist. (`broken` = pytest error in setup/teardown; `failed` = assertion failure.) -2. For each failing test, extract `statusDetails.message` head with: - `grep -E '"status": "(failed|broken)"' {RUN_DIR}/allure-results-step*/*result.json | cut -c1-200`. -3. Group failures by likely root cause (same exception class + message head, same node crash, same infra symptom). **Note which step(s) each group hits** — a failure that appears only in step2 or step3 is much more interesting than one that already fails in step1. Treat one node crash that flunks many tests as a single group. -4. For each group: list affected tests (truncate to ~10 with a "+N more" tail), give the most informative 1–3 lines of error context, mark the step(s) affected, and classify as one of `node-bug | test-bug | infra-flake | env-issue | upgrade-regression | unknown` with a short justification. Use `upgrade-regression` when a test passes in step1 but fails in step2 or step3 — that is the signal this workflow exists to catch. -5. Skim `{RUN_DIR}/errors_all.log` and `{RUN_DIR}/scheduling.log` for anything corroborating (node crash on restart, hard-fork failure, supervisord errors, OOM, repeated tracebacks). When failures look cluster-management related (dead cluster instances, tests stuck waiting for resources), query the `overview` view of the affected step's status database. -6. If a whole step is missing its `allure-results-stepN/` dir, that step likely failed before pytest ran — call this out explicitly and check `errors_all.log` / the workflow log group output for the cause (commonly a `start-cluster` / `supervisord` / hard-fork failure). +2. Group failures by likely root cause (same exception class + message head, same node crash, same infra symptom). **Note which step(s) each group hits** — a failure that appears only in step2 or step3 is much more interesting than one that already fails in step1. Treat one node crash that flunks many tests as a single group. +3. For each group: list affected tests (truncate to ~10 with a "+N more" tail), give the most informative 1–3 lines of error context, mark the step(s) affected, and classify as one of `node-bug | test-bug | infra-flake | env-issue | upgrade-regression | unknown` with a short justification. Use `upgrade-regression` when a test passes in step1 but fails in step2 or step3 — that is the signal this workflow exists to catch. +4. Skim `{RUN_DIR}/errors_all.log` and `{RUN_DIR}/scheduling.log` for anything corroborating (node crash on restart, hard-fork failure, supervisord errors, OOM, repeated tracebacks). When failures look cluster-management related (dead cluster instances, tests stuck waiting for resources), query the `overview` view of the affected step's status database. +5. If a whole step is missing its `allure-results-stepN/` dir, that step likely failed before pytest ran — call this out explicitly and check `errors_all.log` / the workflow log group output for the cause (commonly a `start-cluster` / `supervisord` / hard-fork failure). Known patterns: diff --git a/scripts/count_test_results.py b/scripts/count_test_results.py new file mode 100755 index 000000000..75d406af7 --- /dev/null +++ b/scripts/count_test_results.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +"""Summarize allure test results for preliminary failure analysis. + +Usage: + scripts/count_test_results.py [...] + +For each directory, prints per-test counts by status and one line per +failed/broken test (capped, with a '+N more' tail). Missing and empty +directories are reported and skipped. + +Result files are grouped per test (by Allure `historyId`, which identifies a +test together with its parameters), so tests with multiple result files are +counted once. Multiple files per test are normal: the initial `--skipall` +pass of `runner/run_tests.sh` registers every collected test with a +"Skipped: collected, not run" result file, and tests skipped via `skipif` +markers are recorded under their own skip reason even during that pass. The +newest non-registration result of a test wins. + +This is a standalone script so that AI failure analysis in CI can get the +counts with a single allowlisted command - the CI allowlist permits only +simple command prefixes, so compound shell snippets (pipes, command +substitution) are rejected there. +""" + +import json +import pathlib +import sys +import typing as tp + +# Cap the failed/broken enumeration so a mass-failure run cannot flood the +# consumer - AI agent tool output gets truncated around 30kB. +ENUM_LIMIT = 40 +LINE_WIDTH = 200 + +SKIPALL_MSG = "Skipped: collected, not run" + + +def _plural(count: int, word: str) -> str: + """Return the count together with the word, pluralized when needed. + + Args: + count: The number of items. + word: The singular form of the word. + + Returns: + E.g. "1 test" or "5 tests". + """ + return f"{count} {word}{'' if count == 1 else 's'}" + + +class _TestResult(tp.NamedTuple): + """Authoritative result of a single test.""" + + is_real: bool + start: float + status: str + name: str + msg_head: str + + +def _parse_result_file(fpath: pathlib.Path) -> tuple[str, bool, _TestResult]: + """Parse one allure result file. + + Args: + fpath: Path to a `*-result.json` file. + + Returns: + Tuple of (grouping key, whether the file had a historyId, test result). + + Raises: + OSError: When the file cannot be read. + ValueError: When the file content is not valid JSON. + TypeError: When the file content is not a JSON object. + AttributeError: When a nested field has an unexpected type. + """ + rec = json.loads(fpath.read_text(encoding="utf-8")) + if not isinstance(rec, dict): + err = "not a JSON object" + raise TypeError(err) + + key = rec.get("historyId") + has_history_id = bool(key) and isinstance(key, str) + if not has_history_id: + params = sorted( + (str(p.get("name") or ""), str(p.get("value") or "")) + for p in rec.get("parameters") or [] + ) + key = f"{rec.get('fullName') or rec.get('name') or fpath.name}|{params}" + + message = (rec.get("statusDetails") or {}).get("message") or "" + start = rec.get("start") + result = _TestResult( + is_real=message != SKIPALL_MSG, + start=start if isinstance(start, (int, float)) else 0, + status=str(rec.get("status") or "unknown"), + name=str(rec.get("name") or rec.get("fullName") or "?"), + msg_head=message.splitlines()[0] if message else "", + ) + return str(key), has_history_id, result + + +def _group_records( + result_files: list[pathlib.Path], +) -> tuple[dict[str, _TestResult], int, int]: + """Group result files per test, the newest real result of a test winning. + + Args: + result_files: Allure `*-result.json` files. + + Returns: + Tuple of (test key to authoritative result mapping, number of files + without a historyId, number of unreadable files). + """ + best: dict[str, _TestResult] = {} + no_history_id = 0 + read_errors = 0 + for fpath in result_files: + try: + key, has_history_id, result = _parse_result_file(fpath) + except (OSError, ValueError, TypeError, AttributeError) as exc: + # Printed to stdout so the error stays next to its directory's + # summary when the two streams are merged (as in CI). + print(f"Error: cannot read '{fpath}': {exc}") + read_errors += 1 + continue + if not has_history_id: + no_history_id += 1 + prev = best.get(key) + if prev is None or (result.is_real, result.start) > (prev.is_real, prev.start): + best[key] = result + return best, no_history_id, read_errors + + +def _print_counts(best: dict[str, _TestResult]) -> None: + """Print per-test counts by status. + + Args: + best: Test key to authoritative result mapping. + """ + counts: dict[str, int] = {} + for result in best.values(): + counts[result.status] = counts.get(result.status, 0) + 1 + + total = len(best) + passed = counts.get("passed", 0) + failed = counts.get("failed", 0) + broken = counts.get("broken", 0) + skipped = counts.get("skipped", 0) + print( + f"Total: {total}, Passed: {passed}, Failed: {failed}, Broken: {broken}, Skipped: {skipped}" + ) + other = total - passed - failed - broken - skipped + if other: + print(f"Other statuses: {other}") + + never_run = sum(1 for r in best.values() if not r.is_real) + if never_run: + print( + f"Note: {_plural(never_run, 'test')} registered by the initial skip pass " + "only, with no real result - the testrun was likely interrupted before " + "they could run" + ) + + +def _print_failures(best: dict[str, _TestResult]) -> None: + """Print one line per failed/broken test, sorted by test name and capped. + + Args: + best: Test key to authoritative result mapping. + """ + failures = sorted( + (r.name, f"{r.status}: {r.name}: {r.msg_head}"[:LINE_WIDTH]) + for r in best.values() + if r.status in ("failed", "broken") + ) + if not failures: + return + + print("-- failed/broken tests --") + for _, line in failures[:ENUM_LIMIT]: + print(line) + if len(failures) > ENUM_LIMIT: + print( + f"... +{len(failures) - ENUM_LIMIT} more (see the result JSON files for the full list)" + ) + + +def summarize_dir(results_dir: pathlib.Path) -> int: + """Print a test results summary for one allure results directory. + + Args: + results_dir: Directory with `*-result.json` allure files. + + Returns: + 0 on success (incl. missing or empty directory, which is reported as + an informational message), 1 when the directory or some of its result + files could not be read and the counts may therefore be incomplete. + """ + print(f"== {results_dir} ==") + + if not results_dir.is_dir(): + print(f"Directory not found: {results_dir}") + return 0 + + # `iterdir` instead of `glob`, as `glob` swallows permission errors and an + # unlistable directory would be misreported as having no results. + try: + result_files = sorted(f for f in results_dir.iterdir() if f.name.endswith("-result.json")) + except OSError as exc: + print(f"Error: cannot list '{results_dir}': {exc}") + return 1 + if not result_files: + print( + f"No *-result.json files found in {results_dir} - " + "the testrun likely did not produce results" + ) + return 0 + + best, no_history_id, read_errors = _group_records(result_files) + _print_counts(best) + _print_failures(best) + + if no_history_id: + print( + f"Note: {_plural(no_history_id, 'result file')} without historyId - " + "test grouping is approximate" + ) + if read_errors: + print( + f"Warning: {_plural(read_errors, 'result file')} could not be read - " + "counts may be incomplete" + ) + return 1 + return 0 + + +def main() -> int: + """Summarize each directory given on the command line. + + Returns: + The highest per-directory return code, or 2 on usage error. + """ + if len(sys.argv) < 2: + print( + f"Usage: {sys.argv[0]} [...]", + file=sys.stderr, + ) + return 2 + + exit_rc = 0 + for dir_arg in sys.argv[1:]: + exit_rc = max(exit_rc, summarize_dir(pathlib.Path(dir_arg))) + return exit_rc + + +if __name__ == "__main__": + sys.exit(main())