diff --git a/.github/workflows/regression_reusable.yaml b/.github/workflows/regression_reusable.yaml index df772cafa..59931f7c4 100644 --- a/.github/workflows/regression_reusable.yaml +++ b/.github/workflows/regression_reusable.yaml @@ -149,20 +149,37 @@ jobs: id: analyze-failures if: (success() || failure()) && steps.testing-step.outcome != 'success' && env.HAS_OAUTH_TOKEN == 'true' continue-on-error: true + # The action has its own internal timeout, this is insurance against + # the action itself hanging - without it a stuck step would run into + # the job timeout, and a job timeout is a cancellation that skips the + # artifact upload and mail steps. + timeout-minutes: 30 uses: anthropics/claude-code-base-action@beta with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} model: claude-sonnet-5 - max_turns: "40" - allowed_tools: "Read,Write,Glob,Grep,Bash(ls:*),Bash(wc:*),Bash(head:*),Bash(tail:*),Bash(grep:*),Bash(find:*),Bash(cut:*),Bash(sort:*),Bash(uniq:*),Bash(awk:*),Bash(jq:*),Bash(sqlite3:*)" + max_turns: "80" + allowed_tools: "Read,Write,Glob,Grep,Bash(scripts/count_test_results.py:*),Bash(ls:*),Bash(wc:*),Bash(head:*),Bash(tail:*),Bash(grep:*),Bash(find:*),Bash(cut:*),Bash(sort:*),Bash(uniq:*),Bash(awk:*),Bash(jq:*),Bash(sqlite3:*)" prompt: ${{ env.ANALYSIS_PROMPT }} - name: Read failure analysis into env id: read-analysis if: (success() || failure()) && steps.testing-step.outcome != 'success' && env.HAS_OAUTH_TOKEN == 'true' + env: + EXECUTION_FILE: ${{ steps.analyze-failures.outputs.execution_file }} + ANALYZE_OUTCOME: ${{ steps.analyze-failures.outcome }} run: | + # When the analyze step did not finish cleanly (e.g. ran out of + # turns mid-write), the analysis file may be a partial draft - mark + # it as such everywhere it is surfaced. + partial_note="" + if [ "$ANALYZE_OUTCOME" != "success" ]; then + partial_note="**Note: the analyze step did not finish cleanly (outcome: ${ANALYZE_OUTCOME}) - the analysis below may be incomplete.**" + fi + if [ -s run_workdir/failure_analysis.md ]; then { echo 'FAILURE_ANALYSIS<<__EOF_ANALYSIS42__' + if [ -n "$partial_note" ]; then echo "$partial_note"; echo; fi cat run_workdir/failure_analysis.md # ensure delimiter is on its own line even if the file lacks a trailing newline printf '\n__EOF_ANALYSIS42__\n' @@ -171,14 +188,28 @@ jobs: # Surface the analysis in the workflow run UI itself: full content # in a foldable log group, and a copy in the run summary so it's # reachable without downloading the testrun-files artifact. + if [ -n "$partial_note" ]; then + echo "::warning::Analyze step outcome: ${ANALYZE_OUTCOME} - failure_analysis.md may be partial." + { echo "$partial_note"; echo; } >> "$GITHUB_STEP_SUMMARY" + fi echo "::group::Preliminary failure analysis" cat run_workdir/failure_analysis.md echo echo "::endgroup::" cat run_workdir/failure_analysis.md >> "$GITHUB_STEP_SUMMARY" else - echo "FAILURE_ANALYSIS=(no analysis produced)" >> "$GITHUB_ENV" - echo "::warning::No run_workdir/failure_analysis.md produced by the analyze step." + echo "FAILURE_ANALYSIS=(no analysis produced; analyze step outcome: ${ANALYZE_OUTCOME})" >> "$GITHUB_ENV" + echo "::error::No run_workdir/failure_analysis.md produced by the analyze step (outcome: ${ANALYZE_OUTCOME})." + fi + + # Keep the Claude execution log with the testrun files so failed or + # incomplete analysis runs can be debugged from the artifact alone. + # The testing step can die before it creates run_workdir. + if [ -f "${EXECUTION_FILE:-}" ]; then + mkdir -p run_workdir + cp "$EXECUTION_FILE" run_workdir/claude-execution-output.json + else + echo "::warning::No Claude execution log available (execution_file: '${EXECUTION_FILE:-}')." fi - name: Report test results if: (success() || failure()) && inputs.testrun_name @@ -225,6 +256,7 @@ jobs: run_workdir/requirements_coverage.json run_workdir/monitor.log run_workdir/failure_analysis.md + run_workdir/claude-execution-output.json - name: ↟ Upload CLI coverage uses: actions/upload-artifact@v7 if: success() || failure() diff --git a/.github/workflows/upgrade_reusable.yaml b/.github/workflows/upgrade_reusable.yaml index fc47eaea1..38d03fc3a 100644 --- a/.github/workflows/upgrade_reusable.yaml +++ b/.github/workflows/upgrade_reusable.yaml @@ -97,20 +97,37 @@ jobs: id: analyze-failures if: (success() || failure()) && steps.testing-step.outcome != 'success' && env.HAS_OAUTH_TOKEN == 'true' continue-on-error: true + # The action has its own internal timeout, this is insurance against + # the action itself hanging - without it a stuck step would run into + # the job timeout, and a job timeout is a cancellation that skips the + # artifact upload and mail steps. + timeout-minutes: 30 uses: anthropics/claude-code-base-action@beta with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} model: claude-sonnet-5 - max_turns: "40" - allowed_tools: "Read,Write,Glob,Grep,Bash(ls:*),Bash(wc:*),Bash(head:*),Bash(tail:*),Bash(grep:*),Bash(find:*),Bash(cut:*),Bash(sort:*),Bash(uniq:*),Bash(awk:*),Bash(jq:*),Bash(sqlite3:*)" + max_turns: "80" + allowed_tools: "Read,Write,Glob,Grep,Bash(scripts/count_test_results.py:*),Bash(ls:*),Bash(wc:*),Bash(head:*),Bash(tail:*),Bash(grep:*),Bash(find:*),Bash(cut:*),Bash(sort:*),Bash(uniq:*),Bash(awk:*),Bash(jq:*),Bash(sqlite3:*)" prompt: ${{ env.ANALYSIS_PROMPT }} - name: Read failure analysis into env id: read-analysis if: (success() || failure()) && steps.testing-step.outcome != 'success' && env.HAS_OAUTH_TOKEN == 'true' + env: + EXECUTION_FILE: ${{ steps.analyze-failures.outputs.execution_file }} + ANALYZE_OUTCOME: ${{ steps.analyze-failures.outcome }} run: | + # When the analyze step did not finish cleanly (e.g. ran out of + # turns mid-write), the analysis file may be a partial draft - mark + # it as such everywhere it is surfaced. + partial_note="" + if [ "$ANALYZE_OUTCOME" != "success" ]; then + partial_note="**Note: the analyze step did not finish cleanly (outcome: ${ANALYZE_OUTCOME}) - the analysis below may be incomplete.**" + fi + if [ -s run_workdir/failure_analysis.md ]; then { echo 'FAILURE_ANALYSIS<<__EOF_ANALYSIS42__' + if [ -n "$partial_note" ]; then echo "$partial_note"; echo; fi cat run_workdir/failure_analysis.md # ensure delimiter is on its own line even if the file lacks a trailing newline printf '\n__EOF_ANALYSIS42__\n' @@ -119,14 +136,28 @@ jobs: # Surface the analysis in the workflow run UI itself: full content # in a foldable log group, and a copy in the run summary so it's # reachable without downloading the testrun-files artifact. + if [ -n "$partial_note" ]; then + echo "::warning::Analyze step outcome: ${ANALYZE_OUTCOME} - failure_analysis.md may be partial." + { echo "$partial_note"; echo; } >> "$GITHUB_STEP_SUMMARY" + fi echo "::group::Preliminary failure analysis" cat run_workdir/failure_analysis.md echo echo "::endgroup::" cat run_workdir/failure_analysis.md >> "$GITHUB_STEP_SUMMARY" else - echo "FAILURE_ANALYSIS=(no analysis produced)" >> "$GITHUB_ENV" - echo "::warning::No run_workdir/failure_analysis.md produced by the analyze step." + echo "FAILURE_ANALYSIS=(no analysis produced; analyze step outcome: ${ANALYZE_OUTCOME})" >> "$GITHUB_ENV" + echo "::error::No run_workdir/failure_analysis.md produced by the analyze step (outcome: ${ANALYZE_OUTCOME})." + fi + + # Keep the Claude execution log with the testrun files so failed or + # incomplete analysis runs can be debugged from the artifact alone. + # The testing step can die before it creates run_workdir. + if [ -f "${EXECUTION_FILE:-}" ]; then + mkdir -p run_workdir + cp "$EXECUTION_FILE" run_workdir/claude-execution-output.json + else + echo "::warning::No Claude execution log available (execution_file: '${EXECUTION_FILE:-}')." fi - name: ↟ Upload testing artifacts on failure uses: actions/upload-artifact@v7 @@ -172,6 +203,7 @@ jobs: run_workdir/scheduling.log run_workdir/errors_all.log run_workdir/failure_analysis.md + run_workdir/claude-execution-output.json - name: ↟ Upload CLI coverage uses: actions/upload-artifact@v7 if: success() || failure() diff --git a/agent_docs/ci_analysis_prompt.md b/agent_docs/ci_analysis_prompt.md index 53ee8586d..2114b224b 100644 --- a/agent_docs/ci_analysis_prompt.md +++ b/agent_docs/ci_analysis_prompt.md @@ -3,6 +3,7 @@ Additional constraints for automated CI runs: - Output a single markdown file at: `{RUN_DIR}/failure_analysis.md`. +- Write a first rough draft of that file as soon as you have the test counts and the list of failed/broken tests, then refine it as you dig deeper. You have a limited number of turns and may be cut off at any point - a partial analysis must already be on disk when that happens. - Keep it under ~300 lines. Start now. diff --git a/agent_docs/failure_analysis_prompt.md b/agent_docs/failure_analysis_prompt.md index 8c5acfb4c..5df817617 100644 --- a/agent_docs/failure_analysis_prompt.md +++ b/agent_docs/failure_analysis_prompt.md @@ -16,26 +16,17 @@ Inputs available under `{RUN_DIR}/` (use only what exists): Counting tests (IMPORTANT): -Run following code snippet to get the correct counts: +Run following command to get the correct counts and the list of failed/broken tests: ```sh -passed=$(grep -l '"status": "passed"' {RUN_DIR}/allure-results/*result.json | wc -l) -failed=$(grep -l '"status": "failed"' {RUN_DIR}/allure-results/*result.json | wc -l) -broken=$(grep -l '"status": "broken"' {RUN_DIR}/allure-results/*result.json | wc -l) -total=$(grep -l '"message": "Skipped: collected, not run"' {RUN_DIR}/allure-results/*result.json | wc -l) -if [ "$total" -gt 0 ]; then - skipped=$(( total - passed - failed - broken )) -else - skipped=$(grep -l '"status": "skipped"' {RUN_DIR}/allure-results/*result.json | wc -l) - total=$(( passed + failed + broken + skipped )) -fi -echo "Total: $total, Passed: $passed, Failed: $failed, Broken: $broken, Skipped: $skipped" +scripts/count_test_results.py {RUN_DIR}/allure-results ``` +Run it exactly as shown, as a single plain command - do not wrap it in pipes, command substitution or other compound shell constructs, those are rejected by the CI command allowlist. + Steps: -1. Enumerate failed/broken tests: - `grep -E '"status": "(failed|broken)"' {RUN_DIR}/allure-results/*result.json | cut -c1-200`. +1. Enumerate failed/broken tests from the `count_test_results.py` output above. (`broken` = pytest error in setup/teardown; `failed` = assertion failure.) 2. Group failures by likely root cause (same exception class + message head, same node crash, same infra symptom). Treat one node crash that flunks many tests as a single group. 3. For each group: list affected tests (truncate to ~10 with a "+N more" tail), give the most informative 1–3 lines of error context, and classify as one of `node-bug | test-bug | infra-flake | env-issue | unknown` with a short justification. diff --git a/agent_docs/upgrade_failure_analysis_prompt.md b/agent_docs/upgrade_failure_analysis_prompt.md index d888c3c0c..e0338e548 100644 --- a/agent_docs/upgrade_failure_analysis_prompt.md +++ b/agent_docs/upgrade_failure_analysis_prompt.md @@ -21,15 +21,14 @@ Inputs available under `{RUN_DIR}/` (use only what exists): Steps: -1. Enumerate failed/broken tests per step: - `for s in step1 step2 step3; do grep -l -E '"status": "(failed|broken)"' {RUN_DIR}/allure-results-$s/*result.json 2>/dev/null | sed "s|^|$s: |"; done` +1. Get per-step counts and failed/broken test lists (with `statusDetails.message` heads): + `scripts/count_test_results.py {RUN_DIR}/allure-results-step1 {RUN_DIR}/allure-results-step2 {RUN_DIR}/allure-results-step3` + Run it exactly as shown, as a single plain command - do not wrap it in pipes, command substitution or other compound shell constructs, those are rejected by the CI command allowlist. (`broken` = pytest error in setup/teardown; `failed` = assertion failure.) -2. For each failing test, extract `statusDetails.message` head with: - `grep -E '"status": "(failed|broken)"' {RUN_DIR}/allure-results-step*/*result.json | cut -c1-200`. -3. Group failures by likely root cause (same exception class + message head, same node crash, same infra symptom). **Note which step(s) each group hits** — a failure that appears only in step2 or step3 is much more interesting than one that already fails in step1. Treat one node crash that flunks many tests as a single group. -4. For each group: list affected tests (truncate to ~10 with a "+N more" tail), give the most informative 1–3 lines of error context, mark the step(s) affected, and classify as one of `node-bug | test-bug | infra-flake | env-issue | upgrade-regression | unknown` with a short justification. Use `upgrade-regression` when a test passes in step1 but fails in step2 or step3 — that is the signal this workflow exists to catch. -5. Skim `{RUN_DIR}/errors_all.log` and `{RUN_DIR}/scheduling.log` for anything corroborating (node crash on restart, hard-fork failure, supervisord errors, OOM, repeated tracebacks). When failures look cluster-management related (dead cluster instances, tests stuck waiting for resources), query the `overview` view of the affected step's status database. -6. If a whole step is missing its `allure-results-stepN/` dir, that step likely failed before pytest ran — call this out explicitly and check `errors_all.log` / the workflow log group output for the cause (commonly a `start-cluster` / `supervisord` / hard-fork failure). +2. Group failures by likely root cause (same exception class + message head, same node crash, same infra symptom). **Note which step(s) each group hits** — a failure that appears only in step2 or step3 is much more interesting than one that already fails in step1. Treat one node crash that flunks many tests as a single group. +3. For each group: list affected tests (truncate to ~10 with a "+N more" tail), give the most informative 1–3 lines of error context, mark the step(s) affected, and classify as one of `node-bug | test-bug | infra-flake | env-issue | upgrade-regression | unknown` with a short justification. Use `upgrade-regression` when a test passes in step1 but fails in step2 or step3 — that is the signal this workflow exists to catch. +4. Skim `{RUN_DIR}/errors_all.log` and `{RUN_DIR}/scheduling.log` for anything corroborating (node crash on restart, hard-fork failure, supervisord errors, OOM, repeated tracebacks). When failures look cluster-management related (dead cluster instances, tests stuck waiting for resources), query the `overview` view of the affected step's status database. +5. If a whole step is missing its `allure-results-stepN/` dir, that step likely failed before pytest ran — call this out explicitly and check `errors_all.log` / the workflow log group output for the cause (commonly a `start-cluster` / `supervisord` / hard-fork failure). Known patterns: diff --git a/scripts/count_test_results.py b/scripts/count_test_results.py new file mode 100755 index 000000000..75d406af7 --- /dev/null +++ b/scripts/count_test_results.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +"""Summarize allure test results for preliminary failure analysis. + +Usage: + scripts/count_test_results.py [...] + +For each directory, prints per-test counts by status and one line per +failed/broken test (capped, with a '+N more' tail). Missing and empty +directories are reported and skipped. + +Result files are grouped per test (by Allure `historyId`, which identifies a +test together with its parameters), so tests with multiple result files are +counted once. Multiple files per test are normal: the initial `--skipall` +pass of `runner/run_tests.sh` registers every collected test with a +"Skipped: collected, not run" result file, and tests skipped via `skipif` +markers are recorded under their own skip reason even during that pass. The +newest non-registration result of a test wins. + +This is a standalone script so that AI failure analysis in CI can get the +counts with a single allowlisted command - the CI allowlist permits only +simple command prefixes, so compound shell snippets (pipes, command +substitution) are rejected there. +""" + +import json +import pathlib +import sys +import typing as tp + +# Cap the failed/broken enumeration so a mass-failure run cannot flood the +# consumer - AI agent tool output gets truncated around 30kB. +ENUM_LIMIT = 40 +LINE_WIDTH = 200 + +SKIPALL_MSG = "Skipped: collected, not run" + + +def _plural(count: int, word: str) -> str: + """Return the count together with the word, pluralized when needed. + + Args: + count: The number of items. + word: The singular form of the word. + + Returns: + E.g. "1 test" or "5 tests". + """ + return f"{count} {word}{'' if count == 1 else 's'}" + + +class _TestResult(tp.NamedTuple): + """Authoritative result of a single test.""" + + is_real: bool + start: float + status: str + name: str + msg_head: str + + +def _parse_result_file(fpath: pathlib.Path) -> tuple[str, bool, _TestResult]: + """Parse one allure result file. + + Args: + fpath: Path to a `*-result.json` file. + + Returns: + Tuple of (grouping key, whether the file had a historyId, test result). + + Raises: + OSError: When the file cannot be read. + ValueError: When the file content is not valid JSON. + TypeError: When the file content is not a JSON object. + AttributeError: When a nested field has an unexpected type. + """ + rec = json.loads(fpath.read_text(encoding="utf-8")) + if not isinstance(rec, dict): + err = "not a JSON object" + raise TypeError(err) + + key = rec.get("historyId") + has_history_id = bool(key) and isinstance(key, str) + if not has_history_id: + params = sorted( + (str(p.get("name") or ""), str(p.get("value") or "")) + for p in rec.get("parameters") or [] + ) + key = f"{rec.get('fullName') or rec.get('name') or fpath.name}|{params}" + + message = (rec.get("statusDetails") or {}).get("message") or "" + start = rec.get("start") + result = _TestResult( + is_real=message != SKIPALL_MSG, + start=start if isinstance(start, (int, float)) else 0, + status=str(rec.get("status") or "unknown"), + name=str(rec.get("name") or rec.get("fullName") or "?"), + msg_head=message.splitlines()[0] if message else "", + ) + return str(key), has_history_id, result + + +def _group_records( + result_files: list[pathlib.Path], +) -> tuple[dict[str, _TestResult], int, int]: + """Group result files per test, the newest real result of a test winning. + + Args: + result_files: Allure `*-result.json` files. + + Returns: + Tuple of (test key to authoritative result mapping, number of files + without a historyId, number of unreadable files). + """ + best: dict[str, _TestResult] = {} + no_history_id = 0 + read_errors = 0 + for fpath in result_files: + try: + key, has_history_id, result = _parse_result_file(fpath) + except (OSError, ValueError, TypeError, AttributeError) as exc: + # Printed to stdout so the error stays next to its directory's + # summary when the two streams are merged (as in CI). + print(f"Error: cannot read '{fpath}': {exc}") + read_errors += 1 + continue + if not has_history_id: + no_history_id += 1 + prev = best.get(key) + if prev is None or (result.is_real, result.start) > (prev.is_real, prev.start): + best[key] = result + return best, no_history_id, read_errors + + +def _print_counts(best: dict[str, _TestResult]) -> None: + """Print per-test counts by status. + + Args: + best: Test key to authoritative result mapping. + """ + counts: dict[str, int] = {} + for result in best.values(): + counts[result.status] = counts.get(result.status, 0) + 1 + + total = len(best) + passed = counts.get("passed", 0) + failed = counts.get("failed", 0) + broken = counts.get("broken", 0) + skipped = counts.get("skipped", 0) + print( + f"Total: {total}, Passed: {passed}, Failed: {failed}, Broken: {broken}, Skipped: {skipped}" + ) + other = total - passed - failed - broken - skipped + if other: + print(f"Other statuses: {other}") + + never_run = sum(1 for r in best.values() if not r.is_real) + if never_run: + print( + f"Note: {_plural(never_run, 'test')} registered by the initial skip pass " + "only, with no real result - the testrun was likely interrupted before " + "they could run" + ) + + +def _print_failures(best: dict[str, _TestResult]) -> None: + """Print one line per failed/broken test, sorted by test name and capped. + + Args: + best: Test key to authoritative result mapping. + """ + failures = sorted( + (r.name, f"{r.status}: {r.name}: {r.msg_head}"[:LINE_WIDTH]) + for r in best.values() + if r.status in ("failed", "broken") + ) + if not failures: + return + + print("-- failed/broken tests --") + for _, line in failures[:ENUM_LIMIT]: + print(line) + if len(failures) > ENUM_LIMIT: + print( + f"... +{len(failures) - ENUM_LIMIT} more (see the result JSON files for the full list)" + ) + + +def summarize_dir(results_dir: pathlib.Path) -> int: + """Print a test results summary for one allure results directory. + + Args: + results_dir: Directory with `*-result.json` allure files. + + Returns: + 0 on success (incl. missing or empty directory, which is reported as + an informational message), 1 when the directory or some of its result + files could not be read and the counts may therefore be incomplete. + """ + print(f"== {results_dir} ==") + + if not results_dir.is_dir(): + print(f"Directory not found: {results_dir}") + return 0 + + # `iterdir` instead of `glob`, as `glob` swallows permission errors and an + # unlistable directory would be misreported as having no results. + try: + result_files = sorted(f for f in results_dir.iterdir() if f.name.endswith("-result.json")) + except OSError as exc: + print(f"Error: cannot list '{results_dir}': {exc}") + return 1 + if not result_files: + print( + f"No *-result.json files found in {results_dir} - " + "the testrun likely did not produce results" + ) + return 0 + + best, no_history_id, read_errors = _group_records(result_files) + _print_counts(best) + _print_failures(best) + + if no_history_id: + print( + f"Note: {_plural(no_history_id, 'result file')} without historyId - " + "test grouping is approximate" + ) + if read_errors: + print( + f"Warning: {_plural(read_errors, 'result file')} could not be read - " + "counts may be incomplete" + ) + return 1 + return 0 + + +def main() -> int: + """Summarize each directory given on the command line. + + Returns: + The highest per-directory return code, or 2 on usage error. + """ + if len(sys.argv) < 2: + print( + f"Usage: {sys.argv[0]} [...]", + file=sys.stderr, + ) + return 2 + + exit_rc = 0 + for dir_arg in sys.argv[1:]: + exit_rc = max(exit_rc, summarize_dir(pathlib.Path(dir_arg))) + return exit_rc + + +if __name__ == "__main__": + sys.exit(main())