fix(ci): make AI failure analysis reliable

.github/workflows/regression_reusable.yaml

@@ -149,20 +149,37 @@ jobs:
         id: analyze-failures
         if: (success() || failure()) && steps.testing-step.outcome != 'success' && env.HAS_OAUTH_TOKEN == 'true'
         continue-on-error: true
+        # The action has its own internal timeout, this is insurance against
+        # the action itself hanging - without it a stuck step would run into
+        # the job timeout, and a job timeout is a cancellation that skips the
+        # artifact upload and mail steps.
+        timeout-minutes: 30
         uses: anthropics/claude-code-base-action@beta
         with:
           claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
           model: claude-sonnet-5
-          max_turns: "40"
-          allowed_tools: "Read,Write,Glob,Grep,Bash(ls:*),Bash(wc:*),Bash(head:*),Bash(tail:*),Bash(grep:*),Bash(find:*),Bash(cut:*),Bash(sort:*),Bash(uniq:*),Bash(awk:*),Bash(jq:*),Bash(sqlite3:*)"
+          max_turns: "80"
+          allowed_tools: "Read,Write,Glob,Grep,Bash(scripts/count_test_results.py:*),Bash(ls:*),Bash(wc:*),Bash(head:*),Bash(tail:*),Bash(grep:*),Bash(find:*),Bash(cut:*),Bash(sort:*),Bash(uniq:*),Bash(awk:*),Bash(jq:*),Bash(sqlite3:*)"
           prompt: ${{ env.ANALYSIS_PROMPT }}
       - name: Read failure analysis into env
         id: read-analysis
         if: (success() || failure()) && steps.testing-step.outcome != 'success' && env.HAS_OAUTH_TOKEN == 'true'
+        env:
+          EXECUTION_FILE: ${{ steps.analyze-failures.outputs.execution_file }}
+          ANALYZE_OUTCOME: ${{ steps.analyze-failures.outcome }}
         run: |
+          # When the analyze step did not finish cleanly (e.g. ran out of
+          # turns mid-write), the analysis file may be a partial draft - mark
+          # it as such everywhere it is surfaced.
+          partial_note=""
+          if [ "$ANALYZE_OUTCOME" != "success" ]; then
+            partial_note="**Note: the analyze step did not finish cleanly (outcome: ${ANALYZE_OUTCOME}) - the analysis below may be incomplete.**"
+          fi
+
           if [ -s run_workdir/failure_analysis.md ]; then
             {
               echo 'FAILURE_ANALYSIS<<__EOF_ANALYSIS42__'
+              if [ -n "$partial_note" ]; then echo "$partial_note"; echo; fi
               cat run_workdir/failure_analysis.md
               # ensure delimiter is on its own line even if the file lacks a trailing newline
               printf '\n__EOF_ANALYSIS42__\n'
@@ -171,14 +188,28 @@ jobs:
             # Surface the analysis in the workflow run UI itself: full content
             # in a foldable log group, and a copy in the run summary so it's
             # reachable without downloading the testrun-files artifact.
+            if [ -n "$partial_note" ]; then
+              echo "::warning::Analyze step outcome: ${ANALYZE_OUTCOME} - failure_analysis.md may be partial."
+              { echo "$partial_note"; echo; } >> "$GITHUB_STEP_SUMMARY"
+            fi
             echo "::group::Preliminary failure analysis"
             cat run_workdir/failure_analysis.md
             echo
             echo "::endgroup::"
             cat run_workdir/failure_analysis.md >> "$GITHUB_STEP_SUMMARY"
           else
-            echo "FAILURE_ANALYSIS=(no analysis produced)" >> "$GITHUB_ENV"
-            echo "::warning::No run_workdir/failure_analysis.md produced by the analyze step."
+            echo "FAILURE_ANALYSIS=(no analysis produced; analyze step outcome: ${ANALYZE_OUTCOME})" >> "$GITHUB_ENV"
+            echo "::error::No run_workdir/failure_analysis.md produced by the analyze step (outcome: ${ANALYZE_OUTCOME})."
+          fi
+
+          # Keep the Claude execution log with the testrun files so failed or
+          # incomplete analysis runs can be debugged from the artifact alone.
+          # The testing step can die before it creates run_workdir.
+          if [ -f "${EXECUTION_FILE:-}" ]; then
+            mkdir -p run_workdir
+            cp "$EXECUTION_FILE" run_workdir/claude-execution-output.json
+          else
+            echo "::warning::No Claude execution log available (execution_file: '${EXECUTION_FILE:-}')."
           fi
       - name: Report test results
         if: (success() || failure()) && inputs.testrun_name
@@ -225,6 +256,7 @@ jobs:
             run_workdir/requirements_coverage.json
             run_workdir/monitor.log
             run_workdir/failure_analysis.md
+            run_workdir/claude-execution-output.json
       - name: ↟ Upload CLI coverage
         uses: actions/upload-artifact@v7
         if: success() || failure()

.github/workflows/upgrade_reusable.yaml

@@ -97,20 +97,37 @@ jobs:
         id: analyze-failures
         if: (success() || failure()) && steps.testing-step.outcome != 'success' && env.HAS_OAUTH_TOKEN == 'true'
         continue-on-error: true
+        # The action has its own internal timeout, this is insurance against
+        # the action itself hanging - without it a stuck step would run into
+        # the job timeout, and a job timeout is a cancellation that skips the
+        # artifact upload and mail steps.
+        timeout-minutes: 30
         uses: anthropics/claude-code-base-action@beta
         with:
           claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
           model: claude-sonnet-5
-          max_turns: "40"
-          allowed_tools: "Read,Write,Glob,Grep,Bash(ls:*),Bash(wc:*),Bash(head:*),Bash(tail:*),Bash(grep:*),Bash(find:*),Bash(cut:*),Bash(sort:*),Bash(uniq:*),Bash(awk:*),Bash(jq:*),Bash(sqlite3:*)"
+          max_turns: "80"
+          allowed_tools: "Read,Write,Glob,Grep,Bash(scripts/count_test_results.py:*),Bash(ls:*),Bash(wc:*),Bash(head:*),Bash(tail:*),Bash(grep:*),Bash(find:*),Bash(cut:*),Bash(sort:*),Bash(uniq:*),Bash(awk:*),Bash(jq:*),Bash(sqlite3:*)"
           prompt: ${{ env.ANALYSIS_PROMPT }}
       - name: Read failure analysis into env
         id: read-analysis
         if: (success() || failure()) && steps.testing-step.outcome != 'success' && env.HAS_OAUTH_TOKEN == 'true'
+        env:
+          EXECUTION_FILE: ${{ steps.analyze-failures.outputs.execution_file }}
+          ANALYZE_OUTCOME: ${{ steps.analyze-failures.outcome }}
         run: |
+          # When the analyze step did not finish cleanly (e.g. ran out of
+          # turns mid-write), the analysis file may be a partial draft - mark
+          # it as such everywhere it is surfaced.
+          partial_note=""
+          if [ "$ANALYZE_OUTCOME" != "success" ]; then
+            partial_note="**Note: the analyze step did not finish cleanly (outcome: ${ANALYZE_OUTCOME}) - the analysis below may be incomplete.**"
+          fi
+
           if [ -s run_workdir/failure_analysis.md ]; then
             {
               echo 'FAILURE_ANALYSIS<<__EOF_ANALYSIS42__'
+              if [ -n "$partial_note" ]; then echo "$partial_note"; echo; fi
               cat run_workdir/failure_analysis.md
               # ensure delimiter is on its own line even if the file lacks a trailing newline
               printf '\n__EOF_ANALYSIS42__\n'
@@ -119,14 +136,28 @@ jobs:
             # Surface the analysis in the workflow run UI itself: full content
             # in a foldable log group, and a copy in the run summary so it's
             # reachable without downloading the testrun-files artifact.
+            if [ -n "$partial_note" ]; then
+              echo "::warning::Analyze step outcome: ${ANALYZE_OUTCOME} - failure_analysis.md may be partial."
+              { echo "$partial_note"; echo; } >> "$GITHUB_STEP_SUMMARY"
+            fi
             echo "::group::Preliminary failure analysis"
             cat run_workdir/failure_analysis.md
             echo
             echo "::endgroup::"
             cat run_workdir/failure_analysis.md >> "$GITHUB_STEP_SUMMARY"
           else
-            echo "FAILURE_ANALYSIS=(no analysis produced)" >> "$GITHUB_ENV"
-            echo "::warning::No run_workdir/failure_analysis.md produced by the analyze step."
+            echo "FAILURE_ANALYSIS=(no analysis produced; analyze step outcome: ${ANALYZE_OUTCOME})" >> "$GITHUB_ENV"
+            echo "::error::No run_workdir/failure_analysis.md produced by the analyze step (outcome: ${ANALYZE_OUTCOME})."
+          fi
+
+          # Keep the Claude execution log with the testrun files so failed or
+          # incomplete analysis runs can be debugged from the artifact alone.
+          # The testing step can die before it creates run_workdir.
+          if [ -f "${EXECUTION_FILE:-}" ]; then
+            mkdir -p run_workdir
+            cp "$EXECUTION_FILE" run_workdir/claude-execution-output.json
+          else
+            echo "::warning::No Claude execution log available (execution_file: '${EXECUTION_FILE:-}')."
           fi
       - name: ↟ Upload testing artifacts on failure
         uses: actions/upload-artifact@v7
@@ -172,6 +203,7 @@ jobs:
             run_workdir/scheduling.log
             run_workdir/errors_all.log
             run_workdir/failure_analysis.md
+            run_workdir/claude-execution-output.json
       - name: ↟ Upload CLI coverage
         uses: actions/upload-artifact@v7
         if: success() || failure()

agent_docs/ci_analysis_prompt.md

@@ -3,6 +3,7 @@
 Additional constraints for automated CI runs:
 
 - Output a single markdown file at: `{RUN_DIR}/failure_analysis.md`.
+- Write a first rough draft of that file as soon as you have the test counts and the list of failed/broken tests, then refine it as you dig deeper. You have a limited number of turns and may be cut off at any point - a partial analysis must already be on disk when that happens.
 - Keep it under ~300 lines.
 
 Start now.

agent_docs/failure_analysis_prompt.md

@@ -16,26 +16,17 @@ Inputs available under `{RUN_DIR}/` (use only what exists):
 
 Counting tests (IMPORTANT):
 
-Run following code snippet to get the correct counts:
+Run following command to get the correct counts and the list of failed/broken tests:
 
 ```sh
-passed=$(grep -l '"status": "passed"' {RUN_DIR}/allure-results/*result.json | wc -l)
-failed=$(grep -l '"status": "failed"' {RUN_DIR}/allure-results/*result.json | wc -l)
-broken=$(grep -l '"status": "broken"' {RUN_DIR}/allure-results/*result.json | wc -l)
-total=$(grep -l '"message": "Skipped: collected, not run"' {RUN_DIR}/allure-results/*result.json | wc -l)
-if [ "$total" -gt 0 ]; then
-  skipped=$(( total - passed - failed - broken ))
-else
-  skipped=$(grep -l '"status": "skipped"' {RUN_DIR}/allure-results/*result.json | wc -l)
-  total=$(( passed + failed + broken + skipped ))
-fi
-echo "Total: $total, Passed: $passed, Failed: $failed, Broken: $broken, Skipped: $skipped"
+scripts/count_test_results.py {RUN_DIR}/allure-results
 ```
 
+Run it exactly as shown, as a single plain command - do not wrap it in pipes, command substitution or other compound shell constructs, those are rejected by the CI command allowlist.
+
 Steps:
 
-1. Enumerate failed/broken tests:
-   `grep -E '"status": "(failed|broken)"' {RUN_DIR}/allure-results/*result.json | cut -c1-200`.
+1. Enumerate failed/broken tests from the `count_test_results.py` output above.
    (`broken` = pytest error in setup/teardown; `failed` = assertion failure.)
 2. Group failures by likely root cause (same exception class + message head, same node crash, same infra symptom). Treat one node crash that flunks many tests as a single group.
 3. For each group: list affected tests (truncate to ~10 with a "+N more" tail), give the most informative 1–3 lines of error context, and classify as one of `node-bug | test-bug | infra-flake | env-issue | unknown` with a short justification.

agent_docs/upgrade_failure_analysis_prompt.md

@@ -21,15 +21,14 @@ Inputs available under `{RUN_DIR}/` (use only what exists):
 
 Steps:
 
-1. Enumerate failed/broken tests per step:
-   `for s in step1 step2 step3; do grep -l -E '"status": "(failed|broken)"' {RUN_DIR}/allure-results-$s/*result.json 2>/dev/null | sed "s|^|$s: |"; done`
+1. Get per-step counts and failed/broken test lists (with `statusDetails.message` heads):
+   `scripts/count_test_results.py {RUN_DIR}/allure-results-step1 {RUN_DIR}/allure-results-step2 {RUN_DIR}/allure-results-step3`
+   Run it exactly as shown, as a single plain command - do not wrap it in pipes, command substitution or other compound shell constructs, those are rejected by the CI command allowlist.
    (`broken` = pytest error in setup/teardown; `failed` = assertion failure.)
-2. For each failing test, extract `statusDetails.message` head with:
-   `grep -E '"status": "(failed|broken)"' {RUN_DIR}/allure-results-step*/*result.json | cut -c1-200`.
-3. Group failures by likely root cause (same exception class + message head, same node crash, same infra symptom). **Note which step(s) each group hits** — a failure that appears only in step2 or step3 is much more interesting than one that already fails in step1. Treat one node crash that flunks many tests as a single group.
-4. For each group: list affected tests (truncate to ~10 with a "+N more" tail), give the most informative 1–3 lines of error context, mark the step(s) affected, and classify as one of `node-bug | test-bug | infra-flake | env-issue | upgrade-regression | unknown` with a short justification. Use `upgrade-regression` when a test passes in step1 but fails in step2 or step3 — that is the signal this workflow exists to catch.
-5. Skim `{RUN_DIR}/errors_all.log` and `{RUN_DIR}/scheduling.log` for anything corroborating (node crash on restart, hard-fork failure, supervisord errors, OOM, repeated tracebacks). When failures look cluster-management related (dead cluster instances, tests stuck waiting for resources), query the `overview` view of the affected step's status database.
-6. If a whole step is missing its `allure-results-stepN/` dir, that step likely failed before pytest ran — call this out explicitly and check `errors_all.log` / the workflow log group output for the cause (commonly a `start-cluster` / `supervisord` / hard-fork failure).
+2. Group failures by likely root cause (same exception class + message head, same node crash, same infra symptom). **Note which step(s) each group hits** — a failure that appears only in step2 or step3 is much more interesting than one that already fails in step1. Treat one node crash that flunks many tests as a single group.
+3. For each group: list affected tests (truncate to ~10 with a "+N more" tail), give the most informative 1–3 lines of error context, mark the step(s) affected, and classify as one of `node-bug | test-bug | infra-flake | env-issue | upgrade-regression | unknown` with a short justification. Use `upgrade-regression` when a test passes in step1 but fails in step2 or step3 — that is the signal this workflow exists to catch.
+4. Skim `{RUN_DIR}/errors_all.log` and `{RUN_DIR}/scheduling.log` for anything corroborating (node crash on restart, hard-fork failure, supervisord errors, OOM, repeated tracebacks). When failures look cluster-management related (dead cluster instances, tests stuck waiting for resources), query the `overview` view of the affected step's status database.
+5. If a whole step is missing its `allure-results-stepN/` dir, that step likely failed before pytest ran — call this out explicitly and check `errors_all.log` / the workflow log group output for the cause (commonly a `start-cluster` / `supervisord` / hard-fork failure).
 
 Known patterns: