From 365efbe81aeeedfc32d5a9de602416f78c0556e2 Mon Sep 17 00:00:00 2001 From: Carlos Date: Tue, 29 Sep 2026 14:53:53 +0200 Subject: [PATCH] fix: classify auth/billing soft-skip before ERROR sticky comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move Anthropic auth/billing (and rate-limit/overloaded) classification into the review step via scripts/soft_skip.py so fail-on=never sets verdict=SKIPPED before the comment step — empty credits no longer post an ERROR sticky. anthropic-api-key is required: false. Regression fixture uses the real credit- balance stderr string. Docs: key-gate honest (secrets forbidden in jobs.*.if). --- .github/workflows/self-review.yml | 9 +-- AGENTS.md | 2 +- README.md | 17 ++++-- action.yml | 27 +++++++-- docs/FEATURE_MAP.md | 9 +-- fixtures/action/credit_balance_stderr.txt | 1 + fixtures/action/network_crash_stderr.txt | 1 + scripts/soft_skip.py | 44 ++++++++++++++ tests/test_soft_skip.py | 73 +++++++++++++++++++++++ 9 files changed, 162 insertions(+), 21 deletions(-) create mode 100644 fixtures/action/credit_balance_stderr.txt create mode 100644 fixtures/action/network_crash_stderr.txt create mode 100644 scripts/soft_skip.py create mode 100644 tests/test_soft_skip.py diff --git a/.github/workflows/self-review.yml b/.github/workflows/self-review.yml index b5d23b2..e149534 100644 --- a/.github/workflows/self-review.yml +++ b/.github/workflows/self-review.yml @@ -1,10 +1,11 @@ name: self-review # Dogfood the Action on this repo's PRs. -# GitHub does not allow `secrets` in jobs..if (only github/needs/vars/inputs), -# so we use a key-gate job: missing/empty ANTHROPIC_API_KEY skips the review job -# without failing the check. Keep fail-on: never (advisory). Billing/auth soft-exit -# is handled inside action.yml when fail-on=never — empty credits must not red CI. +# Key-gate: GitHub forbids `secrets` in jobs..if (only github/needs/vars/inputs), +# so a gate job exports has_key and the review job runs only when true. Do not put +# secrets in job if:. Missing/empty ANTHROPIC_API_KEY skips the review job without +# failing the check. Keep fail-on: never (advisory). action.yml soft-skips +# auth/billing as SKIPPED before any ERROR sticky — empty credits must not red CI. # Do not put real keys in git; set the secret in repo Settings -> Secrets. on: pull_request: diff --git a/AGENTS.md b/AGENTS.md index f42f159..889c0be 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,7 @@ Derived from `config/harness.json`. Policy lives in config, not in vibes. - **agent-action-gate** is the authorization engine (deterministic allow/deny/approval; zero LLM keys). - **lazycoder** is optional LLM analysis for code review. Verdict aggregation + `replay` + corpus `prove` are deterministic and run with **no** `ANTHROPIC_API_KEY`. - Live review / Action dogfood: **at most one** Anthropic key. Never require two keys to run the stack. Never put real keys in git. -- CI self-review (`.github/workflows/self-review.yml`) uses a key-gate job (GitHub forbids `secrets` in `jobs.*.if`) so an empty secret skips the review job; keeps `fail-on: never`. Auth/billing soft-skips inside the Action when advisory. +- CI self-review (`.github/workflows/self-review.yml`) uses a **key-gate** job (GitHub forbids `secrets` in `jobs..if`) so an empty secret skips the review job; keeps `fail-on: never`. Action soft-skips auth/billing/rate-limit as `SKIPPED` (no ERROR sticky) when advisory. `anthropic-api-key` input is `required: false`. ## Hard rules (non-negotiable) diff --git a/README.md b/README.md index 22c4666..d3eda59 100644 --- a/README.md +++ b/README.md @@ -84,12 +84,17 @@ tuned. Opt in with `fail-on: request-changes` once you have looked at what it reports on your own repo; the roadmap's next step is publishing the number that justifies flipping the default back. -Missing key skips with a warning (never red). When `fail-on: never`, Anthropic -**auth / empty-credits / billing** errors soft-skip with a warning instead of -failing the check; other operational errors (network, crash) still fail. -This repo's `.github/workflows/self-review.yml` uses a key-gate job so an empty -`ANTHROPIC_API_KEY` secret skips the review job (not a red check). Cost note: one -model call per rubric rule per diff hunk (17 × hunks). +`anthropic-api-key` is optional (`required: false`). Missing key skips with a +warning (never red). When `fail-on: never`, Anthropic **auth / empty-credits / +billing** (and rate-limit / overloaded) errors soft-skip as verdict `SKIPPED` +before any sticky comment — no ERROR comment, check stays green. Other +operational errors (network, crash) still fail. + +Self-review CI (`.github/workflows/self-review.yml`) uses a **key-gate** job: +GitHub forbids `secrets` in `jobs..if`, so a tiny gate job exports +`has_key` and the review job runs only when that output is true. Empty secret +→ review job skipped (not a red check). Cost note: one model call per rubric +rule per diff hunk (17 × hunks). Versioning is two-axis: the moving `@v1` tag tracks the action wrapper; the engine defaults to the latest PyPI release and can be pinned via `version`. diff --git a/action.yml b/action.yml index feb73b5..892ba22 100644 --- a/action.yml +++ b/action.yml @@ -9,9 +9,10 @@ branding: inputs: anthropic-api-key: description: >- - Anthropic API key. Missing/empty skips the review with a warning (never - fails the check). With fail-on=never, auth/billing errors also soft-skip. - required: true + Anthropic API key (optional). Missing/empty skips the review with a + warning (never fails the check). With fail-on=never, auth/billing (and + rate-limit/overloaded) errors soft-skip as SKIPPED — no ERROR sticky. + required: false model: description: Model override (LAZYCODER_MODEL) default: "" @@ -84,6 +85,16 @@ runs: code=$? set -e echo "exit_code=$code" >> "$GITHUB_OUTPUT" + # Classify auth/billing BEFORE the comment step. Soft-skip → SKIPPED + # so the ERROR sticky never posts; gate then exits 0. + if [ "$code" -ge 3 ] && [ "${{ inputs.fail-on }}" = "never" ] && \ + python3 "${{ github.action_path }}/scripts/soft_skip.py" \ + "$RUNNER_TEMP/stderr.txt"; then + echo "::warning::lazycoder soft-skipped — Anthropic auth/billing error (fail-on=never); not failing the check" + echo "verdict=SKIPPED" >> "$GITHUB_OUTPUT" + echo "lazycoder verdict: SKIPPED (exit $code, soft-skip)" + exit 0 + fi case $code in 0) verdict=APPROVE ;; 1) verdict=REQUEST_CHANGES ;; @@ -124,12 +135,16 @@ runs: shell: bash run: | code="${{ steps.review.outputs.exit_code }}" + verdict="${{ steps.review.outputs.verdict }}" + if [ "$verdict" = "SKIPPED" ]; then + exit 0 + fi if [ "$code" -ge 3 ]; then cat "$RUNNER_TEMP/stderr.txt" >&2 || true - err=$(cat "$RUNNER_TEMP/stderr.txt" 2>/dev/null || true) - # Auth / empty-credits must not fail the PR check when advisory. + # Defense in depth: same classifier as the review step. if [ "${{ inputs.fail-on }}" = "never" ] && \ - echo "$err" | grep -qiE 'credit balance|billing|authentication|unauthorized|invalid.?api.?key|api.?key.*(invalid|missing)|401|403'; then + python3 "${{ github.action_path }}/scripts/soft_skip.py" \ + "$RUNNER_TEMP/stderr.txt"; then echo "::warning::lazycoder soft-skipped — Anthropic auth/billing error (fail-on=never); not failing the check" exit 0 fi diff --git a/docs/FEATURE_MAP.md b/docs/FEATURE_MAP.md index 6765166..4e80ab3 100644 --- a/docs/FEATURE_MAP.md +++ b/docs/FEATURE_MAP.md @@ -50,7 +50,7 @@ from a second embedded copy. | Refuse APPROVE on empty diff | `test_cli.py` | Exit 3 | | `--log` writes one record | `test_decision_log.py` | | | `lazycoder replay LOG` | `test_cli.py` | No Anthropic client constructed | -| GitHub Action wrapper | — | `action.yml`; advisory `fail-on: never`; missing key → SKIPPED; auth/billing soft-skip when `fail-on=never` | +| GitHub Action wrapper | `test_soft_skip.py` | `action.yml`; advisory `fail-on: never`; missing key → SKIPPED; auth/billing soft-skip **before** ERROR sticky (`scripts/soft_skip.py`) | ## How to run @@ -66,7 +66,8 @@ lazycoder replay runs.jsonl CI: - `.github/workflows/test.yml` — `uv sync --extra dev` → `pytest -q` → ruff → black → mypy (no Anthropic secret). -- `.github/workflows/self-review.yml` — dogfood Action; key-gate skips the review - job when `ANTHROPIC_API_KEY` is empty (GitHub forbids `secrets` in `jobs.*.if`); - `fail-on: never`; auth/billing soft-skip inside the Action. Replay coverage is +- `.github/workflows/self-review.yml` — dogfood Action; **key-gate** job exports + `has_key` (GitHub forbids `secrets` in `jobs..if` — do not put secrets in + job `if:`); empty secret skips the review job. `fail-on: never`; auth/billing + soft-skip as `SKIPPED` before any ERROR sticky. Replay + soft-skip coverage inside pytest. diff --git a/fixtures/action/credit_balance_stderr.txt b/fixtures/action/credit_balance_stderr.txt new file mode 100644 index 0000000..514b4f2 --- /dev/null +++ b/fixtures/action/credit_balance_stderr.txt @@ -0,0 +1 @@ +error: Error code: 400 - {'type': 'error', 'error': {'type': 'invalid_request_error', 'message': 'Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.'}, 'request_id': 'req_011Cbr53dad6tQFhL2qejsTR'} diff --git a/fixtures/action/network_crash_stderr.txt b/fixtures/action/network_crash_stderr.txt new file mode 100644 index 0000000..9c80601 --- /dev/null +++ b/fixtures/action/network_crash_stderr.txt @@ -0,0 +1 @@ +error: Connection error. diff --git a/scripts/soft_skip.py b/scripts/soft_skip.py new file mode 100644 index 0000000..ed5169c --- /dev/null +++ b/scripts/soft_skip.py @@ -0,0 +1,44 @@ +"""Classify Anthropic stderr as soft-skip (auth/billing) vs hard fail. + +Used by action.yml in the review step — before the sticky comment — when +fail-on=never, so an empty-credits / auth error becomes verdict=SKIPPED and +never posts an ERROR comment. + +Exit codes (CLI): 0 = soft-skip, 1 = not a soft-skip / unreadable input. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +# Match Anthropic auth / empty-credits / billing. Also rate_limit / overloaded: +# under fail-on=never those are transient capacity, not a review finding. +SOFT_SKIP_RE = re.compile( + r"credit balance|billing|authentication|unauthorized|" + r"invalid.?api.?key|api.?key.*(invalid|missing)|" + r"\b401\b|\b403\b|" + r"rate.?limit|overloaded|\b529\b", + re.IGNORECASE, +) + + +def is_soft_skip(stderr: str) -> bool: + """True when stderr looks like Anthropic auth/billing (or rate-limit).""" + return bool(SOFT_SKIP_RE.search(stderr)) + + +def main(argv: list[str] | None = None) -> int: + args = list(sys.argv[1:] if argv is None else argv) + if not args: + return 1 + try: + text = Path(args[0]).read_text(encoding="utf-8", errors="replace") + except OSError: + return 1 + return 0 if is_soft_skip(text) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_soft_skip.py b/tests/test_soft_skip.py new file mode 100644 index 0000000..bc15c27 --- /dev/null +++ b/tests/test_soft_skip.py @@ -0,0 +1,73 @@ +"""Regression: auth/billing soft-skip must exit soft (no hard-fail path). + +Mirrors action.yml review-step classification so CI proves empty-credits +stderr never becomes an ERROR sticky / red check under fail-on=never. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +from soft_skip import is_soft_skip, main as soft_skip_main # noqa: E402, I001 + +ROOT = Path(__file__).resolve().parent.parent +CREDIT_STDERR = ROOT / "fixtures" / "action" / "credit_balance_stderr.txt" +NETWORK_STDERR = ROOT / "fixtures" / "action" / "network_crash_stderr.txt" + + +def test_credit_balance_fixture_contains_real_anthropic_string() -> None: + text = CREDIT_STDERR.read_text(encoding="utf-8") + assert "Your credit balance is too low to access the Anthropic API" in text + assert "Plans & Billing" in text + + +def test_credit_balance_is_soft_skip() -> None: + text = CREDIT_STDERR.read_text(encoding="utf-8") + assert is_soft_skip(text) is True + + +def test_network_crash_is_not_soft_skip() -> None: + text = NETWORK_STDERR.read_text(encoding="utf-8") + assert is_soft_skip(text) is False + + +def test_cli_soft_skip_exit_0_on_credit_balance() -> None: + assert soft_skip_main([str(CREDIT_STDERR)]) == 0 + + +def test_cli_hard_path_exit_1_on_network_crash() -> None: + assert soft_skip_main([str(NETWORK_STDERR)]) == 1 + + +def test_subprocess_matches_action_yml_invocation() -> None: + """Same invocation action.yml uses: python3 scripts/soft_skip.py .""" + script = ROOT / "scripts" / "soft_skip.py" + soft = subprocess.run( + [sys.executable, str(script), str(CREDIT_STDERR)], + check=False, + ) + hard = subprocess.run( + [sys.executable, str(script), str(NETWORK_STDERR)], + check=False, + ) + assert soft.returncode == 0, "credit-balance must soft-skip (exit 0)" + assert hard.returncode == 1, "network crash must stay on hard-fail path" + + +@pytest.mark.parametrize( + "snippet", + [ + "AuthenticationError: invalid x-api-key", + "Error code: 401 - unauthorized", + "rate_limit_error: Number of requests", + "Error code: 529 - {'type': 'error', 'error': {'type': 'overloaded_error'", + ], +) +def test_auth_and_capacity_snippets_soft_skip(snippet: str) -> None: + assert is_soft_skip(snippet) is True