diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af800bc..29898c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,9 @@ on: pull_request: branches: [main] +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest @@ -14,8 +17,10 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} cache: pip @@ -32,8 +37,10 @@ jobs: runs-on: ubuntu-latest needs: test steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" cache: pip @@ -48,6 +55,23 @@ jobs: runs-on: ubuntu-latest needs: test steps: - - uses: actions/checkout@v4 - - run: docker build -t code-fix-agent:ci . - - run: docker run --rm --entrypoint python code-fix-agent:ci -c "from code_fix_agent import Agent; print('ok')" + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + cache: pip + # The tag the sandbox runner looks for. + - name: Build the sandbox image + run: docker build -t code-fix-agent:latest . + - name: Install + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + - name: Sandbox tests + run: pytest tests/test_workspace.py -k Docker -rs -v + - name: Mock benchmark in the sandbox + run: python -m benchmarks.run_benchmark --mode mock --sandbox docker --out "$RUNNER_TEMP/bench-docker.json" + - name: Mock benchmark as the image's non-root user, over a copied results file + run: docker run --rm --entrypoint python code-fix-agent:latest -m benchmarks.run_benchmark --mode mock --out benchmarks/results/benchmark.json diff --git a/.gitignore b/.gitignore index 76ac80e..23d0730 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ venv/ *.swp benchmarks/results/*.tmp.json +benchmarks/results/benchmark-mock.json benchmarks/workspaces/ .DS_Store diff --git a/Dockerfile b/Dockerfile index ba49413..0d965cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,11 @@ COPY bugs ./bugs COPY benchmarks ./benchmarks COPY README.md LICENSE ./ -RUN pip install --no-cache-dir -e ".[dev]" +RUN pip install --no-cache-dir -e ".[dev]" \ + && useradd --create-home --uid 10001 app \ + && chown -R app benchmarks/results + +USER app HEALTHCHECK --interval=30s --timeout=15s --start-period=5s --retries=2 \ CMD python -c "from code_fix_agent import Agent; print('ok')" || exit 1 diff --git a/README.md b/README.md index abdb2b7..8c5ea04 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # code-fix-agent -> Autonomous Python bug-fix agent on the Anthropic API. **12/12 pass on Claude Haiku 4.5 live, 3.58 mean iterations per bug, $0.10 total cost.** Real pytest-driven fix loop, isolated workspace per attempt, no jailbreaks. Reproduce live in 90 seconds. +> Autonomous Python bug-fix agent on the Anthropic API. **12/12 pass on Claude Haiku 4.5 live, 3.58 mean iterations per bug, $0.10 total cost** (May 2026 run, before the current sandbox and independent scoring; see below). Real pytest-driven fix loop, one temp workspace per attempt, a network-less Docker runner for the tests (not yet run; see [Isolation and scoring](#isolation-and-scoring)), and every fix scored against a clean copy of the original tests. [![ci](https://github.com/Tajaddin/code-fix-agent/actions/workflows/ci.yml/badge.svg)](https://github.com/Tajaddin/code-fix-agent/actions/workflows/ci.yml) [![License](https://img.shields.io/badge/license-MIT-blue)](LICENSE) @@ -9,28 +9,33 @@ ## Hero metrics Reproducible: -- `python -m benchmarks.run_benchmark --mode mock` (11 seconds, no API key, pipeline smoke) -- `python -m benchmarks.run_benchmark --mode live` (90 seconds, needs ANTHROPIC_API_KEY, real Haiku 4.5) +- `python -m benchmarks.run_benchmark --mode mock` (no API key, pipeline smoke; writes `benchmarks/results/benchmark-mock.json`, not committed) +- `python -m benchmarks.run_benchmark --mode live` (needs ANTHROPIC_API_KEY, real Haiku 4.5; writes `benchmarks/results/benchmark.json`) | Mode | Pass rate | Mean iterations / bug | Mean tool calls / bug | Total tokens | Realized cost (Haiku 4.5) | |---|---:|---:|---:|---:|---:| | **mock (pipeline smoke)** | **12 / 12** | 3.0 | 3.0 | 540 | $0.0013 | -| **live (Haiku 4.5)** | **12 / 12** | **3.58** | **5.58** | 77,457 | **$0.1012** | +| **live (Haiku 4.5), 2026-05-20** | **12 / 12** | **3.58** | **5.58** | 77,457 | **$0.1012** | Live run logged 2026-05-20: all 12 bugs fixed, no skips, no errors. The agent average 3.58 LLM rounds and 5.58 tool calls per bug. Total spend across the 12-bug sweep was ~10 cents. Per-bug records are in `benchmarks/results/benchmark.json`. +That run predates the current harness. Its pass was the agent's own last `run_tests` call, not the independent scoring described under [Isolation and scoring](#isolation-and-scoring), its prompt included the bug id (which names the bug kind), its tests ran without a sandbox, and its records do not include tool inputs or diffs. Each record's summary shows the bug's full test count passing, so no bug passed on a subset of its tests. It has not yet been re-run on the current harness. + The mock mode scripts the canonical (read -> edit -> run_tests) for each of the 12 bugs. A perfect 12/12 in mock means the workspace, tool dispatch, and agent loop are healthy; CI runs this on every push. Live mode hands the same 12 bugs to Claude Haiku 4.5 via the Anthropic API, with no scripted fixes. To reproduce: ```bash export ANTHROPIC_API_KEY=sk-ant-... +docker build -t code-fix-agent:latest . # the test sandbox; see below python -m benchmarks.run_benchmark --mode live --max-iterations 8 ``` +Live runs refuse to start without the sandbox image unless you pass `--sandbox local`, because the local runner cannot keep the API key from model-written code (see [Isolation and scoring](#isolation-and-scoring)). + ## The dataset -12 hand-crafted Python bugs in `bugs/`. Each subdirectory contains `buggy.py` plus `test_buggy.py`. The fix is a 1-3 line surgical edit. The test file is held back from the agent in its system prompt instructions (the agent is told not to modify the test). +12 hand-crafted Python bugs in `bugs/`. Each subdirectory contains `buggy.py` plus `test_buggy.py`. The fix is a 1-3 line surgical edit. The agent may read the test file but is told not to change it; scoring ignores any change it makes. | ID | Bug kind | Expected iters | |---|---|---:| @@ -53,49 +58,38 @@ Each bug has 3-4 pytest assertions; the agent needs to satisfy all of them. | Path | Purpose | |---|---| -| `src/code_fix_agent/workspace.py` | Isolated tempdir per attempt. Read, edit (str_replace with uniqueness check), search, run_tests via subprocess. | +| `src/code_fix_agent/workspace.py` | Tempdir per attempt. Read, edit (str_replace with uniqueness check), search and run_tests, all confined to the workspace root; `score()` and `diff()` for the harness. | +| `src/code_fix_agent/sandbox.py` | Runs pytest in a network-less Docker container, or in a subprocess with a minimal environment when Docker is unavailable. | | `src/code_fix_agent/tools.py` | Five Anthropic tool definitions and their dispatcher: list_dir, read_file, edit_file, search_code, run_tests. | | `src/code_fix_agent/agent.py` | The fix loop. Reads response content, dispatches tool_use blocks, sends tool_result back, stops when run_tests reports PASS or iteration budget is exhausted. | | `src/code_fix_agent/llm.py` | `AnthropicLLM` + `MockLLM` (scripted tool-call sequence). Both return the same `(content, stop_reason, usage)` shape. | | `benchmarks/run_benchmark.py` | Iterates the dataset, captures pass / iteration / token / cost stats per bug, writes JSON. | -## Why this matters for production - -Recruiter signal this maps to: -- **SWE-agent / coding-agent JDs** (Mango Languages, Sekai, M3 USA, Cohere Agent Infrastructure) -- **Tool use + multi-turn loops** (any "Senior AI Engineer, Agentic Systems" role in 2026) -- **Real-pytest-driven evaluation** (not LLM-as-judge, not vibe-graded) -- **Bounded iteration budgets + token budgets** (the diff between a demo agent and a billable agent) - -This is the small-bench cousin of SWE-bench-Lite. It runs in seconds on a laptop and costs cents per full sweep. Drop SWE-bench-verified in next to scale; the agent loop does not change. - ## Quick start ```bash pip install -e ".[dev]" export ANTHROPIC_API_KEY=sk-ant-... -# Fix one bug, print JSON result +# Fix one bug, print JSON result (needs the sandbox image, or --sandbox local) code-fix-agent run bugs/01_off_by_one # Or run the full 12-bug benchmark python -m benchmarks.run_benchmark --mode live ``` -Sample output (`run bugs/01_off_by_one`): - -```json -{ - "bug_id": "01_off_by_one", - "passed": true, - "iterations": 3, - "tool_calls": 3, - "elapsed_seconds": 4.21, - "total_input_tokens": 2843, - "total_output_tokens": 142, - "final_summary": "PASS (exit 0) summary='4 passed in 0.03s'" -} -``` +`run` prints one JSON object and exits 0 when the fix passes scoring, 1 otherwise. Its fields: + +| Field | Meaning | +|---|---| +| `bug_id`, `sandbox` | The bug folder's name and the runner the tests used (`docker` or `local`). | +| `passed` | The verdict: the original tests on a clean copy (see [Scoring](#isolation-and-scoring)). | +| `self_reported_pass`, `tests_modified` | The agent's own last `run_tests` said PASS; a test file differs from its setup SHA-256. | +| `iterations`, `tool_calls`, `elapsed_seconds` | Model rounds, tool calls and wall time, scoring included. | +| `total_input_tokens`, `total_output_tokens` | Token usage summed over the rounds. | +| `final_summary` | Status line of the scoring run, e.g. `PASS (exit 0) summary='4 passed in ...'`. | +| `error` | The model call's error, when one ended the loop; otherwise `null`. | +| `final_diff` | Unified diff of the workspace since setup. | ## How the loop works @@ -124,18 +118,31 @@ Sample output (`run bugs/01_off_by_one`): | yes v - +-----+-----+ - | PASS | - +-----------+ + +-----------+-----------+ + | score: full original | (also runs when the budget + | suite on a clean copy | runs out) + +-----------------------+ ``` -The agent does NOT see the test file's assertions in its prompt. It learns the bug shape from `read_file("buggy.py")` and the failure trace from the first `run_tests` call. This mirrors what a junior engineer sees on the first day. +The prompt names neither the bug nor its kind. The agent reads `buggy.py` and `test_buggy.py` with `read_file` (the system prompt tells it to) and gets a short traceback (`--tb=short`, last 4,000 characters) from every failing `run_tests` call. + +## Isolation and scoring + +**Files.** Each attempt runs in a fresh `tempfile.mkdtemp(prefix="cfa-")`. Every tool resolves its path, symlinks included, and refuses anything outside the workspace root with the same error whether or not the outside path exists. `run_tests` accepts only a test file inside the workspace (optionally `file::test`), never pytest options. -## Workspace isolation +**Test runs.** The model writes the code that `run_tests` executes, so it is treated as untrusted. `--sandbox` on the CLI and the benchmark picks the runner: -Each bug fix runs in a fresh `tempfile.mkdtemp(prefix="cfa--")`. The agent cannot escape via relative paths (`..`); `edit_file` checks `path.resolve()` falls under the workspace root. Test execution is subprocess-isolated with a 30-second timeout. +| `--sandbox` | Runner | +|---|---| +| `docker` | Throwaway container from `code-fix-agent:latest` (override with `CFA_SANDBOX_IMAGE`): `--network none`, read-only root and workspace mount, all capabilities dropped, `no-new-privileges`, 512 MB / 1 CPU / 128 processes, non-root, no host environment. Fails if the image is missing. | +| `local` | Subprocess with only `PATH`, `SYSTEMROOT` and the temp-dir variables in its environment, in its own session (POSIX) or process group (Windows). The clean environment stops the API key from being inherited, not from being read: the tests run as your user, next to the agent process that holds the key, and can read it from there (on Linux, `/proc//environ`). They also have your file-system and network access. The 30-second timeout kills the test process's group or tree; a process that calls `setsid()`, or on Windows one whose parent already exited, survives it. A fallback, not a sandbox. | +| `auto` (default) | `docker` when `docker image inspect` finds the image. Otherwise `local` for mock runs, and an error for live runs (`code-fix-agent run`, `--mode live`): pass `--sandbox local` to run those without the image. | + +Build the image once with `docker build -t code-fix-agent:latest .`. Every result records the runner used. -This means a single failed agent run cannot corrupt the bug fixtures or any other repo state. +**The Docker runner is untested so far:** it has not run on any machine yet. The CI `docker` job builds the image, runs the container test (`TestDockerSandbox`) and the mock benchmark with `--sandbox docker`, and runs the benchmark inside the image as its non-root user; until that job passes, treat the `docker` row above as a design, not a result. + +**Scoring.** The agent's own `run_tests` result is not the verdict: it chooses which tests to run and can edit the test file. After the loop, the harness rebuilds the bug in a fresh directory from the files present at setup (tests from the setup snapshot, other files from the workspace, nothing the run created) and runs the whole suite with the same runner. That is `passed`, and exit code 0 is not enough: every test collected from the bug as it was at setup must be reported `PASSED` and the counts must hold nothing else, so a `pytest.skip()`, a `pytest.xfail()` or an early `os._exit(0)` in the source fails. The verdict is read from pytest's output, and under either runner the code under test runs inside that pytest process: code written to forge the output (fake result lines, patched reporting) is not caught. `self_reported_pass` keeps the agent's claim, and `tests_modified` flags test files that no longer match their SHA-256 from setup. Benchmark records also keep every tool input and the final diff. ## Testing @@ -143,21 +150,23 @@ This means a single failed agent run cannot corrupt the bug fixtures or any othe pytest --cov=code_fix_agent ``` -22 tests: -- `test_workspace.py` — copy, list, read, edit (success + missing-substring error), run_tests (fails on unfixed, passes on canonical fix). -- `test_tools.py` — schema completeness, dispatch, error surfacing. -- `test_agent.py` — full read-edit-test loop with `MockLLM` on bugs 1 and 2, max-iteration safety, LLM exception handling, token accounting. -- `test_bench_smoke.py` — runs the full mock benchmark (12 bugs) and asserts 12/12 pass. +66 tests (the Docker one is skipped when the image is not built, the symlink one where creating symlinks needs admin rights): +- `test_workspace.py` — copy, list, read, edit (success + missing-substring error), run_tests (fails on unfixed, passes on canonical fix), no inherited secrets, timeout kills child processes, sandbox selection (live runs do not fall back to the local runner) and the Docker command line. +- `test_tools.py` — schema completeness, dispatch, error surfacing, tracebacks in failing output, path confinement for every tool (`..`, absolute paths, symlinks). +- `test_agent.py` — full read-edit-test loop with `MockLLM` on bugs 1 and 2, max-iteration safety, LLM exception handling, token accounting, scoring against subset runs, edited tests, a planted `conftest.py` and source that skips, xfails or exits early; no bug id in the prompt, tool inputs and the diff (created and deleted files included) in the result, API timeout. +- `test_bench_smoke.py` — runs the full mock benchmark (12 bugs) and asserts 12/12 pass; the mock run exits non-zero on any failure and does not overwrite the live record. The bench smoke test is the canary: if it ever drops below 12/12 in CI, either the dataset or the workspace layer regressed. ## Docker ```bash -docker compose run --rm bench # full mock benchmark -docker compose run --rm fix run bugs/01_off_by_one # one bug with live API +docker compose run --rm bench # full mock benchmark +docker compose run --rm fix run --sandbox local bugs/01_off_by_one # one bug with live API ``` +The image runs as a non-root user. It is also the test sandbox image (`code-fix-agent:latest`). Inside a compose container there is no Docker CLI, so tests there use the local runner within that container, which holds the API key: the `fix` service passes `--sandbox local` to accept that. + ## Project layout ``` @@ -178,7 +187,7 @@ src/code_fix_agent/ benchmarks/ run_benchmark.py # iterates all 12, captures stats results/ - benchmark.json + benchmark.json # the live run's record ``` ## What this is NOT diff --git a/benchmarks/run_benchmark.py b/benchmarks/run_benchmark.py index a0e1481..ea13dba 100644 --- a/benchmarks/run_benchmark.py +++ b/benchmarks/run_benchmark.py @@ -25,6 +25,7 @@ from code_fix_agent.agent import Agent from code_fix_agent.llm import AnthropicLLM, LLMClient, MockLLM +from code_fix_agent.sandbox import SANDBOX_MODES, resolve_sandbox from code_fix_agent.workspace import Workspace ROOT = Path(__file__).resolve().parent.parent @@ -118,15 +119,18 @@ def _build_mock_llm(bug_id: str) -> MockLLM: return MockLLM(script=script) -def _run_one(bug: dict, llm: LLMClient, max_iterations: int) -> dict: +def _run_one(bug: dict, llm: LLMClient, max_iterations: int, sandbox: str = "auto") -> dict: bug_dir = BUGS_DIR / bug["id"] - with Workspace.from_bug_dir(bug_dir) as ws: + with Workspace.from_bug_dir(bug_dir, sandbox=sandbox) as ws: agent = Agent(llm=llm, max_iterations=max_iterations) result = agent.fix(ws) return { "id": bug["id"], "kind": bug.get("kind", ""), + "sandbox": ws.sandbox, "passed": result.passed, + "self_reported_pass": result.self_reported_pass, + "tests_modified": result.tests_modified, "iterations": result.iterations, "tool_calls": result.tool_calls, "elapsed_seconds": result.elapsed_seconds, @@ -134,6 +138,8 @@ def _run_one(bug: dict, llm: LLMClient, max_iterations: int) -> dict: "output_tokens": result.total_output_tokens, "final_summary": result.final_summary, "error": result.error, + "final_diff": result.final_diff, + "tool_trail": result.tool_trail, } @@ -147,10 +153,22 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--mode", choices=["mock", "live"], default="mock") parser.add_argument("--max-iterations", type=int, default=8) parser.add_argument("--model", default="claude-haiku-4-5-20251001") - parser.add_argument("--out", type=Path, default=RESULTS_DIR / "benchmark.json") + parser.add_argument( + "--out", + type=Path, + default=None, + help="default: results/benchmark.json for live runs, results/benchmark-mock.json for mock", + ) + parser.add_argument("--sandbox", choices=SANDBOX_MODES, default="auto") args = parser.parse_args(argv) + if args.out is None: + # benchmark.json is the committed live record; a mock run must not replace it. + args.out = RESULTS_DIR / ("benchmark.json" if args.mode == "live" else "benchmark-mock.json") bugs = _load_manifest() + # Live runs execute model-written code, so they only use the local runner + # when asked to; mock runs replay known fixes. + sandbox = resolve_sandbox(args.sandbox, allow_local_fallback=args.mode == "mock") records: list[dict] = [] for bug in bugs: if args.mode == "live": @@ -158,12 +176,13 @@ def main(argv: list[str] | None = None) -> int: else: llm = _build_mock_llm(bug["id"]) print(f" [{args.mode}] {bug['id']}", file=sys.stderr) - records.append(_run_one(bug, llm, args.max_iterations)) + records.append(_run_one(bug, llm, args.max_iterations, sandbox)) n = len(records) passed = sum(1 for r in records if r["passed"]) summary = { "mode": args.mode, + "sandbox": sandbox, "n_bugs": n, "passed": passed, "pass_rate_pct": round(100 * passed / n, 1) if n else 0, @@ -184,7 +203,9 @@ def main(argv: list[str] | None = None) -> int: print("\n=== code-fix-agent benchmark summary ===") print(json.dumps(summary, indent=2)) print(f"\nWrote {args.out}", file=sys.stderr) - return 0 if summary["pass_rate_pct"] >= 75 else 0 # benchmark never fails CI; readme calls out target + # Mock mode replays known fixes, so anything short of 12/12 is a pipeline + # regression. Live results are reported, not enforced. + return 1 if args.mode == "mock" and passed < n else 0 if __name__ == "__main__": diff --git a/docker-compose.yml b/docker-compose.yml index 56dc9d9..2f87418 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: image: code-fix-agent:latest environment: - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} - command: ["run", "bugs/01_off_by_one"] + command: ["run", "--sandbox", "local", "bugs/01_off_by_one"] bench: build: . diff --git a/pyproject.toml b/pyproject.toml index cb52305..c1c1ae8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "code-fix-agent" version = "0.1.0" -description = "Autonomous bug-fix agent on Claude Haiku 4.5. Curated 12-bug Python dataset. Hero: pass-rate target 9/12 (75%) at under 3 iterations per bug, under $0.05 total cost." +description = "Autonomous bug-fix agent on Claude Haiku 4.5. Curated 12-bug Python dataset." readme = "README.md" requires-python = ">=3.10" license = { text = "MIT" } diff --git a/src/code_fix_agent/agent.py b/src/code_fix_agent/agent.py index 1b0a00c..830f08a 100644 --- a/src/code_fix_agent/agent.py +++ b/src/code_fix_agent/agent.py @@ -6,6 +6,8 @@ run tests. 3. Stops when run_tests reports passed, or when the iteration budget is exhausted. +4. Scores the result by running the original test suite on a clean copy + (Workspace.score); the model's own run_tests output is not the verdict. Returns an AgentResult capturing pass/fail, iteration count, total tokens, and the trail of tool calls. @@ -17,7 +19,7 @@ from typing import Any from code_fix_agent.llm import LLMClient -from code_fix_agent.tools import execute, tool_schemas +from code_fix_agent.tools import execute, status_line, tool_schemas from code_fix_agent.workspace import Workspace SYSTEM_PROMPT = ( @@ -41,7 +43,7 @@ @dataclass class AgentResult: bug_id: str - passed: bool + passed: bool # verdict of the original tests, run after the loop iterations: int tool_calls: int elapsed_seconds: float @@ -50,6 +52,9 @@ class AgentResult: tool_trail: list[dict[str, Any]] = field(default_factory=list) final_summary: str = "" error: str | None = None + self_reported_pass: bool = False # the model's own run_tests said PASS + tests_modified: bool = False + final_diff: str = "" def run_agent( @@ -65,10 +70,10 @@ def run_agent( messages: list[dict[str, Any]] = [ { "role": "user", + # No bug id: the ids name the bug class (e.g. 04_mutable_default). "content": ( - f"Workspace contains a buggy Python file. The bug id is " - f"{ws.bug_id!r}. Find and fix the bug so that running pytest " - "on test_buggy.py passes all tests." + "Workspace contains a buggy Python file. Find and fix the bug so " + "that running pytest on test_buggy.py passes all tests." ), } ] @@ -104,7 +109,7 @@ def run_agent( { "iteration": result.iterations, "tool": tool_name, - "input_keys": sorted(tool_input.keys()), + "input": tool_input, "output_preview": output[:160], } ) @@ -116,15 +121,19 @@ def run_agent( } ) if tool_name == "run_tests" and output.startswith("PASS"): - result.passed = True - result.final_summary = output.split("\n", 1)[0] + result.self_reported_pass = True if tool_results: messages.append({"role": "user", "content": tool_results}) - if result.passed or stop_reason != "tool_use": + if result.self_reported_pass or stop_reason != "tool_use": break + verdict = ws.score() + result.passed = verdict["passed"] + result.tests_modified = verdict["tests_modified"] + result.final_summary = status_line(verdict) + result.final_diff = ws.diff() result.elapsed_seconds = round(time.perf_counter() - start, 3) return result diff --git a/src/code_fix_agent/cli.py b/src/code_fix_agent/cli.py index d42bd41..7b44a69 100644 --- a/src/code_fix_agent/cli.py +++ b/src/code_fix_agent/cli.py @@ -13,17 +13,22 @@ from code_fix_agent.agent import Agent from code_fix_agent.llm import AnthropicLLM +from code_fix_agent.sandbox import SANDBOX_MODES, resolve_sandbox from code_fix_agent.workspace import Workspace def _cmd_run(args: argparse.Namespace) -> int: + sandbox = resolve_sandbox(args.sandbox, allow_local_fallback=False) llm = AnthropicLLM(model=args.model) agent = Agent(llm=llm, max_iterations=args.max_iterations) - with Workspace.from_bug_dir(Path(args.bug_dir)) as ws: + with Workspace.from_bug_dir(Path(args.bug_dir), sandbox=sandbox) as ws: result = agent.fix(ws) payload = { "bug_id": result.bug_id, + "sandbox": ws.sandbox, "passed": result.passed, + "self_reported_pass": result.self_reported_pass, + "tests_modified": result.tests_modified, "iterations": result.iterations, "tool_calls": result.tool_calls, "elapsed_seconds": result.elapsed_seconds, @@ -31,6 +36,7 @@ def _cmd_run(args: argparse.Namespace) -> int: "total_output_tokens": result.total_output_tokens, "final_summary": result.final_summary, "error": result.error, + "final_diff": result.final_diff, } print(json.dumps(payload, indent=2)) return 0 if result.passed else 1 @@ -44,6 +50,15 @@ def main(argv: list[str] | None = None) -> int: p_run.add_argument("bug_dir", type=str, help="Path to a bug subdirectory.") p_run.add_argument("--model", default="claude-haiku-4-5-20251001") p_run.add_argument("--max-iterations", type=int, default=8) + p_run.add_argument( + "--sandbox", + choices=SANDBOX_MODES, + default="auto", + help=( + "where tests run: docker (no network, read-only), local (no isolation; the " + "tests can reach the API key), or auto (docker, or an error if the image is missing)" + ), + ) p_run.set_defaults(func=_cmd_run) args = parser.parse_args(argv) diff --git a/src/code_fix_agent/llm.py b/src/code_fix_agent/llm.py index 8863f4d..298be79 100644 --- a/src/code_fix_agent/llm.py +++ b/src/code_fix_agent/llm.py @@ -25,7 +25,12 @@ def respond( class AnthropicLLM: """Wraps anthropic.Anthropic. Returns (content, stop_reason, usage).""" - def __init__(self, model: str = "claude-haiku-4-5-20251001", api_key: str | None = None): + def __init__( + self, + model: str = "claude-haiku-4-5-20251001", + api_key: str | None = None, + timeout: float = 60.0, + ): try: from anthropic import Anthropic except ImportError as exc: @@ -33,7 +38,9 @@ def __init__(self, model: str = "claude-haiku-4-5-20251001", api_key: str | None key = api_key or os.environ.get("ANTHROPIC_API_KEY") if not key: raise RuntimeError("ANTHROPIC_API_KEY not set; use MockLLM in tests") - self._client = Anthropic(api_key=key) + # The SDK default is 10 minutes per request; a stalled call should + # fail the attempt, not hang the benchmark. + self._client = Anthropic(api_key=key, timeout=timeout) self.model = model def respond( diff --git a/src/code_fix_agent/sandbox.py b/src/code_fix_agent/sandbox.py new file mode 100644 index 0000000..0ff4e00 --- /dev/null +++ b/src/code_fix_agent/sandbox.py @@ -0,0 +1,197 @@ +"""Run pytest on model-edited code. + +The code under test is written by the model, so it is treated as untrusted. +Two runners: + +- docker: a throwaway container with no network, a read-only file system and + workspace mount, no capabilities, and memory/CPU/process limits. Nothing + from the caller's environment is passed in. +- local: a subprocess with a minimal environment, in its own session + (POSIX) or process group (Windows); a timeout kills that group or tree, + which misses a process that left the group (setsid) or, on Windows, one + whose parent already exited. The clean environment only stops accidental + inheritance of secrets: the tests run as the caller's user, next to the + agent process that holds ANTHROPIC_API_KEY, and can read it from there + (on Linux, from /proc//environ). They also have the caller's + file-system and network access. It is a fallback, not a sandbox. + +"auto" picks docker when the `docker` CLI can see the sandbox image. Live +runs, where the model writes the code, turn off auto's local fallback, so +running them without the image takes an explicit "local". +""" +from __future__ import annotations + +import os +import shutil +import signal +import subprocess +import sys +import uuid +from collections.abc import Callable +from dataclasses import dataclass +from functools import cache +from pathlib import Path + +PYTEST_ARGS = ["-q", "--no-header", "--tb=short", "-p", "no:cacheprovider"] +SANDBOX_IMAGE = os.environ.get("CFA_SANDBOX_IMAGE", "code-fix-agent:latest") +SANDBOX_MODES = ("auto", "docker", "local") + +# Python on Windows cannot start without SYSTEMROOT; the temp variables only +# say where scratch files go. Nothing else is passed through. +_ENV_ALLOWLIST = ("PATH", "SYSTEMROOT", "TEMP", "TMP", "TMPDIR") + + +@dataclass +class RunResult: + exit_code: int | None # None when the run hit the timeout + stdout: str + stderr: str + + +@cache +def docker_available(image: str = SANDBOX_IMAGE) -> bool: + if shutil.which("docker") is None: + return False + try: + probe = subprocess.run( + ["docker", "image", "inspect", image], capture_output=True, timeout=30, check=False + ) + except (OSError, subprocess.TimeoutExpired): + return False + return probe.returncode == 0 + + +def resolve_sandbox( + mode: str, image: str = SANDBOX_IMAGE, *, allow_local_fallback: bool = True +) -> str: + """Map a requested mode to the runner that will be used: "docker" or "local". + + With allow_local_fallback=False, "auto" fails instead of falling back to + the local runner when the image is missing. + """ + if mode not in SANDBOX_MODES: + raise ValueError(f"sandbox must be one of {SANDBOX_MODES}, got {mode!r}") + if mode == "local": + return "local" + if docker_available(image): + return "docker" + if mode == "docker": + raise RuntimeError( + f"docker sandbox requested but image {image!r} is not available " + f"(build it with `docker build -t {image} .`)" + ) + if not allow_local_fallback: + raise RuntimeError( + f"sandbox image {image!r} is not available. Under the local runner, " + "model-written code runs as your user and can read ANTHROPIC_API_KEY " + f"from the agent process. Build the image with `docker build -t {image} .`, " + "or pass --sandbox local to run without it." + ) + return "local" + + +def minimal_env() -> dict[str, str]: + env = {key: os.environ[key] for key in _ENV_ALLOWLIST if key in os.environ} + env["PYTHONIOENCODING"] = "utf-8" + return env + + +def docker_command(cwd: Path, args: list[str], *, name: str, image: str = SANDBOX_IMAGE) -> list[str]: + cmd = [ + "docker", "run", "--rm", "--name", name, + "--network", "none", + "--read-only", "--tmpfs", "/tmp", + "--memory", "512m", "--pids-limit", "128", "--cpus", "1", + "--cap-drop", "ALL", "--security-opt", "no-new-privileges", + "--mount", f"type=bind,source={cwd},target=/work,readonly", + "--workdir", "/work", + "--entrypoint", "python", + ] + if hasattr(os, "getuid"): + # The temp workspace is private to the caller's uid on POSIX. + cmd += ["--user", f"{os.getuid()}:{os.getgid()}"] + return [*cmd, image, "-B", "-m", "pytest", *PYTEST_ARGS, *args] + + +def run_pytest( + cwd: Path, args: list[str], *, timeout: float, sandbox: str = "local", image: str = SANDBOX_IMAGE +) -> RunResult: + """Run pytest on `cwd` with the given runner ("docker" or "local").""" + if sandbox == "docker": + name = f"cfa-{uuid.uuid4().hex[:12]}" + # The docker client needs the caller's environment to reach the + # daemon; none of it is forwarded into the container. + return _run( + docker_command(cwd, args, name=name, image=image), + cwd=None, + env=None, + timeout=timeout, + on_timeout=lambda: _docker_kill(name), + ) + if sandbox != "local": + raise ValueError(f"unknown runner: {sandbox!r}") + return _run( + [sys.executable, "-B", "-m", "pytest", *PYTEST_ARGS, *args], + cwd=cwd, + env=minimal_env(), + timeout=timeout, + ) + + +def _run( + cmd: list[str], + *, + cwd: Path | None, + env: dict[str, str] | None, + timeout: float, + on_timeout: Callable[[], object] | None = None, +) -> RunResult: + group = ( + {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} + if os.name == "nt" + else {"start_new_session": True} + ) + proc = subprocess.Popen( + cmd, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + **group, + ) + try: + stdout, stderr = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + if on_timeout is not None: + on_timeout() + _kill_tree(proc) + try: + stdout, _ = proc.communicate(timeout=5) + except subprocess.TimeoutExpired: + stdout = "" + return RunResult(None, stdout or "", "timeout") + return RunResult(proc.returncode, stdout, stderr) + + +def _docker_kill(name: str) -> None: + # Killing the docker client does not stop the container. + try: + subprocess.run(["docker", "kill", name], capture_output=True, timeout=30, check=False) + except (OSError, subprocess.TimeoutExpired): + pass + + +def _kill_tree(proc: subprocess.Popen) -> None: + if os.name == "nt": + subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(proc.pid)], capture_output=True, check=False + ) + else: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.kill() diff --git a/src/code_fix_agent/tools.py b/src/code_fix_agent/tools.py index ac79ca3..1db145e 100644 --- a/src/code_fix_agent/tools.py +++ b/src/code_fix_agent/tools.py @@ -92,6 +92,11 @@ def tool_schemas() -> list[dict[str, Any]]: ] +def status_line(result: dict[str, Any]) -> str: + status = "PASS" if result["passed"] else "FAIL" + return f"{status} (exit {result['exit_code']}) summary={result['summary']!r}" + + def execute(ws: Workspace, tool_name: str, tool_input: dict[str, Any]) -> str: """Dispatch one tool call against the workspace. Returns a string result.""" try: @@ -117,9 +122,8 @@ def execute(ws: Workspace, tool_name: str, tool_input: dict[str, Any]) -> str: if tool_name == "run_tests": result = ws.run_tests(tool_input.get("test_file", "test_buggy.py")) - status = "PASS" if result["passed"] else "FAIL" body = result["stdout"] or result["stderr"] - return f"{status} (exit {result['exit_code']}) summary={result['summary']!r}\n{body}" + return f"{status_line(result)}\n{body}" return f"unknown tool: {tool_name}" except FileNotFoundError as exc: diff --git a/src/code_fix_agent/workspace.py b/src/code_fix_agent/workspace.py index 1767766..f18fa0b 100644 --- a/src/code_fix_agent/workspace.py +++ b/src/code_fix_agent/workspace.py @@ -7,14 +7,18 @@ """ from __future__ import annotations +import difflib +import hashlib +import re import shutil -import subprocess -import sys import tempfile from contextlib import contextmanager from dataclasses import dataclass, field +from functools import cache from pathlib import Path +from code_fix_agent.sandbox import RunResult, resolve_sandbox, run_pytest + @dataclass class Workspace: @@ -31,17 +35,27 @@ class Workspace: root: Path bug_id: str = "" _cleanup: bool = field(default=False, repr=False) + # "auto", "docker" or "local"; resolved at construction to the runner used. + sandbox: str = "auto" + # Top-level files as they were at setup; score() rebuilds the bug from these. + originals: dict[str, bytes] = field(init=False, repr=False) + + def __post_init__(self) -> None: + self.sandbox = resolve_sandbox(self.sandbox) + self.originals = {p.name: p.read_bytes() for p in self.root.iterdir() if p.is_file()} # --- creation --------------------------------------------------------- @classmethod - def from_bug_dir(cls, bug_dir: Path, cleanup: bool = True) -> Workspace: + def from_bug_dir(cls, bug_dir: Path, cleanup: bool = True, sandbox: str = "auto") -> Workspace: bug_dir = Path(bug_dir) - tmp = Path(tempfile.mkdtemp(prefix=f"cfa-{bug_dir.name}-")) + sandbox = resolve_sandbox(sandbox) # fail before creating the temp dir + # The bug id stays out of the path, which shows up in tracebacks. + tmp = Path(tempfile.mkdtemp(prefix="cfa-")) for child in bug_dir.iterdir(): if child.is_file(): shutil.copy2(child, tmp / child.name) - return cls(root=tmp, bug_id=bug_dir.name, _cleanup=cleanup) + return cls(root=tmp, bug_id=bug_dir.name, _cleanup=cleanup, sandbox=sandbox) def __enter__(self) -> Workspace: return self @@ -52,27 +66,35 @@ def __exit__(self, exc_type, exc, tb) -> None: # --- tools ------------------------------------------------------------ + def resolve(self, rel: str) -> Path: + """Resolve `rel` (symlinks included) and refuse anything outside the root. + + Every tool goes through this before touching the file system, so an + error never reveals whether an outside path exists. + """ + root = self.root.resolve() + path = (root / rel).resolve() + if not path.is_relative_to(root): + raise PermissionError(f"path escapes workspace: {rel}") + return path + def list_dir(self, rel: str = ".") -> list[str]: - target = (self.root / rel).resolve() + target = self.resolve(rel) if not target.is_dir(): raise FileNotFoundError(f"not a directory: {rel}") return sorted(p.name for p in target.iterdir()) def read(self, rel: str) -> str: - path = (self.root / rel).resolve() + path = self.resolve(rel) if not path.is_file(): raise FileNotFoundError(f"not a file: {rel}") - if self.root not in path.parents and self.root != path.parent: - raise PermissionError(f"path escapes workspace: {rel}") return path.read_text(encoding="utf-8") def edit(self, rel: str, old: str, new: str) -> str: """str_replace edit. Errors if `old` is not found or not unique.""" - path = (self.root / rel).resolve() + path = self.resolve(rel) if not path.is_file(): raise FileNotFoundError(f"not a file: {rel}") - if self.root not in path.parents and self.root != path.parent: - raise PermissionError(f"path escapes workspace: {rel}") text = path.read_text(encoding="utf-8") if old not in text: raise ValueError("old string not found in file") @@ -84,51 +106,161 @@ def edit(self, rel: str, old: str, new: str) -> str: def search(self, pattern: str, rel: str = ".") -> list[tuple[str, int, str]]: """Substring search across files in the workspace (Python files only).""" - target = (self.root / rel).resolve() + root = self.root.resolve() + target = self.resolve(rel) out: list[tuple[str, int, str]] = [] for path in target.rglob("*.py"): + path = path.resolve() + if not path.is_relative_to(root) or not path.is_file(): + continue for i, line in enumerate(path.read_text(encoding="utf-8", errors="ignore").splitlines(), 1): if pattern in line: - out.append((path.relative_to(self.root).as_posix(), i, line.rstrip())) + out.append((path.relative_to(root).as_posix(), i, line.rstrip())) return out def run_tests(self, test_file: str = "test_buggy.py", timeout: float = 30.0) -> dict: """Run pytest on test_file. Returns dict with passed/failed counts and output.""" - cmd = [sys.executable, "-m", "pytest", "-q", "--no-header", "--no-summary", test_file] - try: - proc = subprocess.run( - cmd, - cwd=self.root, - capture_output=True, - text=True, - timeout=timeout, - check=False, + file_part, sep, node = test_file.partition("::") + path = self.resolve(file_part) + if not path.is_file(): + raise FileNotFoundError(f"not a file: {file_part}") + target = path.relative_to(self.root.resolve()).as_posix() + sep + node + return _as_result(run_pytest(self.root, [target], timeout=timeout, sandbox=self.sandbox)) + + def diff(self) -> str: + """Unified diff of every file changed, created or deleted since setup. + + Created files, in subdirectories too, diff from /dev/null: code the + tests ran can plant a conftest.py or a new module, and the record + should show it. + """ + current = { + path.relative_to(self.root).as_posix(): path.read_bytes() + for path in self.root.rglob("*") + if path.is_file() + } + chunks: list[str] = [] + for name in sorted(self.originals.keys() | current.keys()): + before, after = self.originals.get(name), current.get(name) + if before == after: + continue + chunks.extend( + difflib.unified_diff( + _lines(before), + _lines(after), + fromfile=f"a/{name}" if before is not None else "/dev/null", + tofile=f"b/{name}" if after is not None else "/dev/null", + ) ) - except subprocess.TimeoutExpired: - return { - "passed": False, - "exit_code": -1, - "stdout": "", - "stderr": "timeout", - "summary": "timeout", - } - stdout = proc.stdout - stderr = proc.stderr - passed = proc.returncode == 0 - # Pull the last non-empty line of stdout as a summary; pytest prints - # "X passed in Y.YYs" or "X failed, Y passed in Z.ZZs" there. - summary = "" - for line in reversed(stdout.splitlines()): - if line.strip(): - summary = line.strip() - break + return "".join(chunks) + + # --- scoring ---------------------------------------------------------- + + def score(self, timeout: float = 30.0) -> dict: + """Run the original test suite against the workspace's current source. + + The model can edit the test file, run a subset of it, or (through the + code its tests execute) drop a conftest.py into the workspace, so its + own run_tests result is not a score. This rebuilds the bug in a fresh + directory from the files present at setup, taking the tests from the + setup snapshot and every other file from the workspace, and runs the + whole suite. `tests_modified` reports whether the workspace's test + files still match the snapshot's SHA-256. + + Exit code 0 is not enough: the code under test can skip or xfail the + tests, or exit early, and pytest still exits 0. A pass needs every + test collected from the bug as it was at setup to be reported PASSED, + and nothing else (failed, skipped, xfailed, errors) in the counts. + """ + tests = tuple(sorted(name for name in self.originals if _is_test_file(name))) + expected = _original_tests( + tuple(sorted(self.originals.items())), tests, self.sandbox, timeout + ) + with tempfile.TemporaryDirectory(prefix="cfa-score-", ignore_cleanup_errors=True) as tmp: + dest = Path(tmp) + for name, data in self.originals.items(): + current = self.root / name + if name in tests: + (dest / name).write_bytes(data) + elif current.is_file(): + shutil.copyfile(current, dest / name) + run = run_pytest(dest, ["-rfEsxXp", *tests], timeout=timeout, sandbox=self.sandbox) + result = _as_result(run) + result["passed"] = result["passed"] and _only_these_passed(run.stdout, expected) + result["tests_modified"] = any( + _sha256(self.root / name) != hashlib.sha256(self.originals[name]).hexdigest() + for name in tests + ) + return result + + +@cache +def _original_tests( + files: tuple[tuple[str, bytes], ...], tests: tuple[str, ...], sandbox: str, timeout: float +) -> tuple[str, ...]: + """Node ids pytest collects from `files`, the bug as it was at setup. + + Cached on the file contents: every attempt at the same bug collects the + same tests. + """ + with tempfile.TemporaryDirectory(prefix="cfa-collect-", ignore_cleanup_errors=True) as tmp: + for name, data in files: + (Path(tmp) / name).write_bytes(data) + run = run_pytest(Path(tmp), ["--collect-only", *tests], timeout=timeout, sandbox=sandbox) + ids = [] + for line in run.stdout.splitlines(): + file_part, sep, _ = line.strip().partition("::") + if sep and file_part in tests: + ids.append(line.strip()) + if run.exit_code != 0 or not ids: + raise RuntimeError(f"could not collect the original tests: {run.stdout[-1000:]!r}") + return tuple(ids) + + +def _only_these_passed(stdout: str, expected: tuple[str, ...]) -> bool: + """True if pytest's -rp lines name exactly `expected` and its counts are all passes.""" + lines = stdout.splitlines() + passed = {line.removeprefix("PASSED ").strip() for line in lines if line.startswith("PASSED ")} + # The last line reads e.g. "4 passed, 1 warning in 0.02s". + summary = next((line.strip() for line in reversed(lines) if line.strip()), "") + counts = {kind: int(n) for n, kind in re.findall(r"(\d+) (\w+)", summary.split(" in ")[0])} + counts.pop("warning", None) + counts.pop("warnings", None) + return passed == set(expected) and counts == {"passed": len(expected)} + + +def _is_test_file(name: str) -> bool: + return name == "conftest.py" or ( + name.endswith(".py") and (name.startswith("test_") or name.endswith("_test.py")) + ) + + +def _lines(data: bytes | None) -> list[str]: + return (data or b"").decode("utf-8", errors="replace").splitlines(keepends=True) + + +def _sha256(path: Path) -> str | None: + return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None + + +def _as_result(run: RunResult) -> dict: + if run.exit_code is None: return { - "passed": passed, - "exit_code": proc.returncode, - "stdout": stdout[-4000:], # truncate to keep tool result small - "stderr": stderr[-2000:], - "summary": summary, + "passed": False, + "exit_code": -1, + "stdout": run.stdout[-4000:], + "stderr": "timeout", + "summary": "timeout", } + # pytest prints "X passed in Y.YYs" or "X failed, Y passed in Z.ZZs" last. + summary = next((line.strip() for line in reversed(run.stdout.splitlines()) if line.strip()), "") + return { + "passed": run.exit_code == 0, + "exit_code": run.exit_code, + "stdout": run.stdout[-4000:], # truncate to keep tool result small + "stderr": run.stderr[-2000:], + "summary": summary, + } @contextmanager diff --git a/tests/test_agent.py b/tests/test_agent.py index 3c4794a..6ca7de8 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -3,6 +3,8 @@ from pathlib import Path +import pytest + from code_fix_agent.agent import Agent, run_agent from code_fix_agent.llm import MockLLM from code_fix_agent.workspace import Workspace @@ -95,3 +97,149 @@ def test_token_counts_accumulate(self): result = run_agent(ws, llm) assert result.total_input_tokens > 0 assert result.total_output_tokens > 0 + + +class TestIndependentScoring: + """The verdict comes from the original tests, not from the model's own run.""" + + def test_running_only_an_already_passing_test_is_not_a_fix(self): + llm = MockLLM( + script=[ + ( + [ + { + "type": "tool_use", + "id": "tu_test", + "name": "run_tests", + "input": {"test_file": "test_buggy.py::test_zero"}, + } + ], + "tool_use", + ) + ] + ) + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + result = run_agent(ws, llm) + assert result.self_reported_pass is True + assert result.passed is False + assert "3 failed, 1 passed" in result.final_summary + + def test_editing_the_tests_is_not_a_fix(self): + rewrite = { + "path": "test_buggy.py", + "old": "from buggy import sum_first_n", + "new": "def sum_first_n(n):\n return n * (n + 1) // 2", + } + llm = MockLLM( + script=[ + ([{"type": "tool_use", "id": "tu_edit", "name": "edit_file", "input": rewrite}], "tool_use"), + ([{"type": "tool_use", "id": "tu_test", "name": "run_tests", "input": {}}], "tool_use"), + ] + ) + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + result = run_agent(ws, llm) + assert result.self_reported_pass is True + assert result.tests_modified is True + assert result.passed is False + + def test_a_real_fix_scores_as_passed_with_tests_untouched(self): + script = _build_canonical_script("01_off_by_one", "range(n)", "range(n + 1)") + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + result = run_agent(ws, MockLLM(script=script)) + assert result.passed is True + assert result.self_reported_pass is True + assert result.tests_modified is False + assert result.final_summary.startswith("PASS (exit 0) summary='4 passed") + + def test_score_ignores_files_created_during_the_run(self): + with Workspace.from_bug_dir(_bug_dir("01_off_by_one"), sandbox="local") as ws: + (ws.root / "conftest.py").write_text( + "import buggy\nbuggy.sum_first_n = lambda n: n * (n + 1) // 2\n", encoding="utf-8" + ) + assert ws.run_tests()["passed"] is True + verdict = ws.score() + assert verdict["passed"] is False + assert verdict["summary"].startswith("3 failed, 1 passed") + + @pytest.mark.parametrize( + ("old", "new", "summary"), + [ + (" total = 0", " __import__('pytest').skip('x')\n total = 0", "4 skipped"), + (" total = 0", " __import__('pytest').xfail('x')\n total = 0", "4 xfailed"), + ( + "def sum_first_n", + "import pytest\npytest.skip('x', allow_module_level=True)\ndef sum_first_n", + "1 skipped", + ), + ("def sum_first_n", "import os\nos._exit(0)\ndef sum_first_n", ""), + ], + ids=["skip", "xfail", "module-skip", "exit-0"], + ) + def test_source_that_dodges_the_tests_is_not_a_fix(self, old, new, summary): + # Each edit touches only buggy.py and leaves pytest exiting 0 or 5, + # with no test having passed. + with Workspace.from_bug_dir(_bug_dir("01_off_by_one"), sandbox="local") as ws: + ws.edit("buggy.py", old, new) + verdict = ws.score() + assert verdict["passed"] is False + assert verdict["tests_modified"] is False + assert verdict["summary"].startswith(summary) + + def test_warnings_do_not_fail_a_real_fix(self): + with Workspace.from_bug_dir(_bug_dir("01_off_by_one"), sandbox="local") as ws: + ws.edit("buggy.py", "range(n)", "range(n + 1)") + ws.edit("buggy.py", "def sum_first_n", "import warnings\nwarnings.warn('w')\ndef sum_first_n") + verdict = ws.score() + assert verdict["passed"] is True, verdict["stdout"] + assert verdict["summary"].startswith("4 passed, 1 warning") + + +class TestNoBenchmarkHints: + def test_prompt_and_paths_do_not_name_the_bug(self): + llm = MockLLM(script=[([{"type": "tool_use", "id": "tu_ls", "name": "list_dir", "input": {}}], "tool_use")]) + with Workspace.from_bug_dir(_bug_dir("04_mutable_default")) as ws: + run_agent(ws, llm) + root_name = ws.root.name + first_prompt = llm.last_messages[0][0]["content"] + for hint in ("04_mutable_default", "mutable", "default"): + assert hint not in first_prompt + assert hint not in root_name + + +class TestAuditTrail: + def test_result_records_tool_inputs_and_the_final_diff(self): + script = _build_canonical_script("01_off_by_one", "range(n)", "range(n + 1)") + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + result = run_agent(ws, MockLLM(script=script)) + assert [step["tool"] for step in result.tool_trail] == ["read_file", "edit_file", "run_tests"] + assert result.tool_trail[1]["input"] == {"path": "buggy.py", "old": "range(n)", "new": "range(n + 1)"} + assert "--- a/buggy.py" in result.final_diff + assert "- for i in range(n):" in result.final_diff + assert "+ for i in range(n + 1):" in result.final_diff + assert "test_buggy.py" not in result.final_diff + + def test_diff_shows_created_and_deleted_files(self): + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + (ws.root / "conftest.py").write_bytes(b"import buggy\n") + (ws.root / "pkg").mkdir() + (ws.root / "pkg" / "helper.py").write_bytes(b"X = 1\n") + (ws.root / "test_buggy.py").unlink() + diff = ws.diff() + assert "--- /dev/null\n+++ b/conftest.py\n" in diff + assert "+import buggy\n" in diff + assert "--- /dev/null\n+++ b/pkg/helper.py\n" in diff + assert "--- a/test_buggy.py\n+++ /dev/null\n" in diff + assert "buggy.py\n+++ b/buggy.py" not in diff + + def test_no_edit_means_an_empty_diff(self): + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + result = run_agent(ws, MockLLM(script=[])) + assert result.final_diff == "" + + +class TestAnthropicClient: + def test_requests_have_an_explicit_timeout(self): + from code_fix_agent.llm import AnthropicLLM + + assert AnthropicLLM(api_key="sk-ant-test")._client.timeout == 60.0 + assert AnthropicLLM(api_key="sk-ant-test", timeout=5.0)._client.timeout == 5.0 diff --git a/tests/test_bench_smoke.py b/tests/test_bench_smoke.py index 5853847..8788ffc 100644 --- a/tests/test_bench_smoke.py +++ b/tests/test_bench_smoke.py @@ -6,6 +6,7 @@ """ from __future__ import annotations +from benchmarks import run_benchmark from benchmarks.run_benchmark import _MOCK_FIXES, _build_mock_llm, _load_manifest, _run_one @@ -25,7 +26,36 @@ def test_all_12_bugs_pass_in_mock_mode(self): f"failures: {[r['id'] for r in records if not r['passed']]}" ) + def test_records_carry_the_tool_trail_and_diff(self): + bug = _load_manifest()[0] + record = _run_one(bug, _build_mock_llm(bug["id"]), max_iterations=8) + old, new = _MOCK_FIXES[bug["id"]] + assert record["tool_trail"][1]["input"] == {"path": "buggy.py", "old": old, "new": new} + assert record["final_diff"].startswith("--- a/buggy.py") + def test_manifest_has_a_canonical_fix_for_every_bug(self): bugs = _load_manifest() for bug in bugs: assert bug["id"] in _MOCK_FIXES, f"missing canonical fix for {bug['id']}" + + +class TestExitCode: + def test_mock_run_fails_when_a_scripted_fix_stops_working(self, monkeypatch, tmp_path): + monkeypatch.setitem(_MOCK_FIXES, "01_off_by_one", ("range(n)", "range(n - 1)")) + monkeypatch.setattr(run_benchmark, "_load_manifest", lambda: [{"id": "01_off_by_one"}]) + assert run_benchmark.main(["--mode", "mock", "--out", str(tmp_path / "b.json")]) == 1 + + def test_mock_run_succeeds_when_every_fix_passes(self, monkeypatch, tmp_path): + monkeypatch.setattr(run_benchmark, "_load_manifest", lambda: [{"id": "01_off_by_one"}]) + assert run_benchmark.main(["--mode", "mock", "--out", str(tmp_path / "b.json")]) == 0 + + +class TestOutputPath: + def test_mock_run_does_not_replace_the_live_record(self, monkeypatch, tmp_path): + monkeypatch.setattr(run_benchmark, "RESULTS_DIR", tmp_path) + monkeypatch.setattr(run_benchmark, "_load_manifest", lambda: [{"id": "01_off_by_one"}]) + live_record = tmp_path / "benchmark.json" + live_record.write_text('{"summary": {"mode": "live"}}', encoding="utf-8") + assert run_benchmark.main(["--mode", "mock"]) == 0 + assert live_record.read_text(encoding="utf-8") == '{"summary": {"mode": "live"}}' + assert '"mode": "mock"' in (tmp_path / "benchmark-mock.json").read_text(encoding="utf-8") diff --git a/tests/test_tools.py b/tests/test_tools.py index ec65e84..4ef6d27 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -1,8 +1,11 @@ """Tool dispatcher tests.""" from __future__ import annotations +import os from pathlib import Path +import pytest + from code_fix_agent.tools import execute, tool_schemas from code_fix_agent.workspace import Workspace @@ -40,6 +43,14 @@ def test_edit_file_then_run_tests_pass(self): out = execute(ws, "run_tests", {"test_file": "test_buggy.py"}) assert out.startswith("PASS") + def test_failing_run_shows_the_assertion_that_failed(self): + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + out = execute(ws, "run_tests", {}) + assert out.startswith("FAIL") + assert "assert sum_first_n(5) == 15" in out + assert "assert 10 == 15" in out + assert out.strip().splitlines()[-1].startswith("3 failed, 1 passed") + def test_edit_file_error_surfaces(self): with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: out = execute(ws, "edit_file", {"path": "buggy.py", "old": "missing", "new": "x"}) @@ -54,3 +65,67 @@ def test_search_code(self): with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: out = execute(ws, "search_code", {"pattern": "sum_first_n"}) assert "buggy.py" in out + + +class TestPathConfinement: + """Every tool refuses paths that resolve outside the workspace root.""" + + @pytest.mark.parametrize( + ("tool", "tool_input"), + [ + ("list_dir", {"path": ".."}), + ("list_dir", {"path": "../.."}), + ("read_file", {"path": "../buggy.py"}), + ("edit_file", {"path": "../buggy.py", "old": "a", "new": "b"}), + ("search_code", {"pattern": "def ", "path": ".."}), + ("run_tests", {"test_file": "../test_buggy.py"}), + ("run_tests", {"test_file": "../test_buggy.py::test_one"}), + ], + ) + def test_relative_escape_is_refused(self, tool, tool_input): + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + out = execute(ws, tool, tool_input) + assert out.startswith("error: path escapes workspace"), out + + @pytest.mark.parametrize( + ("tool", "key"), + [("list_dir", "path"), ("read_file", "path"), ("search_code", "path"), ("run_tests", "test_file")], + ) + def test_absolute_path_outside_is_refused(self, tool, key): + outside = _bug_dir("01_off_by_one") + target = outside if tool in ("list_dir", "search_code") else outside / "test_buggy.py" + tool_input = {key: str(target)} + if tool == "search_code": + tool_input["pattern"] = "def " + with Workspace.from_bug_dir(outside) as ws: + out = execute(ws, tool, tool_input) + assert out.startswith("error: path escapes workspace"), out + assert "buggy.py" not in out.replace(str(target), "") + + def test_outside_files_are_not_distinguishable_from_missing_ones(self, tmp_path): + real = tmp_path / "real.py" + real.write_text("x = 1\n", encoding="utf-8") + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + exists = execute(ws, "read_file", {"path": os.path.relpath(real, ws.root)}) + missing = execute(ws, "read_file", {"path": os.path.relpath(tmp_path / "nope.py", ws.root)}) + assert exists.startswith("error: path escapes workspace"), exists + assert missing.startswith("error: path escapes workspace"), missing + + def test_symlink_pointing_outside_is_refused(self, tmp_path): + secret = tmp_path / "secret.py" + secret.write_text("TOKEN = 'outside'\n", encoding="utf-8") + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + link = ws.root / "link.py" + try: + link.symlink_to(secret) + except OSError: + pytest.skip("creating symlinks is not permitted here") + assert execute(ws, "read_file", {"path": "link.py"}).startswith("error: path escapes") + assert "outside" not in execute(ws, "search_code", {"pattern": "TOKEN"}) + + def test_paths_inside_the_workspace_still_work(self): + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + assert "buggy.py" in execute(ws, "list_dir", {"path": "./"}) + assert "def sum_first_n" in execute(ws, "read_file", {"path": "./buggy.py"}) + out = execute(ws, "run_tests", {"test_file": "test_buggy.py::test_zero"}) + assert out.startswith("PASS"), out diff --git a/tests/test_workspace.py b/tests/test_workspace.py index 84d4822..f9497da 100644 --- a/tests/test_workspace.py +++ b/tests/test_workspace.py @@ -1,10 +1,14 @@ """Workspace tests.""" from __future__ import annotations +import textwrap +import time from pathlib import Path import pytest +from benchmarks import run_benchmark +from code_fix_agent import cli, sandbox from code_fix_agent.workspace import Workspace @@ -50,3 +54,139 @@ def test_search_finds_pattern(self): with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: hits = ws.search("range") assert any(h[2].strip().startswith("for i in range") for h in hits) + + +def _write_bug(dir_: Path, test_source: str) -> Path: + (dir_ / "buggy.py").write_text("def f():\n return 1\n", encoding="utf-8") + (dir_ / "test_buggy.py").write_text(textwrap.dedent(test_source), encoding="utf-8") + return dir_ + + +class TestTestRunIsolation: + def test_tests_do_not_inherit_secrets(self, tmp_path, monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-test-not-a-key") + monkeypatch.setenv("CFA_PROBE_SECRET", "visible") + bug = _write_bug( + tmp_path, + """ + import os + + def test_no_secrets(): + assert os.environ.get("ANTHROPIC_API_KEY") is None + assert os.environ.get("CFA_PROBE_SECRET") is None + """, + ) + with Workspace.from_bug_dir(bug, sandbox="local") as ws: + result = ws.run_tests() + assert result["passed"] is True, result["stdout"] + + def test_timeout_kills_processes_the_tests_started(self, tmp_path): + started = tmp_path / "started" + late = tmp_path / "late" + timeout = 20 + # Leave pytest plenty of time to start, and have the child write + # `late` at a fixed time after the deadline, however late it began. + late_at = time.time() + timeout + 2 + child = ( + "import pathlib, time; " + f"pathlib.Path({str(started)!r}).write_text('x'); " + f"time.sleep(max(0, {late_at} - time.time())); " + f"pathlib.Path({str(late)!r}).write_text('x')" + ) + (tmp_path / "bug").mkdir() + bug = _write_bug( + tmp_path / "bug", + f""" + import subprocess, sys, time + + def test_hangs(): + subprocess.Popen([sys.executable, "-c", {child!r}]) + time.sleep(60) + """, + ) + with Workspace.from_bug_dir(bug, sandbox="local") as ws: + result = ws.run_tests(timeout=timeout) + time.sleep(max(0, late_at + 2 - time.time())) + assert result["summary"] == "timeout" + assert started.exists(), "the test never started its child process" + assert not late.exists(), "a process started by the tests outlived the timeout" + + +def _no_model(*args, **kwargs): + raise AssertionError("the run reached the model before checking the sandbox") + + +class TestSandboxSelection: + def test_auto_falls_back_to_local_without_the_image(self, monkeypatch): + monkeypatch.setattr(sandbox, "docker_available", lambda image=sandbox.SANDBOX_IMAGE: False) + assert sandbox.resolve_sandbox("auto") == "local" + + def test_auto_uses_docker_when_the_image_exists(self, monkeypatch): + monkeypatch.setattr(sandbox, "docker_available", lambda image=sandbox.SANDBOX_IMAGE: True) + assert sandbox.resolve_sandbox("auto") == "docker" + + def test_explicit_docker_fails_before_creating_a_workspace(self, monkeypatch, tmp_path): + monkeypatch.setattr(sandbox, "docker_available", lambda image=sandbox.SANDBOX_IMAGE: False) + monkeypatch.setattr("tempfile.tempdir", str(tmp_path)) + with pytest.raises(RuntimeError, match="not available"): + Workspace.from_bug_dir(_bug_dir("01_off_by_one"), sandbox="docker") + assert list(tmp_path.iterdir()) == [] + + def test_unknown_mode_is_rejected(self): + with pytest.raises(ValueError): + sandbox.resolve_sandbox("chroot") + + def test_auto_without_the_image_refuses_when_local_fallback_is_off(self, monkeypatch): + monkeypatch.setattr(sandbox, "docker_available", lambda image=sandbox.SANDBOX_IMAGE: False) + with pytest.raises(RuntimeError, match="--sandbox local"): + sandbox.resolve_sandbox("auto", allow_local_fallback=False) + assert sandbox.resolve_sandbox("local", allow_local_fallback=False) == "local" + + def test_live_cli_run_does_not_fall_back_to_local(self, monkeypatch): + monkeypatch.setattr(sandbox, "docker_available", lambda image=sandbox.SANDBOX_IMAGE: False) + monkeypatch.setattr(cli, "AnthropicLLM", _no_model) + with pytest.raises(RuntimeError, match="--sandbox local"): + cli.main(["run", str(_bug_dir("01_off_by_one"))]) + + def test_live_benchmark_does_not_fall_back_to_local(self, monkeypatch, tmp_path): + monkeypatch.setattr(sandbox, "docker_available", lambda image=sandbox.SANDBOX_IMAGE: False) + monkeypatch.setattr(run_benchmark, "AnthropicLLM", _no_model) + with pytest.raises(RuntimeError, match="--sandbox local"): + run_benchmark.main(["--mode", "live", "--out", str(tmp_path / "b.json")]) + assert not (tmp_path / "b.json").exists() + + def test_docker_command_shuts_off_network_env_and_writes(self, tmp_path): + cmd = sandbox.docker_command(tmp_path, ["test_buggy.py"], name="cfa-test") + pairs = set(zip(cmd, cmd[1:])) + assert ("--network", "none") in pairs + assert ("--cap-drop", "ALL") in pairs + assert ("--security-opt", "no-new-privileges") in pairs + assert "--read-only" in cmd + assert f"type=bind,source={tmp_path},target=/work,readonly" in cmd + assert not {"-e", "--env", "--env-file", "--privileged"} & set(cmd) + image_at = cmd.index(sandbox.SANDBOX_IMAGE) + assert cmd[image_at + 1 :] == ["-B", "-m", "pytest", *sandbox.PYTEST_ARGS, "test_buggy.py"] + + +@pytest.mark.skipif(not sandbox.docker_available(), reason="sandbox image not built") +class TestDockerSandbox: + def test_container_has_no_network_and_no_secrets(self, tmp_path, monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-test-not-a-key") + bug = _write_bug( + tmp_path, + """ + import os, socket + + import pytest + + def test_isolated(): + assert os.environ.get("ANTHROPIC_API_KEY") is None + with pytest.raises(OSError): + socket.create_connection(("1.1.1.1", 53), timeout=3) + with pytest.raises(OSError): + open("written.txt", "w") + """, + ) + with Workspace.from_bug_dir(bug, sandbox="docker") as ws: + result = ws.run_tests() + assert result["passed"] is True, result["stdout"]