From 4173df7c75dce7d4c47cadb68fd0814e9de5821d Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Thu, 5 Nov 2026 23:36:29 -0500 Subject: [PATCH 01/21] Harden CI against supply-chain and token misuse - Pin every third-party action to a full commit SHA (version kept in a comment), so a moved or compromised tag cannot change what runs. - Check out with persist-credentials: false; no step needs the token left in .git/config afterwards. - Default the workflow token to contents: read and grant write scopes only to the jobs that publish. --- .github/workflows/ci.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af800bc..ad1fd89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,9 @@ on: pull_request: branches: [main] +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest @@ -14,8 +17,10 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} cache: pip @@ -32,8 +37,10 @@ jobs: runs-on: ubuntu-latest needs: test steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" cache: pip @@ -48,6 +55,8 @@ jobs: runs-on: ubuntu-latest needs: test steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - run: docker build -t code-fix-agent:ci . - run: docker run --rm --entrypoint python code-fix-agent:ci -c "from code_fix_agent import Agent; print('ok')" From 346493116d6b9b93eb4cd1d39f575072c1a030ec Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Thu, 5 Nov 2026 23:52:12 -0500 Subject: [PATCH 02/21] Confine every workspace tool to the workspace root Only read and edit checked their path. list_dir("..") and absolute paths listed anything on disk, search_code walked any tree and named outside files that matched, and run_tests passed its argument straight to pytest, so a value such as "--basetemp=" was taken as an option. All five tools now go through Workspace.resolve(), which resolves symlinks and rejects paths outside the root before any file access, so the error does not reveal whether an outside path exists. run_tests only accepts a file inside the workspace, optionally with a ::node suffix. --- src/code_fix_agent/workspace.py | 37 +++++++++++++----- tests/test_tools.py | 67 +++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 10 deletions(-) diff --git a/src/code_fix_agent/workspace.py b/src/code_fix_agent/workspace.py index 1767766..16f98cb 100644 --- a/src/code_fix_agent/workspace.py +++ b/src/code_fix_agent/workspace.py @@ -52,27 +52,35 @@ def __exit__(self, exc_type, exc, tb) -> None: # --- tools ------------------------------------------------------------ + def resolve(self, rel: str) -> Path: + """Resolve `rel` (symlinks included) and refuse anything outside the root. + + Every tool goes through this before touching the file system, so an + error never reveals whether an outside path exists. + """ + root = self.root.resolve() + path = (root / rel).resolve() + if not path.is_relative_to(root): + raise PermissionError(f"path escapes workspace: {rel}") + return path + def list_dir(self, rel: str = ".") -> list[str]: - target = (self.root / rel).resolve() + target = self.resolve(rel) if not target.is_dir(): raise FileNotFoundError(f"not a directory: {rel}") return sorted(p.name for p in target.iterdir()) def read(self, rel: str) -> str: - path = (self.root / rel).resolve() + path = self.resolve(rel) if not path.is_file(): raise FileNotFoundError(f"not a file: {rel}") - if self.root not in path.parents and self.root != path.parent: - raise PermissionError(f"path escapes workspace: {rel}") return path.read_text(encoding="utf-8") def edit(self, rel: str, old: str, new: str) -> str: """str_replace edit. Errors if `old` is not found or not unique.""" - path = (self.root / rel).resolve() + path = self.resolve(rel) if not path.is_file(): raise FileNotFoundError(f"not a file: {rel}") - if self.root not in path.parents and self.root != path.parent: - raise PermissionError(f"path escapes workspace: {rel}") text = path.read_text(encoding="utf-8") if old not in text: raise ValueError("old string not found in file") @@ -84,17 +92,26 @@ def edit(self, rel: str, old: str, new: str) -> str: def search(self, pattern: str, rel: str = ".") -> list[tuple[str, int, str]]: """Substring search across files in the workspace (Python files only).""" - target = (self.root / rel).resolve() + root = self.root.resolve() + target = self.resolve(rel) out: list[tuple[str, int, str]] = [] for path in target.rglob("*.py"): + path = path.resolve() + if not path.is_relative_to(root) or not path.is_file(): + continue for i, line in enumerate(path.read_text(encoding="utf-8", errors="ignore").splitlines(), 1): if pattern in line: - out.append((path.relative_to(self.root).as_posix(), i, line.rstrip())) + out.append((path.relative_to(root).as_posix(), i, line.rstrip())) return out def run_tests(self, test_file: str = "test_buggy.py", timeout: float = 30.0) -> dict: """Run pytest on test_file. Returns dict with passed/failed counts and output.""" - cmd = [sys.executable, "-m", "pytest", "-q", "--no-header", "--no-summary", test_file] + file_part, sep, node = test_file.partition("::") + path = self.resolve(file_part) + if not path.is_file(): + raise FileNotFoundError(f"not a file: {file_part}") + target = path.relative_to(self.root.resolve()).as_posix() + sep + node + cmd = [sys.executable, "-m", "pytest", "-q", "--no-header", "--no-summary", target] try: proc = subprocess.run( cmd, diff --git a/tests/test_tools.py b/tests/test_tools.py index ec65e84..618f50e 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -1,8 +1,11 @@ """Tool dispatcher tests.""" from __future__ import annotations +import os from pathlib import Path +import pytest + from code_fix_agent.tools import execute, tool_schemas from code_fix_agent.workspace import Workspace @@ -54,3 +57,67 @@ def test_search_code(self): with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: out = execute(ws, "search_code", {"pattern": "sum_first_n"}) assert "buggy.py" in out + + +class TestPathConfinement: + """Every tool refuses paths that resolve outside the workspace root.""" + + @pytest.mark.parametrize( + ("tool", "tool_input"), + [ + ("list_dir", {"path": ".."}), + ("list_dir", {"path": "../.."}), + ("read_file", {"path": "../buggy.py"}), + ("edit_file", {"path": "../buggy.py", "old": "a", "new": "b"}), + ("search_code", {"pattern": "def ", "path": ".."}), + ("run_tests", {"test_file": "../test_buggy.py"}), + ("run_tests", {"test_file": "../test_buggy.py::test_one"}), + ], + ) + def test_relative_escape_is_refused(self, tool, tool_input): + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + out = execute(ws, tool, tool_input) + assert out.startswith("error: path escapes workspace"), out + + @pytest.mark.parametrize( + ("tool", "key"), + [("list_dir", "path"), ("read_file", "path"), ("search_code", "path"), ("run_tests", "test_file")], + ) + def test_absolute_path_outside_is_refused(self, tool, key): + outside = _bug_dir("01_off_by_one") + target = outside if tool in ("list_dir", "search_code") else outside / "test_buggy.py" + tool_input = {key: str(target)} + if tool == "search_code": + tool_input["pattern"] = "def " + with Workspace.from_bug_dir(outside) as ws: + out = execute(ws, tool, tool_input) + assert out.startswith("error: path escapes workspace"), out + assert "buggy.py" not in out.replace(str(target), "") + + def test_outside_files_are_not_distinguishable_from_missing_ones(self, tmp_path): + real = tmp_path / "real.py" + real.write_text("x = 1\n", encoding="utf-8") + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + exists = execute(ws, "read_file", {"path": os.path.relpath(real, ws.root)}) + missing = execute(ws, "read_file", {"path": os.path.relpath(tmp_path / "nope.py", ws.root)}) + assert exists.startswith("error: path escapes workspace"), exists + assert missing.startswith("error: path escapes workspace"), missing + + def test_symlink_pointing_outside_is_refused(self, tmp_path): + secret = tmp_path / "secret.py" + secret.write_text("TOKEN = 'outside'\n", encoding="utf-8") + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + link = ws.root / "link.py" + try: + link.symlink_to(secret) + except OSError: + pytest.skip("creating symlinks is not permitted here") + assert execute(ws, "read_file", {"path": "link.py"}).startswith("error: path escapes") + assert "outside" not in execute(ws, "search_code", {"pattern": "TOKEN"}) + + def test_paths_inside_the_workspace_still_work(self): + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + assert "buggy.py" in execute(ws, "list_dir", {"path": "./"}) + assert "def sum_first_n" in execute(ws, "read_file", {"path": "./buggy.py"}) + out = execute(ws, "run_tests", {"test_file": "test_buggy.py::test_zero"}) + assert out.startswith("PASS"), out From 30d0a7056ee8581f19166837d8a03343b5689a46 Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Thu, 5 Nov 2026 23:55:03 -0500 Subject: [PATCH 03/21] Run tests without the caller's environment, in their own process group The model writes the code that run_tests executes, and that code inherited the full environment, ANTHROPIC_API_KEY included: a snippet added through edit_file could write the key to a file and read_file returned it. A timeout also killed only pytest, so processes the tests started kept running (and held the output pipes open). Tests now run with only PATH, SYSTEMROOT and the temp-directory variables, in a new session (POSIX) or process group (Windows); a timeout kills the whole tree. -B and -p no:cacheprovider keep __pycache__ and .pytest_cache out of the workspace listing. --- src/code_fix_agent/sandbox.py | 77 +++++++++++++++++++++++++++++++++ src/code_fix_agent/workspace.py | 56 +++++++++--------------- tests/test_workspace.py | 54 +++++++++++++++++++++++ 3 files changed, 152 insertions(+), 35 deletions(-) create mode 100644 src/code_fix_agent/sandbox.py diff --git a/src/code_fix_agent/sandbox.py b/src/code_fix_agent/sandbox.py new file mode 100644 index 0000000..0f2104c --- /dev/null +++ b/src/code_fix_agent/sandbox.py @@ -0,0 +1,77 @@ +"""Run pytest on model-edited code. + +The code under test is written by the model, so it is treated as untrusted: +it gets none of the caller's environment (no API keys), and a timeout kills +every process the tests started, not only pytest. +""" +from __future__ import annotations + +import os +import signal +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +PYTEST_ARGS = ["-q", "--no-header", "--no-summary", "-p", "no:cacheprovider"] + +# Python on Windows cannot start without SYSTEMROOT; the temp variables only +# say where scratch files go. Nothing else is passed through. +_ENV_ALLOWLIST = ("PATH", "SYSTEMROOT", "TEMP", "TMP", "TMPDIR") + + +@dataclass +class RunResult: + exit_code: int | None # None when the run hit the timeout + stdout: str + stderr: str + + +def minimal_env() -> dict[str, str]: + env = {key: os.environ[key] for key in _ENV_ALLOWLIST if key in os.environ} + env["PYTHONIOENCODING"] = "utf-8" + return env + + +def run_pytest(cwd: Path, args: list[str], *, timeout: float) -> RunResult: + """Run pytest in `cwd` in its own process group with a minimal environment.""" + cmd = [sys.executable, "-B", "-m", "pytest", *PYTEST_ARGS, *args] + group = ( + {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} + if os.name == "nt" + else {"start_new_session": True} + ) + proc = subprocess.Popen( + cmd, + cwd=cwd, + env=minimal_env(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + **group, + ) + try: + stdout, stderr = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + _kill_tree(proc) + try: + stdout, _ = proc.communicate(timeout=5) + except subprocess.TimeoutExpired: + stdout = "" + return RunResult(None, stdout or "", "timeout") + return RunResult(proc.returncode, stdout, stderr) + + +def _kill_tree(proc: subprocess.Popen) -> None: + if os.name == "nt": + subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(proc.pid)], capture_output=True, check=False + ) + else: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.kill() diff --git a/src/code_fix_agent/workspace.py b/src/code_fix_agent/workspace.py index 16f98cb..10554d7 100644 --- a/src/code_fix_agent/workspace.py +++ b/src/code_fix_agent/workspace.py @@ -8,13 +8,13 @@ from __future__ import annotations import shutil -import subprocess -import sys import tempfile from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path +from code_fix_agent.sandbox import RunResult, run_pytest + @dataclass class Workspace: @@ -111,41 +111,27 @@ def run_tests(self, test_file: str = "test_buggy.py", timeout: float = 30.0) -> if not path.is_file(): raise FileNotFoundError(f"not a file: {file_part}") target = path.relative_to(self.root.resolve()).as_posix() + sep + node - cmd = [sys.executable, "-m", "pytest", "-q", "--no-header", "--no-summary", target] - try: - proc = subprocess.run( - cmd, - cwd=self.root, - capture_output=True, - text=True, - timeout=timeout, - check=False, - ) - except subprocess.TimeoutExpired: - return { - "passed": False, - "exit_code": -1, - "stdout": "", - "stderr": "timeout", - "summary": "timeout", - } - stdout = proc.stdout - stderr = proc.stderr - passed = proc.returncode == 0 - # Pull the last non-empty line of stdout as a summary; pytest prints - # "X passed in Y.YYs" or "X failed, Y passed in Z.ZZs" there. - summary = "" - for line in reversed(stdout.splitlines()): - if line.strip(): - summary = line.strip() - break + return _as_result(run_pytest(self.root, [target], timeout=timeout)) + + +def _as_result(run: RunResult) -> dict: + if run.exit_code is None: return { - "passed": passed, - "exit_code": proc.returncode, - "stdout": stdout[-4000:], # truncate to keep tool result small - "stderr": stderr[-2000:], - "summary": summary, + "passed": False, + "exit_code": -1, + "stdout": run.stdout[-4000:], + "stderr": "timeout", + "summary": "timeout", } + # pytest prints "X passed in Y.YYs" or "X failed, Y passed in Z.ZZs" last. + summary = next((line.strip() for line in reversed(run.stdout.splitlines()) if line.strip()), "") + return { + "passed": run.exit_code == 0, + "exit_code": run.exit_code, + "stdout": run.stdout[-4000:], # truncate to keep tool result small + "stderr": run.stderr[-2000:], + "summary": summary, + } @contextmanager diff --git a/tests/test_workspace.py b/tests/test_workspace.py index 84d4822..f776721 100644 --- a/tests/test_workspace.py +++ b/tests/test_workspace.py @@ -1,6 +1,8 @@ """Workspace tests.""" from __future__ import annotations +import textwrap +import time from pathlib import Path import pytest @@ -50,3 +52,55 @@ def test_search_finds_pattern(self): with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: hits = ws.search("range") assert any(h[2].strip().startswith("for i in range") for h in hits) + + +def _write_bug(dir_: Path, test_source: str) -> Path: + (dir_ / "buggy.py").write_text("def f():\n return 1\n", encoding="utf-8") + (dir_ / "test_buggy.py").write_text(textwrap.dedent(test_source), encoding="utf-8") + return dir_ + + +class TestTestRunIsolation: + def test_tests_do_not_inherit_secrets(self, tmp_path, monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-test-not-a-key") + monkeypatch.setenv("CFA_PROBE_SECRET", "visible") + bug = _write_bug( + tmp_path, + """ + import os + + def test_no_secrets(): + assert os.environ.get("ANTHROPIC_API_KEY") is None + assert os.environ.get("CFA_PROBE_SECRET") is None + """, + ) + with Workspace.from_bug_dir(bug) as ws: + result = ws.run_tests() + assert result["passed"] is True, result["stdout"] + + def test_timeout_kills_processes_the_tests_started(self, tmp_path): + started = tmp_path / "started" + late = tmp_path / "late" + child = ( + "import pathlib, time; " + f"pathlib.Path({str(started)!r}).write_text('x'); " + "time.sleep(6); " + f"pathlib.Path({str(late)!r}).write_text('x')" + ) + (tmp_path / "bug").mkdir() + bug = _write_bug( + tmp_path / "bug", + f""" + import subprocess, sys, time + + def test_hangs(): + subprocess.Popen([sys.executable, "-c", {child!r}]) + time.sleep(60) + """, + ) + with Workspace.from_bug_dir(bug) as ws: + result = ws.run_tests(timeout=6) + time.sleep(5) + assert result["summary"] == "timeout" + assert started.exists(), "the test never started its child process" + assert not late.exists(), "a process started by the tests outlived the timeout" From 005c1788101f1484d5cde9825c2cd77fa97479ba Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Thu, 5 Nov 2026 23:55:44 -0500 Subject: [PATCH 04/21] Show the model short failure tracebacks --no-summary hid every failure detail: a failing run came back as "FFF. 3 failed, 1 passed", while the system prompt tells the model to read the traceback. Use --tb=short; output stays capped at the last 4,000 characters, which keeps the summary line. --- src/code_fix_agent/sandbox.py | 2 +- tests/test_tools.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/code_fix_agent/sandbox.py b/src/code_fix_agent/sandbox.py index 0f2104c..49f430a 100644 --- a/src/code_fix_agent/sandbox.py +++ b/src/code_fix_agent/sandbox.py @@ -13,7 +13,7 @@ from dataclasses import dataclass from pathlib import Path -PYTEST_ARGS = ["-q", "--no-header", "--no-summary", "-p", "no:cacheprovider"] +PYTEST_ARGS = ["-q", "--no-header", "--tb=short", "-p", "no:cacheprovider"] # Python on Windows cannot start without SYSTEMROOT; the temp variables only # say where scratch files go. Nothing else is passed through. diff --git a/tests/test_tools.py b/tests/test_tools.py index 618f50e..4ef6d27 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -43,6 +43,14 @@ def test_edit_file_then_run_tests_pass(self): out = execute(ws, "run_tests", {"test_file": "test_buggy.py"}) assert out.startswith("PASS") + def test_failing_run_shows_the_assertion_that_failed(self): + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + out = execute(ws, "run_tests", {}) + assert out.startswith("FAIL") + assert "assert sum_first_n(5) == 15" in out + assert "assert 10 == 15" in out + assert out.strip().splitlines()[-1].startswith("3 failed, 1 passed") + def test_edit_file_error_surfaces(self): with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: out = execute(ws, "edit_file", {"path": "buggy.py", "old": "missing", "new": "x"}) From bb68c55726636b4cd40856064c095b4feb9c2fc4 Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Thu, 5 Nov 2026 23:59:23 -0500 Subject: [PATCH 05/21] Run tests in a network-less container when the image is available The local runner keeps secrets out of the environment, but model-written code still runs with the caller's file-system and network access. With --sandbox docker each test run is a throwaway container: no network, read-only root and workspace mount, all capabilities dropped, no-new-privileges, 512 MB / 1 CPU / 128 processes, no host environment. On timeout the container is killed by name, since killing the docker client does not stop it. --sandbox auto (the default for the CLI and the benchmark) uses Docker when `docker image inspect code-fix-agent:latest` succeeds and falls back to the local runner otherwise; --sandbox docker fails instead of falling back. The runner used is recorded in every result. The image now runs as a non-root user. --- Dockerfile | 6 +- benchmarks/run_benchmark.py | 11 ++- src/code_fix_agent/cli.py | 10 ++- src/code_fix_agent/sandbox.py | 115 ++++++++++++++++++++++++++++++-- src/code_fix_agent/workspace.py | 14 ++-- tests/test_workspace.py | 62 ++++++++++++++++- 6 files changed, 200 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index ba49413..240eee2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,11 @@ COPY bugs ./bugs COPY benchmarks ./benchmarks COPY README.md LICENSE ./ -RUN pip install --no-cache-dir -e ".[dev]" +RUN pip install --no-cache-dir -e ".[dev]" \ + && useradd --create-home --uid 10001 app \ + && chown app benchmarks/results + +USER app HEALTHCHECK --interval=30s --timeout=15s --start-period=5s --retries=2 \ CMD python -c "from code_fix_agent import Agent; print('ok')" || exit 1 diff --git a/benchmarks/run_benchmark.py b/benchmarks/run_benchmark.py index a0e1481..28f5dae 100644 --- a/benchmarks/run_benchmark.py +++ b/benchmarks/run_benchmark.py @@ -25,6 +25,7 @@ from code_fix_agent.agent import Agent from code_fix_agent.llm import AnthropicLLM, LLMClient, MockLLM +from code_fix_agent.sandbox import SANDBOX_MODES, resolve_sandbox from code_fix_agent.workspace import Workspace ROOT = Path(__file__).resolve().parent.parent @@ -118,14 +119,15 @@ def _build_mock_llm(bug_id: str) -> MockLLM: return MockLLM(script=script) -def _run_one(bug: dict, llm: LLMClient, max_iterations: int) -> dict: +def _run_one(bug: dict, llm: LLMClient, max_iterations: int, sandbox: str = "auto") -> dict: bug_dir = BUGS_DIR / bug["id"] - with Workspace.from_bug_dir(bug_dir) as ws: + with Workspace.from_bug_dir(bug_dir, sandbox=sandbox) as ws: agent = Agent(llm=llm, max_iterations=max_iterations) result = agent.fix(ws) return { "id": bug["id"], "kind": bug.get("kind", ""), + "sandbox": ws.sandbox, "passed": result.passed, "iterations": result.iterations, "tool_calls": result.tool_calls, @@ -148,9 +150,11 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--max-iterations", type=int, default=8) parser.add_argument("--model", default="claude-haiku-4-5-20251001") parser.add_argument("--out", type=Path, default=RESULTS_DIR / "benchmark.json") + parser.add_argument("--sandbox", choices=SANDBOX_MODES, default="auto") args = parser.parse_args(argv) bugs = _load_manifest() + sandbox = resolve_sandbox(args.sandbox) records: list[dict] = [] for bug in bugs: if args.mode == "live": @@ -158,12 +162,13 @@ def main(argv: list[str] | None = None) -> int: else: llm = _build_mock_llm(bug["id"]) print(f" [{args.mode}] {bug['id']}", file=sys.stderr) - records.append(_run_one(bug, llm, args.max_iterations)) + records.append(_run_one(bug, llm, args.max_iterations, sandbox)) n = len(records) passed = sum(1 for r in records if r["passed"]) summary = { "mode": args.mode, + "sandbox": sandbox, "n_bugs": n, "passed": passed, "pass_rate_pct": round(100 * passed / n, 1) if n else 0, diff --git a/src/code_fix_agent/cli.py b/src/code_fix_agent/cli.py index d42bd41..91d5f18 100644 --- a/src/code_fix_agent/cli.py +++ b/src/code_fix_agent/cli.py @@ -13,16 +13,18 @@ from code_fix_agent.agent import Agent from code_fix_agent.llm import AnthropicLLM +from code_fix_agent.sandbox import SANDBOX_MODES from code_fix_agent.workspace import Workspace def _cmd_run(args: argparse.Namespace) -> int: llm = AnthropicLLM(model=args.model) agent = Agent(llm=llm, max_iterations=args.max_iterations) - with Workspace.from_bug_dir(Path(args.bug_dir)) as ws: + with Workspace.from_bug_dir(Path(args.bug_dir), sandbox=args.sandbox) as ws: result = agent.fix(ws) payload = { "bug_id": result.bug_id, + "sandbox": ws.sandbox, "passed": result.passed, "iterations": result.iterations, "tool_calls": result.tool_calls, @@ -44,6 +46,12 @@ def main(argv: list[str] | None = None) -> int: p_run.add_argument("bug_dir", type=str, help="Path to a bug subdirectory.") p_run.add_argument("--model", default="claude-haiku-4-5-20251001") p_run.add_argument("--max-iterations", type=int, default=8) + p_run.add_argument( + "--sandbox", + choices=SANDBOX_MODES, + default="auto", + help="where tests run: docker (no network, read-only), local, or auto (docker if the image exists)", + ) p_run.set_defaults(func=_cmd_run) args = parser.parse_args(argv) diff --git a/src/code_fix_agent/sandbox.py b/src/code_fix_agent/sandbox.py index 49f430a..1d013da 100644 --- a/src/code_fix_agent/sandbox.py +++ b/src/code_fix_agent/sandbox.py @@ -1,19 +1,34 @@ """Run pytest on model-edited code. -The code under test is written by the model, so it is treated as untrusted: -it gets none of the caller's environment (no API keys), and a timeout kills -every process the tests started, not only pytest. +The code under test is written by the model, so it is treated as untrusted. +Two runners: + +- docker: a throwaway container with no network, a read-only file system and + workspace mount, no capabilities, and memory/CPU/process limits. Nothing + from the caller's environment is passed in. +- local: a subprocess with a minimal environment (no API keys), in its own + process group so a timeout kills every process the tests started. It still + runs with the caller's file-system rights, so it is the fallback for when + Docker or the sandbox image is not available. + +"auto" picks docker when the `docker` CLI can see the sandbox image. """ from __future__ import annotations import os +import shutil import signal import subprocess import sys +import uuid +from collections.abc import Callable from dataclasses import dataclass +from functools import cache from pathlib import Path PYTEST_ARGS = ["-q", "--no-header", "--tb=short", "-p", "no:cacheprovider"] +SANDBOX_IMAGE = os.environ.get("CFA_SANDBOX_IMAGE", "code-fix-agent:latest") +SANDBOX_MODES = ("auto", "docker", "local") # Python on Windows cannot start without SYSTEMROOT; the temp variables only # say where scratch files go. Nothing else is passed through. @@ -27,15 +42,91 @@ class RunResult: stderr: str +@cache +def docker_available(image: str = SANDBOX_IMAGE) -> bool: + if shutil.which("docker") is None: + return False + try: + probe = subprocess.run( + ["docker", "image", "inspect", image], capture_output=True, timeout=30, check=False + ) + except (OSError, subprocess.TimeoutExpired): + return False + return probe.returncode == 0 + + +def resolve_sandbox(mode: str, image: str = SANDBOX_IMAGE) -> str: + """Map a requested mode to the runner that will be used: "docker" or "local".""" + if mode not in SANDBOX_MODES: + raise ValueError(f"sandbox must be one of {SANDBOX_MODES}, got {mode!r}") + if mode == "local": + return "local" + if docker_available(image): + return "docker" + if mode == "docker": + raise RuntimeError( + f"docker sandbox requested but image {image!r} is not available " + f"(build it with `docker build -t {image} .`)" + ) + return "local" + + def minimal_env() -> dict[str, str]: env = {key: os.environ[key] for key in _ENV_ALLOWLIST if key in os.environ} env["PYTHONIOENCODING"] = "utf-8" return env -def run_pytest(cwd: Path, args: list[str], *, timeout: float) -> RunResult: - """Run pytest in `cwd` in its own process group with a minimal environment.""" - cmd = [sys.executable, "-B", "-m", "pytest", *PYTEST_ARGS, *args] +def docker_command(cwd: Path, args: list[str], *, name: str, image: str = SANDBOX_IMAGE) -> list[str]: + cmd = [ + "docker", "run", "--rm", "--name", name, + "--network", "none", + "--read-only", "--tmpfs", "/tmp", + "--memory", "512m", "--pids-limit", "128", "--cpus", "1", + "--cap-drop", "ALL", "--security-opt", "no-new-privileges", + "--mount", f"type=bind,source={cwd},target=/work,readonly", + "--workdir", "/work", + "--entrypoint", "python", + ] + if hasattr(os, "getuid"): + # The temp workspace is private to the caller's uid on POSIX. + cmd += ["--user", f"{os.getuid()}:{os.getgid()}"] + return [*cmd, image, "-B", "-m", "pytest", *PYTEST_ARGS, *args] + + +def run_pytest( + cwd: Path, args: list[str], *, timeout: float, sandbox: str = "local", image: str = SANDBOX_IMAGE +) -> RunResult: + """Run pytest on `cwd` with the given runner ("docker" or "local").""" + if sandbox == "docker": + name = f"cfa-{uuid.uuid4().hex[:12]}" + # The docker client needs the caller's environment to reach the + # daemon; none of it is forwarded into the container. + return _run( + docker_command(cwd, args, name=name, image=image), + cwd=None, + env=None, + timeout=timeout, + on_timeout=lambda: _docker_kill(name), + ) + if sandbox != "local": + raise ValueError(f"unknown runner: {sandbox!r}") + return _run( + [sys.executable, "-B", "-m", "pytest", *PYTEST_ARGS, *args], + cwd=cwd, + env=minimal_env(), + timeout=timeout, + ) + + +def _run( + cmd: list[str], + *, + cwd: Path | None, + env: dict[str, str] | None, + timeout: float, + on_timeout: Callable[[], object] | None = None, +) -> RunResult: group = ( {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} if os.name == "nt" @@ -44,7 +135,7 @@ def run_pytest(cwd: Path, args: list[str], *, timeout: float) -> RunResult: proc = subprocess.Popen( cmd, cwd=cwd, - env=minimal_env(), + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, @@ -55,6 +146,8 @@ def run_pytest(cwd: Path, args: list[str], *, timeout: float) -> RunResult: try: stdout, stderr = proc.communicate(timeout=timeout) except subprocess.TimeoutExpired: + if on_timeout is not None: + on_timeout() _kill_tree(proc) try: stdout, _ = proc.communicate(timeout=5) @@ -64,6 +157,14 @@ def run_pytest(cwd: Path, args: list[str], *, timeout: float) -> RunResult: return RunResult(proc.returncode, stdout, stderr) +def _docker_kill(name: str) -> None: + # Killing the docker client does not stop the container. + try: + subprocess.run(["docker", "kill", name], capture_output=True, timeout=30, check=False) + except (OSError, subprocess.TimeoutExpired): + pass + + def _kill_tree(proc: subprocess.Popen) -> None: if os.name == "nt": subprocess.run( diff --git a/src/code_fix_agent/workspace.py b/src/code_fix_agent/workspace.py index 10554d7..a9ff033 100644 --- a/src/code_fix_agent/workspace.py +++ b/src/code_fix_agent/workspace.py @@ -13,7 +13,7 @@ from dataclasses import dataclass, field from pathlib import Path -from code_fix_agent.sandbox import RunResult, run_pytest +from code_fix_agent.sandbox import RunResult, resolve_sandbox, run_pytest @dataclass @@ -31,17 +31,23 @@ class Workspace: root: Path bug_id: str = "" _cleanup: bool = field(default=False, repr=False) + # "auto", "docker" or "local"; resolved at construction to the runner used. + sandbox: str = "auto" + + def __post_init__(self) -> None: + self.sandbox = resolve_sandbox(self.sandbox) # --- creation --------------------------------------------------------- @classmethod - def from_bug_dir(cls, bug_dir: Path, cleanup: bool = True) -> Workspace: + def from_bug_dir(cls, bug_dir: Path, cleanup: bool = True, sandbox: str = "auto") -> Workspace: bug_dir = Path(bug_dir) + sandbox = resolve_sandbox(sandbox) # fail before creating the temp dir tmp = Path(tempfile.mkdtemp(prefix=f"cfa-{bug_dir.name}-")) for child in bug_dir.iterdir(): if child.is_file(): shutil.copy2(child, tmp / child.name) - return cls(root=tmp, bug_id=bug_dir.name, _cleanup=cleanup) + return cls(root=tmp, bug_id=bug_dir.name, _cleanup=cleanup, sandbox=sandbox) def __enter__(self) -> Workspace: return self @@ -111,7 +117,7 @@ def run_tests(self, test_file: str = "test_buggy.py", timeout: float = 30.0) -> if not path.is_file(): raise FileNotFoundError(f"not a file: {file_part}") target = path.relative_to(self.root.resolve()).as_posix() + sep + node - return _as_result(run_pytest(self.root, [target], timeout=timeout)) + return _as_result(run_pytest(self.root, [target], timeout=timeout, sandbox=self.sandbox)) def _as_result(run: RunResult) -> dict: diff --git a/tests/test_workspace.py b/tests/test_workspace.py index f776721..a833f76 100644 --- a/tests/test_workspace.py +++ b/tests/test_workspace.py @@ -7,6 +7,7 @@ import pytest +from code_fix_agent import sandbox from code_fix_agent.workspace import Workspace @@ -74,7 +75,7 @@ def test_no_secrets(): assert os.environ.get("CFA_PROBE_SECRET") is None """, ) - with Workspace.from_bug_dir(bug) as ws: + with Workspace.from_bug_dir(bug, sandbox="local") as ws: result = ws.run_tests() assert result["passed"] is True, result["stdout"] @@ -98,9 +99,66 @@ def test_hangs(): time.sleep(60) """, ) - with Workspace.from_bug_dir(bug) as ws: + with Workspace.from_bug_dir(bug, sandbox="local") as ws: result = ws.run_tests(timeout=6) time.sleep(5) assert result["summary"] == "timeout" assert started.exists(), "the test never started its child process" assert not late.exists(), "a process started by the tests outlived the timeout" + + +class TestSandboxSelection: + def test_auto_falls_back_to_local_without_the_image(self, monkeypatch): + monkeypatch.setattr(sandbox, "docker_available", lambda image=sandbox.SANDBOX_IMAGE: False) + assert sandbox.resolve_sandbox("auto") == "local" + + def test_auto_uses_docker_when_the_image_exists(self, monkeypatch): + monkeypatch.setattr(sandbox, "docker_available", lambda image=sandbox.SANDBOX_IMAGE: True) + assert sandbox.resolve_sandbox("auto") == "docker" + + def test_explicit_docker_fails_before_creating_a_workspace(self, monkeypatch, tmp_path): + monkeypatch.setattr(sandbox, "docker_available", lambda image=sandbox.SANDBOX_IMAGE: False) + monkeypatch.setattr("tempfile.tempdir", str(tmp_path)) + with pytest.raises(RuntimeError, match="not available"): + Workspace.from_bug_dir(_bug_dir("01_off_by_one"), sandbox="docker") + assert list(tmp_path.iterdir()) == [] + + def test_unknown_mode_is_rejected(self): + with pytest.raises(ValueError): + sandbox.resolve_sandbox("chroot") + + def test_docker_command_shuts_off_network_env_and_writes(self, tmp_path): + cmd = sandbox.docker_command(tmp_path, ["test_buggy.py"], name="cfa-test") + pairs = set(zip(cmd, cmd[1:])) + assert ("--network", "none") in pairs + assert ("--cap-drop", "ALL") in pairs + assert ("--security-opt", "no-new-privileges") in pairs + assert "--read-only" in cmd + assert f"type=bind,source={tmp_path},target=/work,readonly" in cmd + assert not {"-e", "--env", "--env-file", "--privileged"} & set(cmd) + image_at = cmd.index(sandbox.SANDBOX_IMAGE) + assert cmd[image_at + 1 :] == ["-B", "-m", "pytest", *sandbox.PYTEST_ARGS, "test_buggy.py"] + + +@pytest.mark.skipif(not sandbox.docker_available(), reason="sandbox image not built") +class TestDockerSandbox: + def test_container_has_no_network_and_no_secrets(self, tmp_path, monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-test-not-a-key") + bug = _write_bug( + tmp_path, + """ + import os, socket + + import pytest + + def test_isolated(): + assert os.environ.get("ANTHROPIC_API_KEY") is None + with pytest.raises(OSError): + socket.create_connection(("1.1.1.1", 53), timeout=3) + with pytest.raises(OSError): + open("written.txt", "w") + """, + ) + with Workspace.from_bug_dir(bug, sandbox="docker") as ws: + result = ws.run_tests() + assert result["passed"] is True, result["stdout"] From f0afadbe21da526b30ad931a3794767fa7153b5e Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Fri, 6 Nov 2026 00:05:01 -0500 Subject: [PATCH 06/21] Score each fix against a clean copy of the original tests "passed" was whatever the model's own run_tests call reported, and the model chooses the test target and can edit the test file. Running test_buggy.py::test_zero on the unfixed off-by-one bug scored as a pass while the full suite had 3 failures. The workspace now snapshots its files at setup. After the loop, Workspace.score() rebuilds the bug in a fresh directory (tests from the snapshot, other original files from the workspace, nothing the run created) and runs the whole suite with the same runner; that is the verdict. AgentResult keeps the model's claim as self_reported_pass and flags tests_modified when a test file no longer matches the snapshot's SHA-256. final_summary is now the scoring run's status line. --- benchmarks/run_benchmark.py | 2 ++ src/code_fix_agent/agent.py | 17 ++++++--- src/code_fix_agent/cli.py | 2 ++ src/code_fix_agent/tools.py | 8 +++-- src/code_fix_agent/workspace.py | 43 ++++++++++++++++++++++ tests/test_agent.py | 63 +++++++++++++++++++++++++++++++++ 6 files changed, 128 insertions(+), 7 deletions(-) diff --git a/benchmarks/run_benchmark.py b/benchmarks/run_benchmark.py index 28f5dae..c323385 100644 --- a/benchmarks/run_benchmark.py +++ b/benchmarks/run_benchmark.py @@ -129,6 +129,8 @@ def _run_one(bug: dict, llm: LLMClient, max_iterations: int, sandbox: str = "aut "kind": bug.get("kind", ""), "sandbox": ws.sandbox, "passed": result.passed, + "self_reported_pass": result.self_reported_pass, + "tests_modified": result.tests_modified, "iterations": result.iterations, "tool_calls": result.tool_calls, "elapsed_seconds": result.elapsed_seconds, diff --git a/src/code_fix_agent/agent.py b/src/code_fix_agent/agent.py index 1b0a00c..422707b 100644 --- a/src/code_fix_agent/agent.py +++ b/src/code_fix_agent/agent.py @@ -6,6 +6,8 @@ run tests. 3. Stops when run_tests reports passed, or when the iteration budget is exhausted. +4. Scores the result by running the original test suite on a clean copy + (Workspace.score); the model's own run_tests output is not the verdict. Returns an AgentResult capturing pass/fail, iteration count, total tokens, and the trail of tool calls. @@ -17,7 +19,7 @@ from typing import Any from code_fix_agent.llm import LLMClient -from code_fix_agent.tools import execute, tool_schemas +from code_fix_agent.tools import execute, status_line, tool_schemas from code_fix_agent.workspace import Workspace SYSTEM_PROMPT = ( @@ -41,7 +43,7 @@ @dataclass class AgentResult: bug_id: str - passed: bool + passed: bool # verdict of the original tests, run after the loop iterations: int tool_calls: int elapsed_seconds: float @@ -50,6 +52,8 @@ class AgentResult: tool_trail: list[dict[str, Any]] = field(default_factory=list) final_summary: str = "" error: str | None = None + self_reported_pass: bool = False # the model's own run_tests said PASS + tests_modified: bool = False def run_agent( @@ -116,15 +120,18 @@ def run_agent( } ) if tool_name == "run_tests" and output.startswith("PASS"): - result.passed = True - result.final_summary = output.split("\n", 1)[0] + result.self_reported_pass = True if tool_results: messages.append({"role": "user", "content": tool_results}) - if result.passed or stop_reason != "tool_use": + if result.self_reported_pass or stop_reason != "tool_use": break + verdict = ws.score() + result.passed = verdict["passed"] + result.tests_modified = verdict["tests_modified"] + result.final_summary = status_line(verdict) result.elapsed_seconds = round(time.perf_counter() - start, 3) return result diff --git a/src/code_fix_agent/cli.py b/src/code_fix_agent/cli.py index 91d5f18..274e20c 100644 --- a/src/code_fix_agent/cli.py +++ b/src/code_fix_agent/cli.py @@ -26,6 +26,8 @@ def _cmd_run(args: argparse.Namespace) -> int: "bug_id": result.bug_id, "sandbox": ws.sandbox, "passed": result.passed, + "self_reported_pass": result.self_reported_pass, + "tests_modified": result.tests_modified, "iterations": result.iterations, "tool_calls": result.tool_calls, "elapsed_seconds": result.elapsed_seconds, diff --git a/src/code_fix_agent/tools.py b/src/code_fix_agent/tools.py index ac79ca3..1db145e 100644 --- a/src/code_fix_agent/tools.py +++ b/src/code_fix_agent/tools.py @@ -92,6 +92,11 @@ def tool_schemas() -> list[dict[str, Any]]: ] +def status_line(result: dict[str, Any]) -> str: + status = "PASS" if result["passed"] else "FAIL" + return f"{status} (exit {result['exit_code']}) summary={result['summary']!r}" + + def execute(ws: Workspace, tool_name: str, tool_input: dict[str, Any]) -> str: """Dispatch one tool call against the workspace. Returns a string result.""" try: @@ -117,9 +122,8 @@ def execute(ws: Workspace, tool_name: str, tool_input: dict[str, Any]) -> str: if tool_name == "run_tests": result = ws.run_tests(tool_input.get("test_file", "test_buggy.py")) - status = "PASS" if result["passed"] else "FAIL" body = result["stdout"] or result["stderr"] - return f"{status} (exit {result['exit_code']}) summary={result['summary']!r}\n{body}" + return f"{status_line(result)}\n{body}" return f"unknown tool: {tool_name}" except FileNotFoundError as exc: diff --git a/src/code_fix_agent/workspace.py b/src/code_fix_agent/workspace.py index a9ff033..f4dff10 100644 --- a/src/code_fix_agent/workspace.py +++ b/src/code_fix_agent/workspace.py @@ -7,6 +7,7 @@ """ from __future__ import annotations +import hashlib import shutil import tempfile from contextlib import contextmanager @@ -33,9 +34,12 @@ class Workspace: _cleanup: bool = field(default=False, repr=False) # "auto", "docker" or "local"; resolved at construction to the runner used. sandbox: str = "auto" + # Top-level files as they were at setup; score() rebuilds the bug from these. + originals: dict[str, bytes] = field(init=False, repr=False) def __post_init__(self) -> None: self.sandbox = resolve_sandbox(self.sandbox) + self.originals = {p.name: p.read_bytes() for p in self.root.iterdir() if p.is_file()} # --- creation --------------------------------------------------------- @@ -119,6 +123,45 @@ def run_tests(self, test_file: str = "test_buggy.py", timeout: float = 30.0) -> target = path.relative_to(self.root.resolve()).as_posix() + sep + node return _as_result(run_pytest(self.root, [target], timeout=timeout, sandbox=self.sandbox)) + # --- scoring ---------------------------------------------------------- + + def score(self, timeout: float = 30.0) -> dict: + """Run the original test suite against the workspace's current source. + + The model can edit the test file, run a subset of it, or (through the + code its tests execute) drop a conftest.py into the workspace, so its + own run_tests result is not a score. This rebuilds the bug in a fresh + directory from the files present at setup, taking the tests from the + setup snapshot and every other file from the workspace, and runs the + whole suite. `tests_modified` reports whether the workspace's test + files still match the snapshot's SHA-256. + """ + tests = sorted(name for name in self.originals if _is_test_file(name)) + with tempfile.TemporaryDirectory(prefix="cfa-score-", ignore_cleanup_errors=True) as tmp: + dest = Path(tmp) + for name, data in self.originals.items(): + current = self.root / name + if name in tests: + (dest / name).write_bytes(data) + elif current.is_file(): + shutil.copyfile(current, dest / name) + result = _as_result(run_pytest(dest, tests, timeout=timeout, sandbox=self.sandbox)) + result["tests_modified"] = any( + _sha256(self.root / name) != hashlib.sha256(self.originals[name]).hexdigest() + for name in tests + ) + return result + + +def _is_test_file(name: str) -> bool: + return name == "conftest.py" or ( + name.endswith(".py") and (name.startswith("test_") or name.endswith("_test.py")) + ) + + +def _sha256(path: Path) -> str | None: + return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None + def _as_result(run: RunResult) -> dict: if run.exit_code is None: diff --git a/tests/test_agent.py b/tests/test_agent.py index 3c4794a..37a2a5d 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -95,3 +95,66 @@ def test_token_counts_accumulate(self): result = run_agent(ws, llm) assert result.total_input_tokens > 0 assert result.total_output_tokens > 0 + + +class TestIndependentScoring: + """The verdict comes from the original tests, not from the model's own run.""" + + def test_running_only_an_already_passing_test_is_not_a_fix(self): + llm = MockLLM( + script=[ + ( + [ + { + "type": "tool_use", + "id": "tu_test", + "name": "run_tests", + "input": {"test_file": "test_buggy.py::test_zero"}, + } + ], + "tool_use", + ) + ] + ) + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + result = run_agent(ws, llm) + assert result.self_reported_pass is True + assert result.passed is False + assert "3 failed, 1 passed" in result.final_summary + + def test_editing_the_tests_is_not_a_fix(self): + rewrite = { + "path": "test_buggy.py", + "old": "from buggy import sum_first_n", + "new": "def sum_first_n(n):\n return n * (n + 1) // 2", + } + llm = MockLLM( + script=[ + ([{"type": "tool_use", "id": "tu_edit", "name": "edit_file", "input": rewrite}], "tool_use"), + ([{"type": "tool_use", "id": "tu_test", "name": "run_tests", "input": {}}], "tool_use"), + ] + ) + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + result = run_agent(ws, llm) + assert result.self_reported_pass is True + assert result.tests_modified is True + assert result.passed is False + + def test_a_real_fix_scores_as_passed_with_tests_untouched(self): + script = _build_canonical_script("01_off_by_one", "range(n)", "range(n + 1)") + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + result = run_agent(ws, MockLLM(script=script)) + assert result.passed is True + assert result.self_reported_pass is True + assert result.tests_modified is False + assert result.final_summary.startswith("PASS (exit 0) summary='4 passed") + + def test_score_ignores_files_created_during_the_run(self): + with Workspace.from_bug_dir(_bug_dir("01_off_by_one"), sandbox="local") as ws: + (ws.root / "conftest.py").write_text( + "import buggy\nbuggy.sum_first_n = lambda n: n * (n + 1) // 2\n", encoding="utf-8" + ) + assert ws.run_tests()["passed"] is True + verdict = ws.score() + assert verdict["passed"] is False + assert verdict["summary"].startswith("3 failed, 1 passed") From 95974b04b56b6c981516dcbfa06680927b559c0a Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Fri, 6 Nov 2026 00:06:31 -0500 Subject: [PATCH 07/21] Keep the bug id out of the model's view The first user message included the bug id, and the ids name the bug class (01_off_by_one, 04_mutable_default), which hands the benchmark's answer category to the model. The temp directory was named after the id too, and that path appears in tracebacks. Drop the id from the prompt and use a plain "cfa-" prefix; the id is still recorded in results. --- src/code_fix_agent/agent.py | 6 +++--- src/code_fix_agent/workspace.py | 3 ++- tests/test_agent.py | 12 ++++++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/code_fix_agent/agent.py b/src/code_fix_agent/agent.py index 422707b..98ac098 100644 --- a/src/code_fix_agent/agent.py +++ b/src/code_fix_agent/agent.py @@ -69,10 +69,10 @@ def run_agent( messages: list[dict[str, Any]] = [ { "role": "user", + # No bug id: the ids name the bug class (e.g. 04_mutable_default). "content": ( - f"Workspace contains a buggy Python file. The bug id is " - f"{ws.bug_id!r}. Find and fix the bug so that running pytest " - "on test_buggy.py passes all tests." + "Workspace contains a buggy Python file. Find and fix the bug so " + "that running pytest on test_buggy.py passes all tests." ), } ] diff --git a/src/code_fix_agent/workspace.py b/src/code_fix_agent/workspace.py index f4dff10..d915db3 100644 --- a/src/code_fix_agent/workspace.py +++ b/src/code_fix_agent/workspace.py @@ -47,7 +47,8 @@ def __post_init__(self) -> None: def from_bug_dir(cls, bug_dir: Path, cleanup: bool = True, sandbox: str = "auto") -> Workspace: bug_dir = Path(bug_dir) sandbox = resolve_sandbox(sandbox) # fail before creating the temp dir - tmp = Path(tempfile.mkdtemp(prefix=f"cfa-{bug_dir.name}-")) + # The bug id stays out of the path, which shows up in tracebacks. + tmp = Path(tempfile.mkdtemp(prefix="cfa-")) for child in bug_dir.iterdir(): if child.is_file(): shutil.copy2(child, tmp / child.name) diff --git a/tests/test_agent.py b/tests/test_agent.py index 37a2a5d..c697411 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -158,3 +158,15 @@ def test_score_ignores_files_created_during_the_run(self): verdict = ws.score() assert verdict["passed"] is False assert verdict["summary"].startswith("3 failed, 1 passed") + + +class TestNoBenchmarkHints: + def test_prompt_and_paths_do_not_name_the_bug(self): + llm = MockLLM(script=[([{"type": "tool_use", "id": "tu_ls", "name": "list_dir", "input": {}}], "tool_use")]) + with Workspace.from_bug_dir(_bug_dir("04_mutable_default")) as ws: + run_agent(ws, llm) + root_name = ws.root.name + first_prompt = llm.last_messages[0][0]["content"] + for hint in ("04_mutable_default", "mutable", "default"): + assert hint not in first_prompt + assert hint not in root_name From 46034366d8943ff8ac06c8df15cd42d8fce9f6c4 Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Fri, 6 Nov 2026 00:07:59 -0500 Subject: [PATCH 08/21] Record tool inputs and the final diff for every run The tool trail kept only input key names, so a benchmark record could not show which file the model edited or which tests it ran. Store the full tool inputs, and add the unified diff of every file changed since setup (Workspace.diff) to AgentResult, the CLI output and each benchmark record. --- benchmarks/run_benchmark.py | 2 ++ src/code_fix_agent/agent.py | 4 +++- src/code_fix_agent/cli.py | 1 + src/code_fix_agent/workspace.py | 19 +++++++++++++++++++ tests/test_agent.py | 18 ++++++++++++++++++ tests/test_bench_smoke.py | 7 +++++++ 6 files changed, 50 insertions(+), 1 deletion(-) diff --git a/benchmarks/run_benchmark.py b/benchmarks/run_benchmark.py index c323385..3b31f6a 100644 --- a/benchmarks/run_benchmark.py +++ b/benchmarks/run_benchmark.py @@ -138,6 +138,8 @@ def _run_one(bug: dict, llm: LLMClient, max_iterations: int, sandbox: str = "aut "output_tokens": result.total_output_tokens, "final_summary": result.final_summary, "error": result.error, + "final_diff": result.final_diff, + "tool_trail": result.tool_trail, } diff --git a/src/code_fix_agent/agent.py b/src/code_fix_agent/agent.py index 98ac098..830f08a 100644 --- a/src/code_fix_agent/agent.py +++ b/src/code_fix_agent/agent.py @@ -54,6 +54,7 @@ class AgentResult: error: str | None = None self_reported_pass: bool = False # the model's own run_tests said PASS tests_modified: bool = False + final_diff: str = "" def run_agent( @@ -108,7 +109,7 @@ def run_agent( { "iteration": result.iterations, "tool": tool_name, - "input_keys": sorted(tool_input.keys()), + "input": tool_input, "output_preview": output[:160], } ) @@ -132,6 +133,7 @@ def run_agent( result.passed = verdict["passed"] result.tests_modified = verdict["tests_modified"] result.final_summary = status_line(verdict) + result.final_diff = ws.diff() result.elapsed_seconds = round(time.perf_counter() - start, 3) return result diff --git a/src/code_fix_agent/cli.py b/src/code_fix_agent/cli.py index 274e20c..b9ae6f2 100644 --- a/src/code_fix_agent/cli.py +++ b/src/code_fix_agent/cli.py @@ -35,6 +35,7 @@ def _cmd_run(args: argparse.Namespace) -> int: "total_output_tokens": result.total_output_tokens, "final_summary": result.final_summary, "error": result.error, + "final_diff": result.final_diff, } print(json.dumps(payload, indent=2)) return 0 if result.passed else 1 diff --git a/src/code_fix_agent/workspace.py b/src/code_fix_agent/workspace.py index d915db3..4927e9b 100644 --- a/src/code_fix_agent/workspace.py +++ b/src/code_fix_agent/workspace.py @@ -7,6 +7,7 @@ """ from __future__ import annotations +import difflib import hashlib import shutil import tempfile @@ -124,6 +125,24 @@ def run_tests(self, test_file: str = "test_buggy.py", timeout: float = 30.0) -> target = path.relative_to(self.root.resolve()).as_posix() + sep + node return _as_result(run_pytest(self.root, [target], timeout=timeout, sandbox=self.sandbox)) + def diff(self) -> str: + """Unified diff of every file changed since setup.""" + chunks: list[str] = [] + for name, data in self.originals.items(): + current = self.root / name + after = current.read_bytes() if current.is_file() else b"" + if after == data: + continue + chunks.extend( + difflib.unified_diff( + data.decode("utf-8", errors="replace").splitlines(keepends=True), + after.decode("utf-8", errors="replace").splitlines(keepends=True), + fromfile=f"a/{name}", + tofile=f"b/{name}", + ) + ) + return "".join(chunks) + # --- scoring ---------------------------------------------------------- def score(self, timeout: float = 30.0) -> dict: diff --git a/tests/test_agent.py b/tests/test_agent.py index c697411..524169d 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -170,3 +170,21 @@ def test_prompt_and_paths_do_not_name_the_bug(self): for hint in ("04_mutable_default", "mutable", "default"): assert hint not in first_prompt assert hint not in root_name + + +class TestAuditTrail: + def test_result_records_tool_inputs_and_the_final_diff(self): + script = _build_canonical_script("01_off_by_one", "range(n)", "range(n + 1)") + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + result = run_agent(ws, MockLLM(script=script)) + assert [step["tool"] for step in result.tool_trail] == ["read_file", "edit_file", "run_tests"] + assert result.tool_trail[1]["input"] == {"path": "buggy.py", "old": "range(n)", "new": "range(n + 1)"} + assert "--- a/buggy.py" in result.final_diff + assert "- for i in range(n):" in result.final_diff + assert "+ for i in range(n + 1):" in result.final_diff + assert "test_buggy.py" not in result.final_diff + + def test_no_edit_means_an_empty_diff(self): + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + result = run_agent(ws, MockLLM(script=[])) + assert result.final_diff == "" diff --git a/tests/test_bench_smoke.py b/tests/test_bench_smoke.py index 5853847..05de4b4 100644 --- a/tests/test_bench_smoke.py +++ b/tests/test_bench_smoke.py @@ -25,6 +25,13 @@ def test_all_12_bugs_pass_in_mock_mode(self): f"failures: {[r['id'] for r in records if not r['passed']]}" ) + def test_records_carry_the_tool_trail_and_diff(self): + bug = _load_manifest()[0] + record = _run_one(bug, _build_mock_llm(bug["id"]), max_iterations=8) + old, new = _MOCK_FIXES[bug["id"]] + assert record["tool_trail"][1]["input"] == {"path": "buggy.py", "old": old, "new": new} + assert record["final_diff"].startswith("--- a/buggy.py") + def test_manifest_has_a_canonical_fix_for_every_bug(self): bugs = _load_manifest() for bug in bugs: From ea0fc3d1b129c578bc3476209d0132411a070f0d Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Fri, 6 Nov 2026 00:08:56 -0500 Subject: [PATCH 09/21] Fail the mock benchmark when a scripted fix stops passing main() ended with "0 if pass_rate >= 75 else 0", so the CI bench job could never fail. Mock mode replays known fixes and should be 12/12; exit 1 when it is not. Live mode still exits 0 and only reports. --- benchmarks/run_benchmark.py | 4 +++- tests/test_bench_smoke.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/benchmarks/run_benchmark.py b/benchmarks/run_benchmark.py index 3b31f6a..660a2f3 100644 --- a/benchmarks/run_benchmark.py +++ b/benchmarks/run_benchmark.py @@ -193,7 +193,9 @@ def main(argv: list[str] | None = None) -> int: print("\n=== code-fix-agent benchmark summary ===") print(json.dumps(summary, indent=2)) print(f"\nWrote {args.out}", file=sys.stderr) - return 0 if summary["pass_rate_pct"] >= 75 else 0 # benchmark never fails CI; readme calls out target + # Mock mode replays known fixes, so anything short of 12/12 is a pipeline + # regression. Live results are reported, not enforced. + return 1 if args.mode == "mock" and passed < n else 0 if __name__ == "__main__": diff --git a/tests/test_bench_smoke.py b/tests/test_bench_smoke.py index 05de4b4..ef4aca1 100644 --- a/tests/test_bench_smoke.py +++ b/tests/test_bench_smoke.py @@ -6,6 +6,7 @@ """ from __future__ import annotations +from benchmarks import run_benchmark from benchmarks.run_benchmark import _MOCK_FIXES, _build_mock_llm, _load_manifest, _run_one @@ -36,3 +37,14 @@ def test_manifest_has_a_canonical_fix_for_every_bug(self): bugs = _load_manifest() for bug in bugs: assert bug["id"] in _MOCK_FIXES, f"missing canonical fix for {bug['id']}" + + +class TestExitCode: + def test_mock_run_fails_when_a_scripted_fix_stops_working(self, monkeypatch, tmp_path): + monkeypatch.setitem(_MOCK_FIXES, "01_off_by_one", ("range(n)", "range(n - 1)")) + monkeypatch.setattr(run_benchmark, "_load_manifest", lambda: [{"id": "01_off_by_one"}]) + assert run_benchmark.main(["--mode", "mock", "--out", str(tmp_path / "b.json")]) == 1 + + def test_mock_run_succeeds_when_every_fix_passes(self, monkeypatch, tmp_path): + monkeypatch.setattr(run_benchmark, "_load_manifest", lambda: [{"id": "01_off_by_one"}]) + assert run_benchmark.main(["--mode", "mock", "--out", str(tmp_path / "b.json")]) == 0 From e9f87eb8e863a33e4e165253c41881ec5ff9f619 Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Fri, 6 Nov 2026 00:09:53 -0500 Subject: [PATCH 10/21] Give Anthropic API calls a 60-second timeout The SDK default is 10 minutes per request, so one stalled call could hang a benchmark run. The timeout is a constructor argument; a timed-out call ends that attempt with the error recorded. --- src/code_fix_agent/llm.py | 11 +++++++++-- tests/test_agent.py | 8 ++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/code_fix_agent/llm.py b/src/code_fix_agent/llm.py index 8863f4d..298be79 100644 --- a/src/code_fix_agent/llm.py +++ b/src/code_fix_agent/llm.py @@ -25,7 +25,12 @@ def respond( class AnthropicLLM: """Wraps anthropic.Anthropic. Returns (content, stop_reason, usage).""" - def __init__(self, model: str = "claude-haiku-4-5-20251001", api_key: str | None = None): + def __init__( + self, + model: str = "claude-haiku-4-5-20251001", + api_key: str | None = None, + timeout: float = 60.0, + ): try: from anthropic import Anthropic except ImportError as exc: @@ -33,7 +38,9 @@ def __init__(self, model: str = "claude-haiku-4-5-20251001", api_key: str | None key = api_key or os.environ.get("ANTHROPIC_API_KEY") if not key: raise RuntimeError("ANTHROPIC_API_KEY not set; use MockLLM in tests") - self._client = Anthropic(api_key=key) + # The SDK default is 10 minutes per request; a stalled call should + # fail the attempt, not hang the benchmark. + self._client = Anthropic(api_key=key, timeout=timeout) self.model = model def respond( diff --git a/tests/test_agent.py b/tests/test_agent.py index 524169d..cf56e41 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -188,3 +188,11 @@ def test_no_edit_means_an_empty_diff(self): with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: result = run_agent(ws, MockLLM(script=[])) assert result.final_diff == "" + + +class TestAnthropicClient: + def test_requests_have_an_explicit_timeout(self): + from code_fix_agent.llm import AnthropicLLM + + assert AnthropicLLM(api_key="sk-ant-test")._client.timeout == 60.0 + assert AnthropicLLM(api_key="sk-ant-test", timeout=5.0)._client.timeout == 5.0 From 181c5814cd8548c1723421e67181aee5948ee32b Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Fri, 6 Nov 2026 00:14:27 -0500 Subject: [PATCH 11/21] Correct the README's isolation, traceback and scoring claims The README said the agent never sees the test assertions (the system prompt tells it to read the test file), that only relative escapes were possible and tests were "subprocess-isolated" (they ran with the full environment and user rights), and that the model reads tracebacks (they were suppressed). Describe the confinement, the two runners and their limits, and the independent scoring as they now work. The 2026-05-20 live 12/12 stays, labelled as measured on the earlier harness (self-reported pass, bug id in the prompt, no sandbox) and not yet re-run. Drop the mock and live durations, which scoring changes. --- README.md | 59 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index abdb2b7..e033401 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # code-fix-agent -> Autonomous Python bug-fix agent on the Anthropic API. **12/12 pass on Claude Haiku 4.5 live, 3.58 mean iterations per bug, $0.10 total cost.** Real pytest-driven fix loop, isolated workspace per attempt, no jailbreaks. Reproduce live in 90 seconds. +> Autonomous Python bug-fix agent on the Anthropic API. **12/12 pass on Claude Haiku 4.5 live, 3.58 mean iterations per bug, $0.10 total cost** (May 2026 run, before the current sandbox and independent scoring; see below). Real pytest-driven fix loop, one temp workspace per attempt, test runs in a network-less container when Docker is available, and every fix scored against a clean copy of the original tests. [![ci](https://github.com/Tajaddin/code-fix-agent/actions/workflows/ci.yml/badge.svg)](https://github.com/Tajaddin/code-fix-agent/actions/workflows/ci.yml) [![License](https://img.shields.io/badge/license-MIT-blue)](LICENSE) @@ -9,16 +9,18 @@ ## Hero metrics Reproducible: -- `python -m benchmarks.run_benchmark --mode mock` (11 seconds, no API key, pipeline smoke) -- `python -m benchmarks.run_benchmark --mode live` (90 seconds, needs ANTHROPIC_API_KEY, real Haiku 4.5) +- `python -m benchmarks.run_benchmark --mode mock` (no API key, pipeline smoke) +- `python -m benchmarks.run_benchmark --mode live` (needs ANTHROPIC_API_KEY, real Haiku 4.5) | Mode | Pass rate | Mean iterations / bug | Mean tool calls / bug | Total tokens | Realized cost (Haiku 4.5) | |---|---:|---:|---:|---:|---:| | **mock (pipeline smoke)** | **12 / 12** | 3.0 | 3.0 | 540 | $0.0013 | -| **live (Haiku 4.5)** | **12 / 12** | **3.58** | **5.58** | 77,457 | **$0.1012** | +| **live (Haiku 4.5), 2026-05-20** | **12 / 12** | **3.58** | **5.58** | 77,457 | **$0.1012** | Live run logged 2026-05-20: all 12 bugs fixed, no skips, no errors. The agent average 3.58 LLM rounds and 5.58 tool calls per bug. Total spend across the 12-bug sweep was ~10 cents. Per-bug records are in `benchmarks/results/benchmark.json`. +That run predates the current harness. Its pass was the agent's own last `run_tests` call, not the independent scoring described under [Isolation and scoring](#isolation-and-scoring), its prompt included the bug id (which names the bug kind), its tests ran without a sandbox, and its records do not include tool inputs or diffs. Each record's summary shows the bug's full test count passing, so no bug passed on a subset of its tests. It has not yet been re-run on the current harness. + The mock mode scripts the canonical (read -> edit -> run_tests) for each of the 12 bugs. A perfect 12/12 in mock means the workspace, tool dispatch, and agent loop are healthy; CI runs this on every push. Live mode hands the same 12 bugs to Claude Haiku 4.5 via the Anthropic API, with no scripted fixes. To reproduce: @@ -30,7 +32,7 @@ python -m benchmarks.run_benchmark --mode live --max-iterations 8 ## The dataset -12 hand-crafted Python bugs in `bugs/`. Each subdirectory contains `buggy.py` plus `test_buggy.py`. The fix is a 1-3 line surgical edit. The test file is held back from the agent in its system prompt instructions (the agent is told not to modify the test). +12 hand-crafted Python bugs in `bugs/`. Each subdirectory contains `buggy.py` plus `test_buggy.py`. The fix is a 1-3 line surgical edit. The agent may read the test file but is told not to change it; scoring ignores any change it makes. | ID | Bug kind | Expected iters | |---|---|---:| @@ -53,7 +55,8 @@ Each bug has 3-4 pytest assertions; the agent needs to satisfy all of them. | Path | Purpose | |---|---| -| `src/code_fix_agent/workspace.py` | Isolated tempdir per attempt. Read, edit (str_replace with uniqueness check), search, run_tests via subprocess. | +| `src/code_fix_agent/workspace.py` | Tempdir per attempt. Read, edit (str_replace with uniqueness check), search and run_tests, all confined to the workspace root; `score()` and `diff()` for the harness. | +| `src/code_fix_agent/sandbox.py` | Runs pytest in a network-less Docker container, or in a subprocess with a minimal environment when Docker is unavailable. | | `src/code_fix_agent/tools.py` | Five Anthropic tool definitions and their dispatcher: list_dir, read_file, edit_file, search_code, run_tests. | | `src/code_fix_agent/agent.py` | The fix loop. Reads response content, dispatches tool_use blocks, sends tool_result back, stops when run_tests reports PASS or iteration budget is exhausted. | | `src/code_fix_agent/llm.py` | `AnthropicLLM` + `MockLLM` (scripted tool-call sequence). Both return the same `(content, stop_reason, usage)` shape. | @@ -87,13 +90,18 @@ Sample output (`run bugs/01_off_by_one`): ```json { "bug_id": "01_off_by_one", + "sandbox": "docker", "passed": true, + "self_reported_pass": true, + "tests_modified": false, "iterations": 3, "tool_calls": 3, "elapsed_seconds": 4.21, "total_input_tokens": 2843, "total_output_tokens": 142, - "final_summary": "PASS (exit 0) summary='4 passed in 0.03s'" + "final_summary": "PASS (exit 0) summary='4 passed in 0.03s'", + "error": null, + "final_diff": "--- a/buggy.py\n+++ b/buggy.py\n@@ -1,6 +1,6 @@\n ..." } ``` @@ -124,18 +132,29 @@ Sample output (`run bugs/01_off_by_one`): | yes v - +-----+-----+ - | PASS | - +-----------+ + +-----------+-----------+ + | score: full original | (also runs when the budget + | suite on a clean copy | runs out) + +-----------------------+ ``` -The agent does NOT see the test file's assertions in its prompt. It learns the bug shape from `read_file("buggy.py")` and the failure trace from the first `run_tests` call. This mirrors what a junior engineer sees on the first day. +The prompt names neither the bug nor its kind. The agent reads `buggy.py` and `test_buggy.py` with `read_file` (the system prompt tells it to) and gets a short traceback (`--tb=short`, last 4,000 characters) from every failing `run_tests` call. + +## Isolation and scoring -## Workspace isolation +**Files.** Each attempt runs in a fresh `tempfile.mkdtemp(prefix="cfa-")`. Every tool resolves its path, symlinks included, and refuses anything outside the workspace root with the same error whether or not the outside path exists. `run_tests` accepts only a test file inside the workspace (optionally `file::test`), never pytest options. -Each bug fix runs in a fresh `tempfile.mkdtemp(prefix="cfa--")`. The agent cannot escape via relative paths (`..`); `edit_file` checks `path.resolve()` falls under the workspace root. Test execution is subprocess-isolated with a 30-second timeout. +**Test runs.** The model writes the code that `run_tests` executes, so it is treated as untrusted. `--sandbox` on the CLI and the benchmark picks the runner: -This means a single failed agent run cannot corrupt the bug fixtures or any other repo state. +| `--sandbox` | Runner | +|---|---| +| `docker` | Throwaway container from `code-fix-agent:latest` (override with `CFA_SANDBOX_IMAGE`): `--network none`, read-only root and workspace mount, all capabilities dropped, `no-new-privileges`, 512 MB / 1 CPU / 128 processes, non-root, no host environment. Fails if the image is missing. | +| `local` | Subprocess with only `PATH`, `SYSTEMROOT` and the temp-dir variables (no API key), in its own process group so the 30-second timeout kills everything the tests started. It still has the user's file-system and network access: a fallback, not a sandbox. | +| `auto` (default) | `docker` when `docker image inspect` finds the image, otherwise `local`. | + +Build the image once with `docker build -t code-fix-agent:latest .`. Every result records the runner used. + +**Scoring.** The agent's own `run_tests` result is not the verdict: it chooses which tests to run and can edit the test file. After the loop, the harness rebuilds the bug in a fresh directory from the files present at setup (tests from the setup snapshot, other files from the workspace, nothing the run created) and runs the whole suite with the same runner. That is `passed`. `self_reported_pass` keeps the agent's claim, and `tests_modified` flags test files that no longer match their SHA-256 from setup. Benchmark records also keep every tool input and the final diff. ## Testing @@ -143,11 +162,11 @@ This means a single failed agent run cannot corrupt the bug fixtures or any othe pytest --cov=code_fix_agent ``` -22 tests: -- `test_workspace.py` — copy, list, read, edit (success + missing-substring error), run_tests (fails on unfixed, passes on canonical fix). -- `test_tools.py` — schema completeness, dispatch, error surfacing. -- `test_agent.py` — full read-edit-test loop with `MockLLM` on bugs 1 and 2, max-iteration safety, LLM exception handling, token accounting. -- `test_bench_smoke.py` — runs the full mock benchmark (12 bugs) and asserts 12/12 pass. +56 tests (the Docker one is skipped when the image is not built, the symlink one where creating symlinks needs admin rights): +- `test_workspace.py` — copy, list, read, edit (success + missing-substring error), run_tests (fails on unfixed, passes on canonical fix), no inherited secrets, timeout kills child processes, sandbox selection and the Docker command line. +- `test_tools.py` — schema completeness, dispatch, error surfacing, tracebacks in failing output, path confinement for every tool (`..`, absolute paths, symlinks). +- `test_agent.py` — full read-edit-test loop with `MockLLM` on bugs 1 and 2, max-iteration safety, LLM exception handling, token accounting, scoring against subset runs, edited tests and a planted `conftest.py`, no bug id in the prompt, tool inputs and diff in the result, API timeout. +- `test_bench_smoke.py` — runs the full mock benchmark (12 bugs) and asserts 12/12 pass; the mock run exits non-zero on any failure. The bench smoke test is the canary: if it ever drops below 12/12 in CI, either the dataset or the workspace layer regressed. @@ -158,6 +177,8 @@ docker compose run --rm bench # full mock benchmark docker compose run --rm fix run bugs/01_off_by_one # one bug with live API ``` +The image runs as a non-root user. It is also the test sandbox image (`code-fix-agent:latest`); inside a compose container there is no Docker CLI, so tests there use the local runner within that container. + ## Project layout ``` From d03319c4b1b6b0fca8b01d529dd59b4d36053f51 Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Fri, 6 Nov 2026 02:08:31 -0500 Subject: [PATCH 12/21] Require every original test to pass when scoring score() took pytest's exit code as the verdict, and the code under test controls it. With only buggy.py edited, pytest.skip() or pytest.xfail() in the function scored as passed ("4 skipped", "4 xfailed"), and os._exit(0) at import scored as passed with no output at all. score() now collects the node ids of the bug as it was at setup (cached per file contents, since every attempt at a bug collects the same tests), runs the suite with -rfEsxXp, and passes only when exactly those ids are reported PASSED and the final counts hold nothing but passes (warnings aside). Exit code 0 is still required. The verdict is still read from the output of the pytest process the code under test runs in, so deliberately forged output is not caught; the README says so. --- README.md | 6 ++-- src/code_fix_agent/workspace.py | 51 +++++++++++++++++++++++++++++++-- tests/test_agent.py | 34 ++++++++++++++++++++++ 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e033401..57f3f8f 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ The prompt names neither the bug nor its kind. The agent reads `buggy.py` and `t Build the image once with `docker build -t code-fix-agent:latest .`. Every result records the runner used. -**Scoring.** The agent's own `run_tests` result is not the verdict: it chooses which tests to run and can edit the test file. After the loop, the harness rebuilds the bug in a fresh directory from the files present at setup (tests from the setup snapshot, other files from the workspace, nothing the run created) and runs the whole suite with the same runner. That is `passed`. `self_reported_pass` keeps the agent's claim, and `tests_modified` flags test files that no longer match their SHA-256 from setup. Benchmark records also keep every tool input and the final diff. +**Scoring.** The agent's own `run_tests` result is not the verdict: it chooses which tests to run and can edit the test file. After the loop, the harness rebuilds the bug in a fresh directory from the files present at setup (tests from the setup snapshot, other files from the workspace, nothing the run created) and runs the whole suite with the same runner. That is `passed`, and exit code 0 is not enough: every test collected from the bug as it was at setup must be reported `PASSED` and the counts must hold nothing else, so a `pytest.skip()`, a `pytest.xfail()` or an early `os._exit(0)` in the source fails. The verdict is read from pytest's output, and under either runner the code under test runs inside that pytest process: code written to forge the output (fake result lines, patched reporting) is not caught. `self_reported_pass` keeps the agent's claim, and `tests_modified` flags test files that no longer match their SHA-256 from setup. Benchmark records also keep every tool input and the final diff. ## Testing @@ -162,10 +162,10 @@ Build the image once with `docker build -t code-fix-agent:latest .`. Every resul pytest --cov=code_fix_agent ``` -56 tests (the Docker one is skipped when the image is not built, the symlink one where creating symlinks needs admin rights): +61 tests (the Docker one is skipped when the image is not built, the symlink one where creating symlinks needs admin rights): - `test_workspace.py` — copy, list, read, edit (success + missing-substring error), run_tests (fails on unfixed, passes on canonical fix), no inherited secrets, timeout kills child processes, sandbox selection and the Docker command line. - `test_tools.py` — schema completeness, dispatch, error surfacing, tracebacks in failing output, path confinement for every tool (`..`, absolute paths, symlinks). -- `test_agent.py` — full read-edit-test loop with `MockLLM` on bugs 1 and 2, max-iteration safety, LLM exception handling, token accounting, scoring against subset runs, edited tests and a planted `conftest.py`, no bug id in the prompt, tool inputs and diff in the result, API timeout. +- `test_agent.py` — full read-edit-test loop with `MockLLM` on bugs 1 and 2, max-iteration safety, LLM exception handling, token accounting, scoring against subset runs, edited tests, a planted `conftest.py` and source that skips, xfails or exits early; no bug id in the prompt, tool inputs and diff in the result, API timeout. - `test_bench_smoke.py` — runs the full mock benchmark (12 bugs) and asserts 12/12 pass; the mock run exits non-zero on any failure. The bench smoke test is the canary: if it ever drops below 12/12 in CI, either the dataset or the workspace layer regressed. diff --git a/src/code_fix_agent/workspace.py b/src/code_fix_agent/workspace.py index 4927e9b..3b7a910 100644 --- a/src/code_fix_agent/workspace.py +++ b/src/code_fix_agent/workspace.py @@ -9,10 +9,12 @@ import difflib import hashlib +import re import shutil import tempfile from contextlib import contextmanager from dataclasses import dataclass, field +from functools import cache from pathlib import Path from code_fix_agent.sandbox import RunResult, resolve_sandbox, run_pytest @@ -155,8 +157,16 @@ def score(self, timeout: float = 30.0) -> dict: setup snapshot and every other file from the workspace, and runs the whole suite. `tests_modified` reports whether the workspace's test files still match the snapshot's SHA-256. + + Exit code 0 is not enough: the code under test can skip or xfail the + tests, or exit early, and pytest still exits 0. A pass needs every + test collected from the bug as it was at setup to be reported PASSED, + and nothing else (failed, skipped, xfailed, errors) in the counts. """ - tests = sorted(name for name in self.originals if _is_test_file(name)) + tests = tuple(sorted(name for name in self.originals if _is_test_file(name))) + expected = _original_tests( + tuple(sorted(self.originals.items())), tests, self.sandbox, timeout + ) with tempfile.TemporaryDirectory(prefix="cfa-score-", ignore_cleanup_errors=True) as tmp: dest = Path(tmp) for name, data in self.originals.items(): @@ -165,7 +175,9 @@ def score(self, timeout: float = 30.0) -> dict: (dest / name).write_bytes(data) elif current.is_file(): shutil.copyfile(current, dest / name) - result = _as_result(run_pytest(dest, tests, timeout=timeout, sandbox=self.sandbox)) + run = run_pytest(dest, ["-rfEsxXp", *tests], timeout=timeout, sandbox=self.sandbox) + result = _as_result(run) + result["passed"] = result["passed"] and _only_these_passed(run.stdout, expected) result["tests_modified"] = any( _sha256(self.root / name) != hashlib.sha256(self.originals[name]).hexdigest() for name in tests @@ -173,6 +185,41 @@ def score(self, timeout: float = 30.0) -> dict: return result +@cache +def _original_tests( + files: tuple[tuple[str, bytes], ...], tests: tuple[str, ...], sandbox: str, timeout: float +) -> tuple[str, ...]: + """Node ids pytest collects from `files`, the bug as it was at setup. + + Cached on the file contents: every attempt at the same bug collects the + same tests. + """ + with tempfile.TemporaryDirectory(prefix="cfa-collect-", ignore_cleanup_errors=True) as tmp: + for name, data in files: + (Path(tmp) / name).write_bytes(data) + run = run_pytest(Path(tmp), ["--collect-only", *tests], timeout=timeout, sandbox=sandbox) + ids = [] + for line in run.stdout.splitlines(): + file_part, sep, _ = line.strip().partition("::") + if sep and file_part in tests: + ids.append(line.strip()) + if run.exit_code != 0 or not ids: + raise RuntimeError(f"could not collect the original tests: {run.stdout[-1000:]!r}") + return tuple(ids) + + +def _only_these_passed(stdout: str, expected: tuple[str, ...]) -> bool: + """True if pytest's -rp lines name exactly `expected` and its counts are all passes.""" + lines = stdout.splitlines() + passed = {line.removeprefix("PASSED ").strip() for line in lines if line.startswith("PASSED ")} + # The last line reads e.g. "4 passed, 1 warning in 0.02s". + summary = next((line.strip() for line in reversed(lines) if line.strip()), "") + counts = {kind: int(n) for n, kind in re.findall(r"(\d+) (\w+)", summary.split(" in ")[0])} + counts.pop("warning", None) + counts.pop("warnings", None) + return passed == set(expected) and counts == {"passed": len(expected)} + + def _is_test_file(name: str) -> bool: return name == "conftest.py" or ( name.endswith(".py") and (name.startswith("test_") or name.endswith("_test.py")) diff --git a/tests/test_agent.py b/tests/test_agent.py index cf56e41..7c3bcc7 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -3,6 +3,8 @@ from pathlib import Path +import pytest + from code_fix_agent.agent import Agent, run_agent from code_fix_agent.llm import MockLLM from code_fix_agent.workspace import Workspace @@ -159,6 +161,38 @@ def test_score_ignores_files_created_during_the_run(self): assert verdict["passed"] is False assert verdict["summary"].startswith("3 failed, 1 passed") + @pytest.mark.parametrize( + ("old", "new", "summary"), + [ + (" total = 0", " __import__('pytest').skip('x')\n total = 0", "4 skipped"), + (" total = 0", " __import__('pytest').xfail('x')\n total = 0", "4 xfailed"), + ( + "def sum_first_n", + "import pytest\npytest.skip('x', allow_module_level=True)\ndef sum_first_n", + "1 skipped", + ), + ("def sum_first_n", "import os\nos._exit(0)\ndef sum_first_n", ""), + ], + ids=["skip", "xfail", "module-skip", "exit-0"], + ) + def test_source_that_dodges_the_tests_is_not_a_fix(self, old, new, summary): + # Each edit touches only buggy.py and leaves pytest exiting 0 or 5, + # with no test having passed. + with Workspace.from_bug_dir(_bug_dir("01_off_by_one"), sandbox="local") as ws: + ws.edit("buggy.py", old, new) + verdict = ws.score() + assert verdict["passed"] is False + assert verdict["tests_modified"] is False + assert verdict["summary"].startswith(summary) + + def test_warnings_do_not_fail_a_real_fix(self): + with Workspace.from_bug_dir(_bug_dir("01_off_by_one"), sandbox="local") as ws: + ws.edit("buggy.py", "range(n)", "range(n + 1)") + ws.edit("buggy.py", "def sum_first_n", "import warnings\nwarnings.warn('w')\ndef sum_first_n") + verdict = ws.score() + assert verdict["passed"] is True, verdict["stdout"] + assert verdict["summary"].startswith("4 passed, 1 warning") + class TestNoBenchmarkHints: def test_prompt_and_paths_do_not_name_the_bug(self): From 4dcdd2282afc9b2e9d1108fa02d3faf7f19ba4c8 Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Fri, 6 Nov 2026 02:10:35 -0500 Subject: [PATCH 13/21] Refuse live runs on the local runner unless asked The local runner gives the tests a clean environment, but they run as the same user next to the agent process, which holds ANTHROPIC_API_KEY. On Linux a test reads it from /proc//environ (shown in WSL), and the runner has network access. So the clean environment only stops accidental inheritance; the key is kept from model-written code only under --sandbox docker. Behaviour change: `code-fix-agent run` and `run_benchmark --mode live` no longer let --sandbox auto fall back to the local runner when the sandbox image is missing; they fail before calling the model and ask for the image or an explicit --sandbox local. Mock runs, which replay known fixes, still fall back. The compose fix service, which has no Docker CLI inside, now passes --sandbox local. The README and the sandbox docstring also stop saying the timeout kills everything the tests started: a process that calls setsid(), or on Windows one whose parent has exited, is not in the killed group/tree. --- README.md | 19 +++++++++++-------- benchmarks/run_benchmark.py | 4 +++- docker-compose.yml | 2 +- src/code_fix_agent/cli.py | 10 +++++++--- src/code_fix_agent/sandbox.py | 35 +++++++++++++++++++++++++++-------- tests/test_workspace.py | 26 +++++++++++++++++++++++++- 6 files changed, 74 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 57f3f8f..b734e27 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,12 @@ Live mode hands the same 12 bugs to Claude Haiku 4.5 via the Anthropic API, with ```bash export ANTHROPIC_API_KEY=sk-ant-... +docker build -t code-fix-agent:latest . # the test sandbox; see below python -m benchmarks.run_benchmark --mode live --max-iterations 8 ``` +Live runs refuse to start without the sandbox image unless you pass `--sandbox local`, because the local runner cannot keep the API key from model-written code (see [Isolation and scoring](#isolation-and-scoring)). + ## The dataset 12 hand-crafted Python bugs in `bugs/`. Each subdirectory contains `buggy.py` plus `test_buggy.py`. The fix is a 1-3 line surgical edit. The agent may read the test file but is told not to change it; scoring ignores any change it makes. @@ -78,7 +81,7 @@ This is the small-bench cousin of SWE-bench-Lite. It runs in seconds on a laptop pip install -e ".[dev]" export ANTHROPIC_API_KEY=sk-ant-... -# Fix one bug, print JSON result +# Fix one bug, print JSON result (needs the sandbox image, or --sandbox local) code-fix-agent run bugs/01_off_by_one # Or run the full 12-bug benchmark @@ -149,8 +152,8 @@ The prompt names neither the bug nor its kind. The agent reads `buggy.py` and `t | `--sandbox` | Runner | |---|---| | `docker` | Throwaway container from `code-fix-agent:latest` (override with `CFA_SANDBOX_IMAGE`): `--network none`, read-only root and workspace mount, all capabilities dropped, `no-new-privileges`, 512 MB / 1 CPU / 128 processes, non-root, no host environment. Fails if the image is missing. | -| `local` | Subprocess with only `PATH`, `SYSTEMROOT` and the temp-dir variables (no API key), in its own process group so the 30-second timeout kills everything the tests started. It still has the user's file-system and network access: a fallback, not a sandbox. | -| `auto` (default) | `docker` when `docker image inspect` finds the image, otherwise `local`. | +| `local` | Subprocess with only `PATH`, `SYSTEMROOT` and the temp-dir variables in its environment, in its own session (POSIX) or process group (Windows). The clean environment stops the API key from being inherited, not from being read: the tests run as your user, next to the agent process that holds the key, and can read it from there (on Linux, `/proc//environ`). They also have your file-system and network access. The 30-second timeout kills the test process's group or tree; a process that calls `setsid()`, or on Windows one whose parent already exited, survives it. A fallback, not a sandbox. | +| `auto` (default) | `docker` when `docker image inspect` finds the image. Otherwise `local` for mock runs, and an error for live runs (`code-fix-agent run`, `--mode live`): pass `--sandbox local` to run those without the image. | Build the image once with `docker build -t code-fix-agent:latest .`. Every result records the runner used. @@ -162,8 +165,8 @@ Build the image once with `docker build -t code-fix-agent:latest .`. Every resul pytest --cov=code_fix_agent ``` -61 tests (the Docker one is skipped when the image is not built, the symlink one where creating symlinks needs admin rights): -- `test_workspace.py` — copy, list, read, edit (success + missing-substring error), run_tests (fails on unfixed, passes on canonical fix), no inherited secrets, timeout kills child processes, sandbox selection and the Docker command line. +64 tests (the Docker one is skipped when the image is not built, the symlink one where creating symlinks needs admin rights): +- `test_workspace.py` — copy, list, read, edit (success + missing-substring error), run_tests (fails on unfixed, passes on canonical fix), no inherited secrets, timeout kills child processes, sandbox selection (live runs do not fall back to the local runner) and the Docker command line. - `test_tools.py` — schema completeness, dispatch, error surfacing, tracebacks in failing output, path confinement for every tool (`..`, absolute paths, symlinks). - `test_agent.py` — full read-edit-test loop with `MockLLM` on bugs 1 and 2, max-iteration safety, LLM exception handling, token accounting, scoring against subset runs, edited tests, a planted `conftest.py` and source that skips, xfails or exits early; no bug id in the prompt, tool inputs and diff in the result, API timeout. - `test_bench_smoke.py` — runs the full mock benchmark (12 bugs) and asserts 12/12 pass; the mock run exits non-zero on any failure. @@ -173,11 +176,11 @@ The bench smoke test is the canary: if it ever drops below 12/12 in CI, either t ## Docker ```bash -docker compose run --rm bench # full mock benchmark -docker compose run --rm fix run bugs/01_off_by_one # one bug with live API +docker compose run --rm bench # full mock benchmark +docker compose run --rm fix run --sandbox local bugs/01_off_by_one # one bug with live API ``` -The image runs as a non-root user. It is also the test sandbox image (`code-fix-agent:latest`); inside a compose container there is no Docker CLI, so tests there use the local runner within that container. +The image runs as a non-root user. It is also the test sandbox image (`code-fix-agent:latest`). Inside a compose container there is no Docker CLI, so tests there use the local runner within that container, which holds the API key: the `fix` service passes `--sandbox local` to accept that. ## Project layout diff --git a/benchmarks/run_benchmark.py b/benchmarks/run_benchmark.py index 660a2f3..f22a7a4 100644 --- a/benchmarks/run_benchmark.py +++ b/benchmarks/run_benchmark.py @@ -158,7 +158,9 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) bugs = _load_manifest() - sandbox = resolve_sandbox(args.sandbox) + # Live runs execute model-written code, so they only use the local runner + # when asked to; mock runs replay known fixes. + sandbox = resolve_sandbox(args.sandbox, allow_local_fallback=args.mode == "mock") records: list[dict] = [] for bug in bugs: if args.mode == "live": diff --git a/docker-compose.yml b/docker-compose.yml index 56dc9d9..2f87418 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: image: code-fix-agent:latest environment: - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} - command: ["run", "bugs/01_off_by_one"] + command: ["run", "--sandbox", "local", "bugs/01_off_by_one"] bench: build: . diff --git a/src/code_fix_agent/cli.py b/src/code_fix_agent/cli.py index b9ae6f2..7b44a69 100644 --- a/src/code_fix_agent/cli.py +++ b/src/code_fix_agent/cli.py @@ -13,14 +13,15 @@ from code_fix_agent.agent import Agent from code_fix_agent.llm import AnthropicLLM -from code_fix_agent.sandbox import SANDBOX_MODES +from code_fix_agent.sandbox import SANDBOX_MODES, resolve_sandbox from code_fix_agent.workspace import Workspace def _cmd_run(args: argparse.Namespace) -> int: + sandbox = resolve_sandbox(args.sandbox, allow_local_fallback=False) llm = AnthropicLLM(model=args.model) agent = Agent(llm=llm, max_iterations=args.max_iterations) - with Workspace.from_bug_dir(Path(args.bug_dir), sandbox=args.sandbox) as ws: + with Workspace.from_bug_dir(Path(args.bug_dir), sandbox=sandbox) as ws: result = agent.fix(ws) payload = { "bug_id": result.bug_id, @@ -53,7 +54,10 @@ def main(argv: list[str] | None = None) -> int: "--sandbox", choices=SANDBOX_MODES, default="auto", - help="where tests run: docker (no network, read-only), local, or auto (docker if the image exists)", + help=( + "where tests run: docker (no network, read-only), local (no isolation; the " + "tests can reach the API key), or auto (docker, or an error if the image is missing)" + ), ) p_run.set_defaults(func=_cmd_run) diff --git a/src/code_fix_agent/sandbox.py b/src/code_fix_agent/sandbox.py index 1d013da..0ff4e00 100644 --- a/src/code_fix_agent/sandbox.py +++ b/src/code_fix_agent/sandbox.py @@ -6,12 +6,18 @@ - docker: a throwaway container with no network, a read-only file system and workspace mount, no capabilities, and memory/CPU/process limits. Nothing from the caller's environment is passed in. -- local: a subprocess with a minimal environment (no API keys), in its own - process group so a timeout kills every process the tests started. It still - runs with the caller's file-system rights, so it is the fallback for when - Docker or the sandbox image is not available. - -"auto" picks docker when the `docker` CLI can see the sandbox image. +- local: a subprocess with a minimal environment, in its own session + (POSIX) or process group (Windows); a timeout kills that group or tree, + which misses a process that left the group (setsid) or, on Windows, one + whose parent already exited. The clean environment only stops accidental + inheritance of secrets: the tests run as the caller's user, next to the + agent process that holds ANTHROPIC_API_KEY, and can read it from there + (on Linux, from /proc//environ). They also have the caller's + file-system and network access. It is a fallback, not a sandbox. + +"auto" picks docker when the `docker` CLI can see the sandbox image. Live +runs, where the model writes the code, turn off auto's local fallback, so +running them without the image takes an explicit "local". """ from __future__ import annotations @@ -55,8 +61,14 @@ def docker_available(image: str = SANDBOX_IMAGE) -> bool: return probe.returncode == 0 -def resolve_sandbox(mode: str, image: str = SANDBOX_IMAGE) -> str: - """Map a requested mode to the runner that will be used: "docker" or "local".""" +def resolve_sandbox( + mode: str, image: str = SANDBOX_IMAGE, *, allow_local_fallback: bool = True +) -> str: + """Map a requested mode to the runner that will be used: "docker" or "local". + + With allow_local_fallback=False, "auto" fails instead of falling back to + the local runner when the image is missing. + """ if mode not in SANDBOX_MODES: raise ValueError(f"sandbox must be one of {SANDBOX_MODES}, got {mode!r}") if mode == "local": @@ -68,6 +80,13 @@ def resolve_sandbox(mode: str, image: str = SANDBOX_IMAGE) -> str: f"docker sandbox requested but image {image!r} is not available " f"(build it with `docker build -t {image} .`)" ) + if not allow_local_fallback: + raise RuntimeError( + f"sandbox image {image!r} is not available. Under the local runner, " + "model-written code runs as your user and can read ANTHROPIC_API_KEY " + f"from the agent process. Build the image with `docker build -t {image} .`, " + "or pass --sandbox local to run without it." + ) return "local" diff --git a/tests/test_workspace.py b/tests/test_workspace.py index a833f76..26487f5 100644 --- a/tests/test_workspace.py +++ b/tests/test_workspace.py @@ -7,7 +7,8 @@ import pytest -from code_fix_agent import sandbox +from benchmarks import run_benchmark +from code_fix_agent import cli, sandbox from code_fix_agent.workspace import Workspace @@ -107,6 +108,10 @@ def test_hangs(): assert not late.exists(), "a process started by the tests outlived the timeout" +def _no_model(*args, **kwargs): + raise AssertionError("the run reached the model before checking the sandbox") + + class TestSandboxSelection: def test_auto_falls_back_to_local_without_the_image(self, monkeypatch): monkeypatch.setattr(sandbox, "docker_available", lambda image=sandbox.SANDBOX_IMAGE: False) @@ -127,6 +132,25 @@ def test_unknown_mode_is_rejected(self): with pytest.raises(ValueError): sandbox.resolve_sandbox("chroot") + def test_auto_without_the_image_refuses_when_local_fallback_is_off(self, monkeypatch): + monkeypatch.setattr(sandbox, "docker_available", lambda image=sandbox.SANDBOX_IMAGE: False) + with pytest.raises(RuntimeError, match="--sandbox local"): + sandbox.resolve_sandbox("auto", allow_local_fallback=False) + assert sandbox.resolve_sandbox("local", allow_local_fallback=False) == "local" + + def test_live_cli_run_does_not_fall_back_to_local(self, monkeypatch): + monkeypatch.setattr(sandbox, "docker_available", lambda image=sandbox.SANDBOX_IMAGE: False) + monkeypatch.setattr(cli, "AnthropicLLM", _no_model) + with pytest.raises(RuntimeError, match="--sandbox local"): + cli.main(["run", str(_bug_dir("01_off_by_one"))]) + + def test_live_benchmark_does_not_fall_back_to_local(self, monkeypatch, tmp_path): + monkeypatch.setattr(sandbox, "docker_available", lambda image=sandbox.SANDBOX_IMAGE: False) + monkeypatch.setattr(run_benchmark, "AnthropicLLM", _no_model) + with pytest.raises(RuntimeError, match="--sandbox local"): + run_benchmark.main(["--mode", "live", "--out", str(tmp_path / "b.json")]) + assert not (tmp_path / "b.json").exists() + def test_docker_command_shuts_off_network_env_and_writes(self, tmp_path): cmd = sandbox.docker_command(tmp_path, ["test_buggy.py"], name="cfa-test") pairs = set(zip(cmd, cmd[1:])) From 45111ddecb135698ec33c736d2091a2e501077fc Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Fri, 6 Nov 2026 02:11:36 -0500 Subject: [PATCH 14/21] Give the image's user the whole results directory chown without -R changed only benchmarks/results itself. The benchmark.json copied in by COPY stayed root-owned and 0644, so as USER app a benchmark run writing that file failed with PermissionError (owning the directory allows replacing a file, not writing into it). Not built here: Docker was not available. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 240eee2..0d965cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ COPY README.md LICENSE ./ RUN pip install --no-cache-dir -e ".[dev]" \ && useradd --create-home --uid 10001 app \ - && chown app benchmarks/results + && chown -R app benchmarks/results USER app From 89fcb8f786e99b13684075dd18c37c6e0f3282fb Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Fri, 6 Nov 2026 02:12:15 -0500 Subject: [PATCH 15/21] Run the Docker sandbox test and mock benchmark in CI The Docker runner had never run anywhere: the CI docker job built the image as code-fix-agent:ci, which the runner does not look for, and only imported the package, so TestDockerSandbox was skipped in CI as well. The job now tags the image code-fix-agent:latest, installs the package on the runner, runs the Docker tests and the mock benchmark with --sandbox docker (which fails outright if the image is not found), and runs the mock benchmark inside the image as its non-root user over the copied benchmark.json, which catches the ownership bug fixed in the previous commit. Until that job passes, the README marks the Docker runner as untested instead of describing its guarantees as working fact. --- .github/workflows/ci.yml | 19 +++++++++++++++++-- README.md | 4 +++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad1fd89..29898c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,5 +58,20 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false - - run: docker build -t code-fix-agent:ci . - - run: docker run --rm --entrypoint python code-fix-agent:ci -c "from code_fix_agent import Agent; print('ok')" + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + cache: pip + # The tag the sandbox runner looks for. + - name: Build the sandbox image + run: docker build -t code-fix-agent:latest . + - name: Install + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + - name: Sandbox tests + run: pytest tests/test_workspace.py -k Docker -rs -v + - name: Mock benchmark in the sandbox + run: python -m benchmarks.run_benchmark --mode mock --sandbox docker --out "$RUNNER_TEMP/bench-docker.json" + - name: Mock benchmark as the image's non-root user, over a copied results file + run: docker run --rm --entrypoint python code-fix-agent:latest -m benchmarks.run_benchmark --mode mock --out benchmarks/results/benchmark.json diff --git a/README.md b/README.md index b734e27..556b024 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # code-fix-agent -> Autonomous Python bug-fix agent on the Anthropic API. **12/12 pass on Claude Haiku 4.5 live, 3.58 mean iterations per bug, $0.10 total cost** (May 2026 run, before the current sandbox and independent scoring; see below). Real pytest-driven fix loop, one temp workspace per attempt, test runs in a network-less container when Docker is available, and every fix scored against a clean copy of the original tests. +> Autonomous Python bug-fix agent on the Anthropic API. **12/12 pass on Claude Haiku 4.5 live, 3.58 mean iterations per bug, $0.10 total cost** (May 2026 run, before the current sandbox and independent scoring; see below). Real pytest-driven fix loop, one temp workspace per attempt, a network-less Docker runner for the tests (not yet run; see [Isolation and scoring](#isolation-and-scoring)), and every fix scored against a clean copy of the original tests. [![ci](https://github.com/Tajaddin/code-fix-agent/actions/workflows/ci.yml/badge.svg)](https://github.com/Tajaddin/code-fix-agent/actions/workflows/ci.yml) [![License](https://img.shields.io/badge/license-MIT-blue)](LICENSE) @@ -157,6 +157,8 @@ The prompt names neither the bug nor its kind. The agent reads `buggy.py` and `t Build the image once with `docker build -t code-fix-agent:latest .`. Every result records the runner used. +**The Docker runner is untested so far:** it has not run on any machine yet. The CI `docker` job builds the image, runs the container test (`TestDockerSandbox`) and the mock benchmark with `--sandbox docker`, and runs the benchmark inside the image as its non-root user; until that job passes, treat the `docker` row above as a design, not a result. + **Scoring.** The agent's own `run_tests` result is not the verdict: it chooses which tests to run and can edit the test file. After the loop, the harness rebuilds the bug in a fresh directory from the files present at setup (tests from the setup snapshot, other files from the workspace, nothing the run created) and runs the whole suite with the same runner. That is `passed`, and exit code 0 is not enough: every test collected from the bug as it was at setup must be reported `PASSED` and the counts must hold nothing else, so a `pytest.skip()`, a `pytest.xfail()` or an early `os._exit(0)` in the source fails. The verdict is read from pytest's output, and under either runner the code under test runs inside that pytest process: code written to forge the output (fake result lines, patched reporting) is not caught. `self_reported_pass` keeps the agent's claim, and `tests_modified` flags test files that no longer match their SHA-256 from setup. Benchmark records also keep every tool input and the final diff. ## Testing From 18f8940ed6f231fe7c491891000da8b77293f4d8 Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Fri, 6 Nov 2026 02:12:35 -0500 Subject: [PATCH 16/21] Describe the CLI output by its fields, not an invented sample The sample JSON for `run bugs/01_off_by_one` came from no run: the earlier sample had been edited to add "sandbox": "docker" (Docker was never run), self_reported_pass, tests_modified and final_diff, next to the old elapsed time and pytest summary, which scoring now changes. Replace it with a table of the fields and what each means, plus the exit code, so the README states no values that were not measured. --- README.md | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 556b024..076eec8 100644 --- a/README.md +++ b/README.md @@ -88,25 +88,18 @@ code-fix-agent run bugs/01_off_by_one python -m benchmarks.run_benchmark --mode live ``` -Sample output (`run bugs/01_off_by_one`): - -```json -{ - "bug_id": "01_off_by_one", - "sandbox": "docker", - "passed": true, - "self_reported_pass": true, - "tests_modified": false, - "iterations": 3, - "tool_calls": 3, - "elapsed_seconds": 4.21, - "total_input_tokens": 2843, - "total_output_tokens": 142, - "final_summary": "PASS (exit 0) summary='4 passed in 0.03s'", - "error": null, - "final_diff": "--- a/buggy.py\n+++ b/buggy.py\n@@ -1,6 +1,6 @@\n ..." -} -``` +`run` prints one JSON object and exits 0 when the fix passes scoring, 1 otherwise. Its fields: + +| Field | Meaning | +|---|---| +| `bug_id`, `sandbox` | The bug folder's name and the runner the tests used (`docker` or `local`). | +| `passed` | The verdict: the original tests on a clean copy (see [Scoring](#isolation-and-scoring)). | +| `self_reported_pass`, `tests_modified` | The agent's own last `run_tests` said PASS; a test file differs from its setup SHA-256. | +| `iterations`, `tool_calls`, `elapsed_seconds` | Model rounds, tool calls and wall time, scoring included. | +| `total_input_tokens`, `total_output_tokens` | Token usage summed over the rounds. | +| `final_summary` | Status line of the scoring run, e.g. `PASS (exit 0) summary='4 passed in ...'`. | +| `error` | The model call's error, when one ended the loop; otherwise `null`. | +| `final_diff` | Unified diff of the workspace since setup. | ## How the loop works From 5bc444becc1379b0993c3578ea4900ba5f8f0105 Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Fri, 6 Nov 2026 02:14:54 -0500 Subject: [PATCH 17/21] Show files created during a run in the final diff Workspace.diff() walked only the files that existed at setup, so a conftest.py or module planted by code the tests ran never appeared in final_diff, though the audit trail is there to expose exactly that. It now also walks the workspace, subdirectories included, and diffs new files from /dev/null; a deleted file diffs to /dev/null instead of showing as emptied. --- README.md | 4 ++-- src/code_fix_agent/workspace.py | 31 ++++++++++++++++++++++--------- tests/test_agent.py | 13 +++++++++++++ 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 076eec8..e5cc261 100644 --- a/README.md +++ b/README.md @@ -160,10 +160,10 @@ Build the image once with `docker build -t code-fix-agent:latest .`. Every resul pytest --cov=code_fix_agent ``` -64 tests (the Docker one is skipped when the image is not built, the symlink one where creating symlinks needs admin rights): +65 tests (the Docker one is skipped when the image is not built, the symlink one where creating symlinks needs admin rights): - `test_workspace.py` — copy, list, read, edit (success + missing-substring error), run_tests (fails on unfixed, passes on canonical fix), no inherited secrets, timeout kills child processes, sandbox selection (live runs do not fall back to the local runner) and the Docker command line. - `test_tools.py` — schema completeness, dispatch, error surfacing, tracebacks in failing output, path confinement for every tool (`..`, absolute paths, symlinks). -- `test_agent.py` — full read-edit-test loop with `MockLLM` on bugs 1 and 2, max-iteration safety, LLM exception handling, token accounting, scoring against subset runs, edited tests, a planted `conftest.py` and source that skips, xfails or exits early; no bug id in the prompt, tool inputs and diff in the result, API timeout. +- `test_agent.py` — full read-edit-test loop with `MockLLM` on bugs 1 and 2, max-iteration safety, LLM exception handling, token accounting, scoring against subset runs, edited tests, a planted `conftest.py` and source that skips, xfails or exits early; no bug id in the prompt, tool inputs and the diff (created and deleted files included) in the result, API timeout. - `test_bench_smoke.py` — runs the full mock benchmark (12 bugs) and asserts 12/12 pass; the mock run exits non-zero on any failure. The bench smoke test is the canary: if it ever drops below 12/12 in CI, either the dataset or the workspace layer regressed. diff --git a/src/code_fix_agent/workspace.py b/src/code_fix_agent/workspace.py index 3b7a910..f18fa0b 100644 --- a/src/code_fix_agent/workspace.py +++ b/src/code_fix_agent/workspace.py @@ -128,19 +128,28 @@ def run_tests(self, test_file: str = "test_buggy.py", timeout: float = 30.0) -> return _as_result(run_pytest(self.root, [target], timeout=timeout, sandbox=self.sandbox)) def diff(self) -> str: - """Unified diff of every file changed since setup.""" + """Unified diff of every file changed, created or deleted since setup. + + Created files, in subdirectories too, diff from /dev/null: code the + tests ran can plant a conftest.py or a new module, and the record + should show it. + """ + current = { + path.relative_to(self.root).as_posix(): path.read_bytes() + for path in self.root.rglob("*") + if path.is_file() + } chunks: list[str] = [] - for name, data in self.originals.items(): - current = self.root / name - after = current.read_bytes() if current.is_file() else b"" - if after == data: + for name in sorted(self.originals.keys() | current.keys()): + before, after = self.originals.get(name), current.get(name) + if before == after: continue chunks.extend( difflib.unified_diff( - data.decode("utf-8", errors="replace").splitlines(keepends=True), - after.decode("utf-8", errors="replace").splitlines(keepends=True), - fromfile=f"a/{name}", - tofile=f"b/{name}", + _lines(before), + _lines(after), + fromfile=f"a/{name}" if before is not None else "/dev/null", + tofile=f"b/{name}" if after is not None else "/dev/null", ) ) return "".join(chunks) @@ -226,6 +235,10 @@ def _is_test_file(name: str) -> bool: ) +def _lines(data: bytes | None) -> list[str]: + return (data or b"").decode("utf-8", errors="replace").splitlines(keepends=True) + + def _sha256(path: Path) -> str | None: return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None diff --git a/tests/test_agent.py b/tests/test_agent.py index 7c3bcc7..6ca7de8 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -218,6 +218,19 @@ def test_result_records_tool_inputs_and_the_final_diff(self): assert "+ for i in range(n + 1):" in result.final_diff assert "test_buggy.py" not in result.final_diff + def test_diff_shows_created_and_deleted_files(self): + with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: + (ws.root / "conftest.py").write_bytes(b"import buggy\n") + (ws.root / "pkg").mkdir() + (ws.root / "pkg" / "helper.py").write_bytes(b"X = 1\n") + (ws.root / "test_buggy.py").unlink() + diff = ws.diff() + assert "--- /dev/null\n+++ b/conftest.py\n" in diff + assert "+import buggy\n" in diff + assert "--- /dev/null\n+++ b/pkg/helper.py\n" in diff + assert "--- a/test_buggy.py\n+++ /dev/null\n" in diff + assert "buggy.py\n+++ b/buggy.py" not in diff + def test_no_edit_means_an_empty_diff(self): with Workspace.from_bug_dir(_bug_dir("01_off_by_one")) as ws: result = run_agent(ws, MockLLM(script=[])) From 26254c830e11e6bbd87a234102b3cc4742d2f6cd Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Fri, 6 Nov 2026 02:16:48 -0500 Subject: [PATCH 18/21] Write mock benchmark results to their own file `run_benchmark --mode mock`, the first command in the README, wrote by default to benchmarks/results/benchmark.json and so replaced the committed record of the 2026-05-20 live run. Without --out, mock runs now write benchmark-mock.json (git-ignored) and live runs keep writing benchmark.json. --- .gitignore | 1 + README.md | 10 +++++----- benchmarks/run_benchmark.py | 10 +++++++++- tests/test_bench_smoke.py | 11 +++++++++++ 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 76ac80e..23d0730 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ venv/ *.swp benchmarks/results/*.tmp.json +benchmarks/results/benchmark-mock.json benchmarks/workspaces/ .DS_Store diff --git a/README.md b/README.md index e5cc261..d7e8366 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ ## Hero metrics Reproducible: -- `python -m benchmarks.run_benchmark --mode mock` (no API key, pipeline smoke) -- `python -m benchmarks.run_benchmark --mode live` (needs ANTHROPIC_API_KEY, real Haiku 4.5) +- `python -m benchmarks.run_benchmark --mode mock` (no API key, pipeline smoke; writes `benchmarks/results/benchmark-mock.json`, not committed) +- `python -m benchmarks.run_benchmark --mode live` (needs ANTHROPIC_API_KEY, real Haiku 4.5; writes `benchmarks/results/benchmark.json`) | Mode | Pass rate | Mean iterations / bug | Mean tool calls / bug | Total tokens | Realized cost (Haiku 4.5) | |---|---:|---:|---:|---:|---:| @@ -160,11 +160,11 @@ Build the image once with `docker build -t code-fix-agent:latest .`. Every resul pytest --cov=code_fix_agent ``` -65 tests (the Docker one is skipped when the image is not built, the symlink one where creating symlinks needs admin rights): +66 tests (the Docker one is skipped when the image is not built, the symlink one where creating symlinks needs admin rights): - `test_workspace.py` — copy, list, read, edit (success + missing-substring error), run_tests (fails on unfixed, passes on canonical fix), no inherited secrets, timeout kills child processes, sandbox selection (live runs do not fall back to the local runner) and the Docker command line. - `test_tools.py` — schema completeness, dispatch, error surfacing, tracebacks in failing output, path confinement for every tool (`..`, absolute paths, symlinks). - `test_agent.py` — full read-edit-test loop with `MockLLM` on bugs 1 and 2, max-iteration safety, LLM exception handling, token accounting, scoring against subset runs, edited tests, a planted `conftest.py` and source that skips, xfails or exits early; no bug id in the prompt, tool inputs and the diff (created and deleted files included) in the result, API timeout. -- `test_bench_smoke.py` — runs the full mock benchmark (12 bugs) and asserts 12/12 pass; the mock run exits non-zero on any failure. +- `test_bench_smoke.py` — runs the full mock benchmark (12 bugs) and asserts 12/12 pass; the mock run exits non-zero on any failure and does not overwrite the live record. The bench smoke test is the canary: if it ever drops below 12/12 in CI, either the dataset or the workspace layer regressed. @@ -197,7 +197,7 @@ src/code_fix_agent/ benchmarks/ run_benchmark.py # iterates all 12, captures stats results/ - benchmark.json + benchmark.json # the live run's record ``` ## What this is NOT diff --git a/benchmarks/run_benchmark.py b/benchmarks/run_benchmark.py index f22a7a4..ea13dba 100644 --- a/benchmarks/run_benchmark.py +++ b/benchmarks/run_benchmark.py @@ -153,9 +153,17 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--mode", choices=["mock", "live"], default="mock") parser.add_argument("--max-iterations", type=int, default=8) parser.add_argument("--model", default="claude-haiku-4-5-20251001") - parser.add_argument("--out", type=Path, default=RESULTS_DIR / "benchmark.json") + parser.add_argument( + "--out", + type=Path, + default=None, + help="default: results/benchmark.json for live runs, results/benchmark-mock.json for mock", + ) parser.add_argument("--sandbox", choices=SANDBOX_MODES, default="auto") args = parser.parse_args(argv) + if args.out is None: + # benchmark.json is the committed live record; a mock run must not replace it. + args.out = RESULTS_DIR / ("benchmark.json" if args.mode == "live" else "benchmark-mock.json") bugs = _load_manifest() # Live runs execute model-written code, so they only use the local runner diff --git a/tests/test_bench_smoke.py b/tests/test_bench_smoke.py index ef4aca1..8788ffc 100644 --- a/tests/test_bench_smoke.py +++ b/tests/test_bench_smoke.py @@ -48,3 +48,14 @@ def test_mock_run_fails_when_a_scripted_fix_stops_working(self, monkeypatch, tmp def test_mock_run_succeeds_when_every_fix_passes(self, monkeypatch, tmp_path): monkeypatch.setattr(run_benchmark, "_load_manifest", lambda: [{"id": "01_off_by_one"}]) assert run_benchmark.main(["--mode", "mock", "--out", str(tmp_path / "b.json")]) == 0 + + +class TestOutputPath: + def test_mock_run_does_not_replace_the_live_record(self, monkeypatch, tmp_path): + monkeypatch.setattr(run_benchmark, "RESULTS_DIR", tmp_path) + monkeypatch.setattr(run_benchmark, "_load_manifest", lambda: [{"id": "01_off_by_one"}]) + live_record = tmp_path / "benchmark.json" + live_record.write_text('{"summary": {"mode": "live"}}', encoding="utf-8") + assert run_benchmark.main(["--mode", "mock"]) == 0 + assert live_record.read_text(encoding="utf-8") == '{"summary": {"mode": "live"}}' + assert '"mode": "mock"' in (tmp_path / "benchmark-mock.json").read_text(encoding="utf-8") From de2c7597d0ff4b050f2e6defc1fd0f746349b5ce Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Fri, 6 Nov 2026 02:18:44 -0500 Subject: [PATCH 19/21] Make the timeout test independent of pytest start-up time The test ran the suite with a 6-second timeout, and pytest alone takes 3-4 seconds to start on Windows, so a slower runner could time out before the test spawned its child and fail with "the test never started its child process". The run now gets 20 seconds, and the child writes its "late" file at a fixed time 2 seconds after the deadline rather than 6 seconds after it started. Checked that it still fails when the timeout kills only pytest and not its process tree. --- tests/test_workspace.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_workspace.py b/tests/test_workspace.py index 26487f5..f9497da 100644 --- a/tests/test_workspace.py +++ b/tests/test_workspace.py @@ -83,10 +83,14 @@ def test_no_secrets(): def test_timeout_kills_processes_the_tests_started(self, tmp_path): started = tmp_path / "started" late = tmp_path / "late" + timeout = 20 + # Leave pytest plenty of time to start, and have the child write + # `late` at a fixed time after the deadline, however late it began. + late_at = time.time() + timeout + 2 child = ( "import pathlib, time; " f"pathlib.Path({str(started)!r}).write_text('x'); " - "time.sleep(6); " + f"time.sleep(max(0, {late_at} - time.time())); " f"pathlib.Path({str(late)!r}).write_text('x')" ) (tmp_path / "bug").mkdir() @@ -101,8 +105,8 @@ def test_hangs(): """, ) with Workspace.from_bug_dir(bug, sandbox="local") as ws: - result = ws.run_tests(timeout=6) - time.sleep(5) + result = ws.run_tests(timeout=timeout) + time.sleep(max(0, late_at + 2 - time.time())) assert result["summary"] == "timeout" assert started.exists(), "the test never started its child process" assert not late.exists(), "a process started by the tests outlived the timeout" From ea870f2dce5ff648b0ff553caf37d809066d4cd0 Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Fri, 6 Nov 2026 03:15:44 -0500 Subject: [PATCH 20/21] Drop the recruiter-facing section from the README "Why this matters for production" listed a "Recruiter signal" and named companies and job titles, as the section removed from claude-agent-kit did. Its one engineering point, how this compares with SWE-bench-Lite, is already made under "What this is NOT", and its claim that a SWE-bench-verified set drops in with no change to the agent loop was never tried, so remove the whole section. --- README.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/README.md b/README.md index d7e8366..8c5ea04 100644 --- a/README.md +++ b/README.md @@ -65,16 +65,6 @@ Each bug has 3-4 pytest assertions; the agent needs to satisfy all of them. | `src/code_fix_agent/llm.py` | `AnthropicLLM` + `MockLLM` (scripted tool-call sequence). Both return the same `(content, stop_reason, usage)` shape. | | `benchmarks/run_benchmark.py` | Iterates the dataset, captures pass / iteration / token / cost stats per bug, writes JSON. | -## Why this matters for production - -Recruiter signal this maps to: -- **SWE-agent / coding-agent JDs** (Mango Languages, Sekai, M3 USA, Cohere Agent Infrastructure) -- **Tool use + multi-turn loops** (any "Senior AI Engineer, Agentic Systems" role in 2026) -- **Real-pytest-driven evaluation** (not LLM-as-judge, not vibe-graded) -- **Bounded iteration budgets + token budgets** (the diff between a demo agent and a billable agent) - -This is the small-bench cousin of SWE-bench-Lite. It runs in seconds on a laptop and costs cents per full sweep. Drop SWE-bench-verified in next to scale; the agent loop does not change. - ## Quick start ```bash From f0b100360e704236c0491a89519a0d75c8863de8 Mon Sep 17 00:00:00 2001 From: Tajaddin Gafarov Date: Fri, 6 Nov 2026 03:16:00 -0500 Subject: [PATCH 21/21] Drop the stale pass-rate target from the package description The description still advertised "Hero: pass-rate target 9/12 (75%) at under 3 iterations per bug, under $0.05 total cost", the goal set before the live run. The 2026-05-20 run measured 12/12, 3.58 mean iterations and $0.10, on the harness that predates the current sandbox and scoring, so neither set of numbers belongs in a one-line summary; the README states the measured run with its caveats. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cb52305..c1c1ae8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "code-fix-agent" version = "0.1.0" -description = "Autonomous bug-fix agent on Claude Haiku 4.5. Curated 12-bug Python dataset. Hero: pass-rate target 9/12 (75%) at under 3 iterations per bug, under $0.05 total cost." +description = "Autonomous bug-fix agent on Claude Haiku 4.5. Curated 12-bug Python dataset." readme = "README.md" requires-python = ">=3.10" license = { text = "MIT" }