From 860b1d09ba6693be0536b4662777b9eb485715cf Mon Sep 17 00:00:00 2001 From: hanhainebula <2512674094@qq.com> Date: Thu, 19 Nov 2026 13:19:26 +0800 Subject: [PATCH 1/5] Recover incomplete scientific paper QA judge votes --- tasks/scientific-paper-qa/README.md | 4 +- .../authoring/tests/test_answer_repeats.py | 63 +++++++++++- tasks/scientific-paper-qa/tests/grader.py | 95 +++++++++++-------- 3 files changed, 120 insertions(+), 42 deletions(-) diff --git a/tasks/scientific-paper-qa/README.md b/tasks/scientific-paper-qa/README.md index 71f5ff9..e7bd91f 100644 --- a/tasks/scientific-paper-qa/README.md +++ b/tasks/scientific-paper-qa/README.md @@ -156,7 +156,9 @@ assets, credentials, or hardware. ### Repeated verifier measurements -After evidence matching, the fixed submitted answer is judged independently three times and the binary decisions are averaged arithmetically. Query values are then averaged equally. Retrieval, answer generation and evidence comparison run once. Each decision retains its own response/reason; transport or malformed-decision failures invalidate the measurement and leave zero reward, rather than averaging fewer decisions. Three valid decisions are distinct from the existing bounded transport retries. The answer-grader supervisor is 2,700 seconds: up to 30 judgments in five workers, each with three 120-second attempts and retry backoff, plus overhead. The overall 14,400-second verifier budget and single trajectory audit remain unchanged. +After evidence matching, the fixed submitted answer is judged independently three times. The score is the mean of the three binary decisions, averaged equally across queries. Retrieval, answer generation and evidence comparison run once. + +The `answer-judge-v2` protocol starts each vote at 512 output tokens. It accepts the first parseable decision, including a negative vote. Empty or malformed replies and transport errors retry within the same three 120-second attempts. If the provider reports a length limit and the decision is incomplete, the next attempt gets four times the output budget, up to 8,192 tokens. Each completed vote is saved in the verifier's private results directory. Restarting the grader there with the same inputs, judge and protocol reuses those votes. A missing required vote invalidates the measurement and leaves zero reward. The answer-grader supervisor allows 2,700 seconds for up to 30 judgments with five workers. The overall verifier budget is 14,400 seconds and includes one trajectory audit. ## References diff --git a/tasks/scientific-paper-qa/authoring/tests/test_answer_repeats.py b/tasks/scientific-paper-qa/authoring/tests/test_answer_repeats.py index f711023..c28bebb 100644 --- a/tasks/scientific-paper-qa/authoring/tests/test_answer_repeats.py +++ b/tasks/scientific-paper-qa/authoring/tests/test_answer_repeats.py @@ -1,12 +1,15 @@ """Offline evidence-gated repeated answer judging and failure propagation.""" +import io +import json import sys +import tempfile +import urllib.error from pathlib import Path TASK_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(TASK_ROOT / "tests")) -import json import unittest from unittest.mock import patch @@ -14,14 +17,17 @@ class AnswerRepeatTests(unittest.TestCase): - def run_grade(self, responses, evidence="doc"): + def run_grade(self, responses, evidence="doc", *, directory=None, answer="fixed answer", model="model"): + if directory is None: + directory = self.enterContext(tempfile.TemporaryDirectory()) queries = [{"query_id": "q", "question": "question"}] gold = {"q": {"evidence": "doc", "answer": "reference"}} - prediction = {"q": {"evidence": evidence, "answer": "fixed answer"}} + prediction = {"q": {"evidence": evidence, "answer": answer}} with patch.object(grader, "load_reference_data", return_value=(queries, gold, {"doc"})), \ patch.object(grader, "load_predictions", return_value=prediction), \ - patch.object(grader, "judge_config", return_value=grader.JudgeConfig("base", "key", "model")), \ + patch.object(grader, "judge_config", return_value=grader.JudgeConfig("base", "key", model)), \ patch.object(grader, "call_judge", side_effect=responses) as judge, \ + patch.object(grader, "RESULTS_DIR", Path(directory)), \ patch.object(grader, "write_report") as write, \ patch.object(grader, "EXECUTION_ERROR", ""): status = grader.main() @@ -53,6 +59,55 @@ def test_partial_or_malformed_judging_invalidates_measurement(self): self.assertEqual(report["answer_judge_call_count"], 2) self.assertTrue(report["errors"]) + def test_restart_preserves_both_negative_and_positive_votes(self): + with tempfile.TemporaryDirectory() as directory: + status, _, _ = self.run_grade(['{"correct":false}', '{"correct":true}', OSError("offline")], + directory=directory) + self.assertEqual(status, 1) + status, report, judge = self.run_grade(['{"correct":true}'], directory=directory) + self.assertEqual(status, 0) + judge.assert_called_once() + samples = report["per_query_metrics"][0]["judge_samples"] + self.assertEqual([vote["from_checkpoint"] for vote in samples], [True, True, False]) + self.assertAlmostEqual(report["primary_metric"]["value"], 2 / 3) + for changes in ({"answer": "different"}, {"model": "different"}): + with self.subTest(changes=changes): + _, _, judge = self.run_grade(['{"correct":false}'] * 3, directory=directory, **changes) + self.assertEqual(judge.call_count, 3) + + def judge_requests(self, replies, *, fails=False): + def reply(item): + if isinstance(item, Exception): + return item + content, finish = item + body = {"choices": [{"message": {"content": content}, "finish_reason": finish}]} + return io.BytesIO(json.dumps(body).encode()) + + with patch.object(grader.urllib.request, "urlopen", side_effect=[reply(item) for item in replies]) as post, \ + patch.object(grader.time, "sleep"): + args = dict(question="q", candidate_answer="a", reference_answer="g") + config = grader.JudgeConfig("https://example.invalid", "test", "model") + if fails: + with self.assertRaises(ValueError): + grader.call_judge(config, **args) + else: + result = grader.call_judge(config, **args) + self.assertEqual(json.loads(result), {"correct": False}) + return [json.loads(call.args[0].data)["max_tokens"] for call in post.call_args_list] + + def test_only_an_unparseable_length_limited_vote_raises_the_budget(self): + valid = '{"correct":false}' + self.assertEqual(self.judge_requests([(None, "length"), ('{"correct":', "length"), (valid, "stop")]), + [512, 2048, 8192]) + self.assertEqual(self.judge_requests([(valid, "length")]), [512]) + + def test_transport_empty_and_malformed_replies_retry_within_one_vote(self): + valid = ('{"correct":false}', "stop") + self.assertEqual(self.judge_requests([urllib.error.URLError("offline"), ("", "stop"), valid]), + [512, 512, 512]) + self.assertEqual(self.judge_requests([("invalid", "stop"), valid]), [512, 512]) + self.assertEqual(self.judge_requests([("invalid", "stop")] * 3, fails=True), [512, 512, 512]) + if __name__ == "__main__": unittest.main() diff --git a/tasks/scientific-paper-qa/tests/grader.py b/tasks/scientific-paper-qa/tests/grader.py index 9ba1248..568a1aa 100644 --- a/tasks/scientific-paper-qa/tests/grader.py +++ b/tasks/scientific-paper-qa/tests/grader.py @@ -3,11 +3,13 @@ from __future__ import annotations +import fcntl +import hashlib +import http.client import json import os import re import time -import urllib.error import urllib.request from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass @@ -25,6 +27,7 @@ REPORT_PATH = RESULTS_DIR / "evaluation.json" EXECUTION_ERROR = os.environ.get("SEARCH_SWE_EXECUTION_ERROR", "") MAX_ANSWER_CHARS = 20_000 +ANSWER_JUDGE_PROTOCOL = "answer-judge-v2" SYSTEM_PROMPT = """ You are a strict answer-equivalence evaluator. Determine only whether the @@ -243,23 +246,21 @@ def call_judge( }, ensure_ascii=False, ) - body = json.dumps( - { - "model": config.model, - "messages": [ - {"role": "system", "content": SYSTEM_PROMPT}, - {"role": "user", "content": user_payload}, - ], - "temperature": 0, - "max_tokens": 512, - } - ).encode("utf-8") + body = { + "model": config.model, + "messages": [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": user_payload}, + ], + "temperature": 0, + "max_tokens": 512, + } last_error: Exception | None = None for attempt in range(retries): request = urllib.request.Request( endpoint, - data=body, + data=json.dumps(body).encode("utf-8"), headers={ "Authorization": f"Bearer {config.api_key}", "Content-Type": "application/json", @@ -269,21 +270,22 @@ def call_judge( try: with urllib.request.urlopen(request, timeout=120) as response: payload = json.loads(response.read().decode("utf-8")) - choices = payload.get("choices") - if not isinstance(choices, list) or not choices: + choices = payload.get("choices") if isinstance(payload, dict) else None + if not isinstance(choices, list) or not choices or not isinstance(choices[0], dict): raise ValueError("judge response contains no choices") - message = choices[0].get("message") + choice = choices[0] + message = choice.get("message") content = message.get("content") if isinstance(message, dict) else None + if isinstance(content, str) and content.strip() and not parse_judgement(content)[1]: + return content + if choice.get("finish_reason") == "length": + previous_limit = body["max_tokens"] + body["max_tokens"] = min(8192, previous_limit * 4) + raise ValueError(f"judge decision truncated at {previous_limit} tokens") if not isinstance(content, str) or not content.strip(): raise ValueError("judge response contains no text") - return content - except ( - OSError, - ValueError, - json.JSONDecodeError, - urllib.error.HTTPError, - urllib.error.URLError, - ) as error: + raise ValueError("answer judge returned a malformed decision") + except (OSError, ValueError, http.client.HTTPException) as error: last_error = error if attempt + 1 < retries: time.sleep(2**attempt) @@ -350,19 +352,37 @@ def judge_one(row: dict[str, str]) -> tuple[str, dict[str, Any]]: calls = 0 try: assert config is not None - for _ in range(3): - calls += 1 - response = call_judge( - config, - question=row["question"], - candidate_answer=predictions[query_id]["answer"], - reference_answer=references[query_id]["answer"], - ) - correct, parse_error, reason = parse_judgement(response) - samples.append({"correct": correct, "parse_error": parse_error, "reason": reason, - "judge_response": response[:5000]}) - if parse_error: - raise ValueError("answer judge returned a malformed decision") + # This directory belongs to the verifier and to this trial. Bind each + # checkpoint to the exact judgment, including the submission and policy. + context = [ANSWER_JUDGE_PROTOCOL, config.api_base, config.model, SYSTEM_PROMPT, + row, references[query_id], predictions[query_id]] + digest = hashlib.sha256(json.dumps(context, sort_keys=True).encode()).hexdigest() + directory = RESULTS_DIR / ANSWER_JUDGE_PROTOCOL / digest + directory.mkdir(mode=0o700, parents=True, exist_ok=True) + with (directory / ".lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + for vote in range(1, 4): + checkpoint = directory / f"{vote}.txt" + cached = checkpoint.exists() + if cached: + response = checkpoint.read_text(encoding="utf-8") + else: + calls += 1 + response = call_judge( + config, + question=row["question"], + candidate_answer=predictions[query_id]["answer"], + reference_answer=references[query_id]["answer"], + ) + correct, parse_error, reason = parse_judgement(response) + if parse_error: + raise ValueError("answer judge returned a malformed decision") + if not cached: + temporary = checkpoint.with_suffix(".tmp") + temporary.write_text(response, encoding="utf-8") + temporary.replace(checkpoint) + samples.append({"correct": correct, "parse_error": False, "reason": reason, + "judge_response": response[:5000], "from_checkpoint": cached}) return query_id, { "answer_correct": sum(sample["correct"] for sample in samples) / len(samples), "parse_error": False, @@ -436,6 +456,7 @@ def judge_one(row: dict[str, str]) -> tuple[str, dict[str, Any]]: "evidence_match_count": evidence_count, "answer_judge_call_count": sum(item["judge_call_count"] for item in per_query), "answer_judge_repetitions": 3, + "answer_judge_protocol": ANSWER_JUDGE_PROTOCOL, "judge_error_count": judge_error_count, "judge_model": config.model if config else None, "per_query_metrics": per_query, From 1974a795c46be92b330c4612d0a33066e06b6d01 Mon Sep 17 00:00:00 2001 From: hanhainebula <2512674094@qq.com> Date: Thu, 19 Nov 2026 13:58:48 +0800 Subject: [PATCH 2/5] Add Antigravity OpenRouter support in CLI 0.3.1 --- .env.example | 4 +- docs/evaluation/agents.md | 17 ++ docs/evaluation/cli.md | 10 +- docs/evaluation/quickstart.md | 2 +- docs/evaluation/runtime.md | 5 +- pyproject.toml | 2 +- scripts/evaluation/agy_openrouter.py | 276 +++++++++++++++++++++++ scripts/evaluation/native_agents.py | 35 ++- scripts/evaluation/run_task.py | 32 ++- skills/run-searchswe-evaluation/SKILL.md | 8 +- src/search_swe/__init__.py | 2 +- src/search_swe/cli.py | 3 + src/search_swe/configuration.py | 11 +- src/search_swe/harbor_native.py | 2 + src/search_swe/model_catalog.py | 2 +- src/search_swe/wizard.py | 4 +- tests/unit/test_agy_openrouter.py | 72 ++++++ tests/unit/test_cli.py | 9 + tests/unit/test_harbor_native.py | 47 +++- 19 files changed, 511 insertions(+), 32 deletions(-) create mode 100644 scripts/evaluation/agy_openrouter.py create mode 100644 tests/unit/test_agy_openrouter.py diff --git a/.env.example b/.env.example index 85b04f4..5ccc286 100644 --- a/.env.example +++ b/.env.example @@ -39,8 +39,8 @@ AGENT_OPENAI_API_KEY= # Claude Code direct mode and Pi's anthropic provider: official Anthropic API key. AGENT_ANTHROPIC_API_KEY= -# OpenRouter Agent mode (--provider openrouter) for Codex, Claude Code or Pi. Separate from -# OPENROUTER_API_KEY used by task submissions and ANSWER_JUDGE_API_KEY. +# OpenRouter coding-agent key for Codex, Claude Code, Pi or Antigravity CLI. +# Task submissions use OPENROUTER_API_KEY; answer judging uses ANSWER_JUDGE_API_KEY. AGENT_OPENROUTER_API_KEY= # Optional Codex or Claude Code reasoning effort; default omits the override. diff --git a/docs/evaluation/agents.md b/docs/evaluation/agents.md index 735239c..097f1d1 100644 --- a/docs/evaluation/agents.md +++ b/docs/evaluation/agents.md @@ -14,6 +14,7 @@ The default is Codex + OpenRouter + `openai/gpt-6-luna` (`max` reasoning). | Claude Code | `direct` | Claude model at the official Anthropic API | `AGENT_ANTHROPIC_API_KEY` | | Pi | `openrouter` | OpenRouter text/tool models; full OpenRouter ID | `AGENT_OPENROUTER_API_KEY` | | Pi | `direct` | `provider/model` from Pi's bundled API-key catalog | Selected provider's key and endpoint settings | +| Antigravity CLI | `openrouter` | OpenRouter text/tool models using Chat Completions | `AGENT_OPENROUTER_API_KEY` | | Any registered Harbor agent | `native` | Native model ID, or the agent's default | Selected agent's `AGENT_*` variables and optional native configuration | ## Harness versions and installation @@ -142,6 +143,22 @@ routes share the Search-SWE event-stream-to-ATIF converter. Inspect the chosen agent's official setup when it needs project files, a subscription or cloud credentials. +## Antigravity CLI through OpenRouter + +Use CLI 0.3.1 or later in the 0.3.x series with a checkout containing this +integration. Set `AGENT_OPENROUTER_API_KEY` in the checkout's private `.env`. + +```bash +search-swe run TASK --agent antigravity-cli --provider openrouter \ + --model google/gemini-4-argon --reasoning-effort high +``` + +The route uses a loopback bridge in the agent container to translate agy's +Gemini requests to OpenRouter Chat Completions. Use the full OpenRouter model +ID. Effort accepts `default` (agy's model default), `low`, or `high`. +The bridge transfers tool calls, results, and reasoning details across turns. +OpenRouter is the coding-model endpoint for this route. + ## Claude Code and Codex subscriptions Both use `--provider native` with explicit credentials in `--agent-config`. diff --git a/docs/evaluation/cli.md b/docs/evaluation/cli.md index 9070adc..9429fdc 100644 --- a/docs/evaluation/cli.md +++ b/docs/evaluation/cli.md @@ -78,9 +78,9 @@ keys to configure; `run` selects the tasks to evaluate. Use each command's `--help` for its flags. The default is **Codex + OpenRouter + `openai/gpt-6-luna`, `max` reasoning**. -Codex, Claude Code and Pi provide integrated OpenRouter routes. All registered -Harbor agents are available through `--provider native`; use `search-swe agents` -to discover them. See the +Codex, Claude Code, Pi and Antigravity CLI provide integrated OpenRouter routes. +Use `search-swe agents` to discover Harbor agents available through +`--provider native`. See the [agent/provider matrix](https://github.com/VectorSpaceLab/Search-SWE/blob/main/docs/evaluation/agents.md) for direct routes, model scope and [harness versions/cache](https://github.com/VectorSpaceLab/Search-SWE/blob/main/docs/evaluation/agents.md#harness-versions-and-installation). `search-swe pi-models` lists Pi providers; add `--provider NAME` for exact model IDs. @@ -90,8 +90,8 @@ add `--provider NAME` for exact model IDs. Search by name or model ID, highlight with ↓/↑ and select with Enter. An empty field keeps the displayed model; full IDs can also be entered manually. Suggestions use text, tool and API metadata: Responses for Codex, Messages for -Claude Code, and Chat Completions for Pi. Model selection makes no inference -request; actual compatibility depends on the model/provider. +Claude Code, and Chat Completions for Pi and Antigravity CLI. Model selection +makes no inference requests. Runtime compatibility depends on the model and provider. The public catalog is cached for 24 hours at `$XDG_CACHE_HOME/search-swe/openrouter-models.json` (default diff --git a/docs/evaluation/quickstart.md b/docs/evaluation/quickstart.md index 1d412d8..4af9a3f 100644 --- a/docs/evaluation/quickstart.md +++ b/docs/evaluation/quickstart.md @@ -105,7 +105,7 @@ URL for a branch or tag. Match the skill to your CLI and checkout. ### Update the skill The skill records its matching CLI release in `SKILL.md` as `metadata.version` -(currently `0.3.0`), with compatible CLI/checkout series `0.3.x`. +(currently `0.3.1`), with compatible CLI/checkout series `0.3.x`. For a project installation, run these commands in your coordinating agent's workspace: diff --git a/docs/evaluation/runtime.md b/docs/evaluation/runtime.md index 3a84fe0..3071992 100644 --- a/docs/evaluation/runtime.md +++ b/docs/evaluation/runtime.md @@ -58,13 +58,14 @@ The single-task backend uses `--openrouter` for OpenRouter; the CLI uses | `--agent codex --openrouter` | Full OpenRouter ID | `AGENT_OPENROUTER_API_KEY` | | `--agent claude-code --openrouter` | Full OpenRouter ID with Messages/tool support | `AGENT_OPENROUTER_API_KEY` | | `--agent pi --openrouter` | Full OpenRouter ID with Chat Completions/tool support | `AGENT_OPENROUTER_API_KEY` | +| `--agent antigravity-cli --openrouter` | Full OpenRouter ID with Chat Completions/tool support | `AGENT_OPENROUTER_API_KEY` | | `--agent codex` | Direct endpoint's model ID | `AGENT_OPENAI_BASE_URL`, `AGENT_OPENAI_API_KEY` | | `--agent claude-code` | Official Anthropic model ID | `AGENT_ANTHROPIC_API_KEY` | | `--agent pi` | Pi `provider/model` ID | Selected provider's key and endpoint settings | | `--agent NAME --native` | Native Harbor model ID or agent default | Agent-only `AGENT_*` variables and optional `--agent-config PATH` | -Use `search-swe agents` to discover Harbor's registered agents. Agents beyond the -three integrated routes select native mode automatically in this backend. +Use `search-swe agents` to discover Harbor's registered agents. Antigravity CLI +selects its native route unless `--openrouter` is passed. `--agent-version VERSION` or `SEARCH_SWE_AGENT_VERSION` overrides the central [harness default](agents.md#harness-versions-and-installation); `default` resets it. The five centrally managed CLIs are installed and version-verified during setup, diff --git a/pyproject.toml b/pyproject.toml index efa6a9a..bfe246b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "search-swe" -version = "0.3.0" +version = "0.3.1" description = "Configure and run Search-SWE coding-agent evaluations" readme = "docs/evaluation/cli.md" requires-python = ">=3.12" diff --git a/scripts/evaluation/agy_openrouter.py b/scripts/evaluation/agy_openrouter.py new file mode 100644 index 0000000..af88c82 --- /dev/null +++ b/scripts/evaluation/agy_openrouter.py @@ -0,0 +1,276 @@ +"""Run agy with a loopback Gemini API backed by one OpenRouter chat model. + +The bridge is uploaded to the agent container and uses only its Python standard +library. It follows the container's existing egress policy. +""" + +import base64 +from collections import defaultdict, deque +from http import HTTPStatus +from http.client import HTTPException +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +import re +import secrets +import signal +import subprocess +import sys +from threading import Thread +from urllib.error import HTTPError +from urllib.parse import parse_qs, urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + + +OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" +MAX_REQUEST_BYTES = 64 * 1024 * 1024 +FINISH_REASONS = { + "stop": "STOP", "tool_calls": "STOP", "length": "MAX_TOKENS", + "content_filter": "SAFETY", +} + + +def encode(value): + return json.dumps(value, separators=(",", ":"), ensure_ascii=False) + + +def content_part(part): + if "text" in part: + return {"type": "text", "text": part["text"]} + if data := part.get("inlineData"): + mime = data.get("mimeType", "") + if mime.startswith("image/"): + return {"type": "image_url", "image_url": {"url": f"data:{mime};base64,{data['data']}"}} + raise ValueError("Unsupported Gemini content part") + + +def chat_messages(request): + messages = [] + if system := request.get("systemInstruction"): + messages.append({"role": "system", "content": [content_part(p) for p in system["parts"]]}) + pending_calls = defaultdict(deque) + for turn in request["contents"]: + parts = turn["parts"] + # agy sends tool results as model-role turns on its built-in path and + # may also use the Gemini API's usual user-role tool result format. + tool_turn = any("functionResponse" in part for part in parts) + if turn["role"] == "model" and not tool_turn: + assistant = {"role": "assistant", "content": None} + text, calls, reasoning = [], [], [] + pending_calls.clear() + for part in parts: + if signature := part.get("thoughtSignature"): + reasoning.extend(json.loads(base64.b64decode(signature, validate=True))) + if call := part.get("functionCall"): + name = call["name"] + call_id = call.get("id") or f"call_{len(messages)}_{len(calls)}" + pending_calls[name].append(call_id) + calls.append({"id": call_id, "type": "function", + "function": {"name": name, "arguments": encode(call.get("args", {}))}}) + elif "text" in part and not part.get("thought"): + text.append(part["text"]) + assistant["content"] = "".join(text) or None + if calls: + assistant["tool_calls"] = calls + if reasoning: + assistant["reasoning_details"] = reasoning + messages.append(assistant) + elif turn["role"] == "user" or tool_turn: + # Some agy tools include media alongside their function responses. + # OpenRouter expects the tool messages before the user media message. + user_parts = [] + for part in parts: + if result := part.get("functionResponse"): + queue = pending_calls[result["name"]] + call_id = result.get("id") or (queue[0] if queue else None) + if call_id not in queue: + raise ValueError("Gemini tool result has no matching call") + queue.remove(call_id) + messages.append({"role": "tool", "tool_call_id": call_id, + "content": encode(result["response"])}) + user_parts.extend(content_part(media) for media in result.get("parts", [])) + else: + user_parts.append(content_part(part)) + if user_parts: + messages.append({"role": "user", "content": user_parts}) + else: + raise ValueError("Unsupported Gemini message role") + return messages + + +def openrouter_request(request, model): + body = {"model": model, "messages": chat_messages(request)} + tools = [] + for group in request.get("tools", []): + if set(group) != {"functionDeclarations"}: + raise ValueError("Only Gemini function declarations are supported") + for function in group["functionDeclarations"]: + declaration = {k: function[k] for k in ("name", "description") if k in function} + declaration["parameters"] = function.get("parametersJsonSchema", function.get("parameters", {})) + tools.append({"type": "function", "function": declaration}) + if tools: + body["tools"] = tools + config = request.get("toolConfig", {}).get("functionCallingConfig", {}) + mode = config.get("mode", "AUTO") + if allowed := config.get("allowedFunctionNames"): + body["tools"] = [tool for tool in tools if tool["function"]["name"] in allowed] + if not body["tools"]: + raise ValueError("No matching tool declarations") + body["tool_choice"] = {"AUTO": "auto", "ANY": "required", "NONE": "none"}[mode] + + config = request.get("generationConfig", {}) + for src, dst in (("maxOutputTokens", "max_tokens"), ("temperature", "temperature"), + ("topP", "top_p"), ("topK", "top_k"), ("stopSequences", "stop"), + ("seed", "seed")): + if src in config: + body[dst] = config[src] + if config.get("responseMimeType") == "application/json": + if schema := config.get("responseJsonSchema", config.get("responseSchema")): + body["response_format"] = {"type": "json_schema", + "json_schema": {"name": "response", "schema": schema}} + else: + body["response_format"] = {"type": "json_object"} + if effort := config.get("thinkingConfig", {}).get("thinkingLevel"): + body["reasoning"] = {"effort": effort.lower()} + return body + + +def gemini_response(response): + choice = response["choices"][0] + message = choice["message"] + parts = [] + if thought := message.get("reasoning"): + parts.append({"text": thought, "thought": True}) + if content := message.get("content"): + parts.append({"text": content}) + calls = [] + for tool in message.get("tool_calls") or []: + function = tool["function"] + args = json.loads(function["arguments"]) + if not isinstance(args, dict): + raise ValueError("OpenRouter tool arguments must be an object") + calls.append({"functionCall": {"name": function["name"], "args": args, "id": tool["id"]}}) + if details := message.get("reasoning_details"): + # agy round-trips this opaque field on the matching assistant turn. + # Preserve OpenRouter's original IDs, ordering and encrypted data. + signed = calls[0] if calls else parts[-1] if parts else {"text": ""} + signed["thoughtSignature"] = base64.b64encode(encode(details).encode()).decode() + if not (calls or parts): + parts.append(signed) + result = { + "candidates": [{"index": 0, "content": {"role": "model", "parts": parts + calls}, + "finishReason": FINISH_REASONS[choice["finish_reason"]]}], + "modelVersion": response["model"], + } + if usage := response.get("usage"): + thoughts = (usage.get("completion_tokens_details") or {}).get("reasoning_tokens") or 0 + result["usageMetadata"] = { + "promptTokenCount": usage["prompt_tokens"], + "candidatesTokenCount": max(0, usage["completion_tokens"] - thoughts), + "thoughtsTokenCount": thoughts, + "totalTokenCount": usage["total_tokens"], + "cachedContentTokenCount": (usage.get("prompt_tokens_details") or {}).get("cached_tokens") or 0, + } + return result + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None + + +def call_openrouter(body, api_key): + request = Request(OPENROUTER_URL, data=encode(body).encode(), headers={ + "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", + }) + with build_opener(NoRedirect()).open(request, timeout=600) as response: + return json.load(response) + + +class GeminiHandler(BaseHTTPRequestHandler): + def log_message(self, *_args): + # agy can carry the loopback token in its URL. Do not log it or prompts. + pass + + def reply(self, status, payload, *, stream=False): + data = encode(payload).encode() + if stream: + data = b"data: " + data + b"\n\n" + self.send_response(status) + self.send_header("Content-Type", "text/event-stream" if stream else "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def error(self, status, message): + self.reply(status, {"error": {"code": status, "message": message, + "status": HTTPStatus(status).name}}) + + def do_POST(self): + path = urlsplit(self.path) + token = self.headers.get("x-goog-api-key") or parse_qs(path.query).get("key", [""])[0] + if not secrets.compare_digest(token.encode(), self.server.loopback_key.encode()): + self.error(401, "Invalid bridge credential") + return + match = re.fullmatch(r"/v1(?:beta)?/models/([^/]+):(generateContent|streamGenerateContent)", path.path) + if not match or match[1] != self.server.model.split("/")[-1]: + self.error(404, "Gemini endpoint or model is unavailable on this bridge") + return + try: + length = int(self.headers.get("Content-Length", "0")) + if not 0 < length <= MAX_REQUEST_BYTES: + raise ValueError("Invalid request length") + body = openrouter_request(json.loads(self.rfile.read(length)), self.server.model) + except (KeyError, TypeError, ValueError) as error: + # Only our field names and static error messages go into agy logs. + self.error(400, f"Unsupported Gemini request ({type(error).__name__})") + return + try: + result = gemini_response(call_openrouter(body, self.server.api_key)) + except HTTPError as error: + error.close() + status = error.code if error.code in {400, 401, 402, 403, 404, 413, 422, 429, 503, 504} else 502 + detail = f"OpenRouter returned HTTP {error.code}" + except (HTTPException, IndexError, KeyError, TypeError, ValueError, OSError) as error: + status = 502 + detail = f"OpenRouter request or response failed ({type(error).__name__})" + else: + self.reply(200, result, stream=match[2] == "streamGenerateContent") + return + self.error(status, detail) + + +def run_agy(model, command, api_key): + with ThreadingHTTPServer(("127.0.0.1", 0), GeminiHandler) as server: + server.model = model + server.api_key = api_key + server.loopback_key = secrets.token_urlsafe(24) + worker = Thread(target=server.serve_forever, daemon=True) + worker.start() + env = dict(os.environ) + env.pop("AGENT_OPENROUTER_API_KEY", None) + env["GEMINI_API_KEY"] = server.loopback_key + env["GOOGLE_GEMINI_BASE_URL"] = f"http://127.0.0.1:{server.server_port}" + bypass = env.get("no_proxy") or env.get("NO_PROXY") or "" + env["NO_PROXY"] = env["no_proxy"] = ",".join(filter(None, (bypass, "127.0.0.1", "localhost"))) + try: + with subprocess.Popen(command, env=env) as child: + # Container teardown also closes in-flight HTTP connections. + def stop(_signum, _frame): + child.terminate() + + previous = signal.signal(signal.SIGTERM, stop) + try: + status = child.wait() + return status if status >= 0 else 128 - status + finally: + signal.signal(signal.SIGTERM, previous) + finally: + server.shutdown() + worker.join() + + +if __name__ == "__main__": + if not (api_key := os.environ.get("AGENT_OPENROUTER_API_KEY")): + sys.exit("Antigravity OpenRouter requires AGENT_OPENROUTER_API_KEY") + sys.exit(run_agy(sys.argv[1], sys.argv[2:], api_key)) diff --git a/scripts/evaluation/native_agents.py b/scripts/evaluation/native_agents.py index 8016deb..4f47bcd 100644 --- a/scripts/evaluation/native_agents.py +++ b/scripts/evaluation/native_agents.py @@ -7,6 +7,7 @@ import re import shlex import sys +from uuid import uuid4 sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) @@ -84,11 +85,36 @@ async def run(self, *args, **kwargs): return await super().run(*args, **kwargs) +class AntigravityOpenRouter: + """Run Harbor's agy commands against the per-process Gemini bridge.""" + + def _find_api_key(self): + # Enable Harbor's headless Gemini configuration. The bridge generates + # the actual loopback key and hands it to the agy child process. + return "search-swe-loopback" + + async def run(self, instruction, environment, context): + if not self._get_env("AGENT_OPENROUTER_API_KEY"): + raise ValueError("Antigravity OpenRouter requires AGENT_OPENROUTER_API_KEY") + self._bridge_path = f"/tmp/search-swe-agy-{uuid4().hex}.py" + await environment.upload_file(Path(__file__).with_name("agy_openrouter.py"), self._bridge_path) + return await super().run(instruction, environment, context) + + async def exec_as_agent(self, environment, command, env=None, **kwargs): + if (env or {}).get("GEMINI_API_KEY") == "search-swe-loopback": + if not command.startswith("$HOME/.local/bin/agy "): + raise RuntimeError("Harbor's Antigravity command format changed") + env = {**env, "AGENT_OPENROUTER_API_KEY": self._get_env("AGENT_OPENROUTER_API_KEY")} + command = (f"/opt/conda/bin/python -I {shlex.quote(self._bridge_path)} " + f"{shlex.quote(self.model_name)} " + command) + return await super().exec_as_agent(environment, command, env=env, **kwargs) + + class NativeAgent: """Factory returning the actual Harbor agent with its native capabilities.""" def __new__(cls, *args, search_swe_native_name, search_swe_setup_hosts=(), - search_swe_files=(), **kwargs): + search_swe_files=(), search_swe_openrouter=False, **kwargs): from search_swe.harbor_native import agent_class, registry from search_swe import harnesses selected_version = harnesses.resolve_version(search_swe_native_name, kwargs.get("version")) @@ -100,7 +126,12 @@ def __new__(cls, *args, search_swe_native_name, search_swe_setup_hosts=(), from harbor.agents.installed.acp_registry import registry_spec_from_agent_name kwargs.setdefault("registry_spec", registry_spec_from_agent_name(search_swe_native_name)) converter = {"pi": PiTrajectory, "antigravity-cli": AntigravityTrajectory}.get(search_swe_native_name) - bases = (FinalizeBoundedLogs, NativeLifecycle, *((converter,) if converter else ()), base) + mixins = (converter,) if converter else () + if search_swe_openrouter: + if search_swe_native_name != "antigravity-cli": + raise ValueError("The Gemini bridge is only available for Antigravity CLI") + mixins += (AntigravityOpenRouter,) + bases = (FinalizeBoundedLogs, NativeLifecycle, *mixins, base) adapted = type("SearchSwe" + base.__name__, bases, {"__module__": base.__module__}) with agent_environment(kwargs.get("extra_env") or {}): instance = adapted(*args, **kwargs) diff --git a/scripts/evaluation/run_task.py b/scripts/evaluation/run_task.py index cd0731f..1fd9b07 100755 --- a/scripts/evaluation/run_task.py +++ b/scripts/evaluation/run_task.py @@ -20,6 +20,7 @@ sys.path.insert(0, str(REPO)) sys.path.insert(0, str(REPO / "src")) from search_swe import harbor_native, harnesses, pi_models +from search_swe.configuration import AGY_OPENROUTER_EFFORTS from scripts.evaluation.docker_dns import parse_servers from scripts.task_paths import select_task, task_key from scripts.assets.download_assets import destination_path, read_manifest, relative_path @@ -145,7 +146,7 @@ def main(): env_options.add_argument("--no-env-file", action="store_true", help="Use only process environment configuration") parser.add_argument( "--reasoning-effort", - help="Override AGENT_REASONING_EFFORT for Codex or Claude Code", + help="Override the selected harness's reasoning effort", ) parser.add_argument("--codex-config", type=Path, help="Optional native Codex TOML configuration") parser.add_argument("--thinking", choices=("default", *PI_THINKING_LEVELS), help="Pi thinking level; default omits the override") @@ -160,10 +161,11 @@ def main(): parser.add_argument("--dry-run", action="store_true", help="Print the command with variable references; do not launch") parser.add_argument("--check", action="store_true", help="Validate model, credentials and configuration without restoring assets or launching") args = parser.parse_args() - if args.agent not in AGENT_IMPORTS: - args.native = True if args.native and args.openrouter: parser.error("Use --native with Harbor's own model/provider configuration, or --openrouter with the integrated route") + agy_openrouter = args.agent == "antigravity-cli" and args.openrouter + if args.agent not in AGENT_IMPORTS and not agy_openrouter: + args.native = True if args.agent_config and not args.native: parser.error("--agent-config requires --native") if args.native and any(value is not None for value in (args.thinking, args.pi_model_config, args.codex_config)): @@ -202,12 +204,14 @@ def main(): if not model and not args.native: parser.error("Set --model or AGENT_MODEL") if args.openrouter: - if args.agent not in OPENROUTER_AGENT_IMPORTS: + if args.agent not in OPENROUTER_AGENT_IMPORTS and not agy_openrouter: parser.error("Unsupported OpenRouter harness") if not OPENROUTER_MODEL.fullmatch(model): parser.error("OpenRouter requires a complete provider/model slug in --model or AGENT_MODEL") if args.agent == "codex" and args.codex_config is not None: parser.error("--openrouter uses its own Codex config; omit --codex-config") + if agy_openrouter and args.codex_config is not None: + parser.error("--codex-config is only valid with --agent codex") if args.native: pass elif args.agent == "pi": @@ -241,7 +245,7 @@ def main(): parser.error(str(error)) task_config = tomllib.loads((task / "task.toml").read_text()) try: - harbor_native.check_trajectory_support(args.agent, args.native, task_config) + harbor_native.check_trajectory_support(args.agent, args.native or agy_openrouter, task_config) storage = StoragePlan.discover(task, task_config) except ValueError as error: parser.error(str(error)) @@ -265,7 +269,7 @@ def main(): command = [ "harbor", "run", "--path", str(task), "--env", "docker", "--agent", - args.agent if args.native else (OPENROUTER_AGENT_IMPORTS if args.openrouter else AGENT_IMPORTS)[args.agent], + args.agent if args.native or agy_openrouter else (OPENROUTER_AGENT_IMPORTS if args.openrouter else AGENT_IMPORTS)[args.agent], "--force-build", "--yes", "-o", str(output), "--agent-setup-timeout-multiplier", "3", "--n-concurrent", "1", "--n-attempts", "1", "--max-retries", "0", @@ -280,7 +284,18 @@ def main(): required = [] agent_host = None native = None - if args.native: + if agy_openrouter: + effort = args.reasoning_effort or env.get("AGENT_REASONING_EFFORT") or "default" + if effort not in AGY_OPENROUTER_EFFORTS: + parser.error("Antigravity CLI with OpenRouter accepts reasoning effort: default, low, high") + native = { + "env": {"AGENT_OPENROUTER_API_KEY": "${AGENT_OPENROUTER_API_KEY}"}, + "kwargs": {"version": agent_version, "reasoning_effort": None if effort == "default" else effort, + "search_swe_openrouter": True}, + "hosts": [OPENROUTER_HOST], "setup_hosts": harbor_native.SETUP_HOSTS, "files": {}, + } + required.append("AGENT_OPENROUTER_API_KEY") + elif args.native: try: native = harbor_native.plan(args.agent, model, env, args.agent_config) if agent_version: @@ -488,7 +503,8 @@ def main(): print("Trusted gateway DNS servers: " + ", ".join(dns_servers)) print("Environment values are passed to Harbor separately from the command:") if native: - print(f"Native agent config: {args.agent_config or 'defaults'}; agent env: {', '.join(sorted(native['env'])) or 'none'}") + route = "Antigravity OpenRouter bridge" if agy_openrouter else f"Native agent config: {args.agent_config or 'defaults'}" + print(f"{route}; agent env: {', '.join(sorted(native['env'])) or 'none'}") print("Native parameters and authentication-file bindings are passed in a private generated job configuration.") print(shlex.join(command)) return 0 diff --git a/skills/run-searchswe-evaluation/SKILL.md b/skills/run-searchswe-evaluation/SKILL.md index 9c1f7f0..eeb69f6 100644 --- a/skills/run-searchswe-evaluation/SKILL.md +++ b/skills/run-searchswe-evaluation/SKILL.md @@ -3,7 +3,7 @@ name: run-searchswe-evaluation description: Configure, launch, monitor and summarize evaluations of existing Search-SWE tasks using the search-swe CLI. Use when the user specifies models, harnesses, providers, tasks or resource settings for benchmark runs, including preparation-only requests. license: Apache-2.0 metadata: - version: "0.3.0" + version: "0.3.1" --- # Run Search-SWE evaluations @@ -201,12 +201,12 @@ search-swe run agentic-search agentic-search-vectorless \ --reasoning-effort max --concurrency 2 --dry-run ``` -For Antigravity CLI through the Gemini API, using a key with access to the -requested model: +For Antigravity CLI through OpenRouter, use CLI 0.3.1+ and +`AGENT_OPENROUTER_API_KEY`: ```bash search-swe run agentic-search \ - --agent antigravity-cli --provider native --model gemini/gemini-4-argon \ + --agent antigravity-cli --provider openrouter --model google/gemini-4-argon \ --reasoning-effort high --dry-run ``` diff --git a/src/search_swe/__init__.py b/src/search_swe/__init__.py index e4a523a..0e17767 100644 --- a/src/search_swe/__init__.py +++ b/src/search_swe/__init__.py @@ -1,3 +1,3 @@ """Search-SWE evaluation CLI.""" -__version__ = "0.3.0" +__version__ = "0.3.1" diff --git a/src/search_swe/cli.py b/src/search_swe/cli.py index 7d55568..a0f6819 100644 --- a/src/search_swe/cli.py +++ b/src/search_swe/cli.py @@ -145,6 +145,9 @@ def configure_run(args, root): setting = resolve_setting(current, args) if setting.provider == "native" and not (root / "scripts/evaluation/native_agents.py").is_file(): raise ValueError("This checkout needs the Harbor native agent backend; update it with search-swe update before using --provider native") + if ((setting.agent, setting.provider) == ("antigravity-cli", "openrouter") + and not (root / "scripts/evaluation/agy_openrouter.py").is_file()): + raise ValueError("Antigravity CLI with OpenRouter requires an updated checkout. Run search-swe update or select a checkout containing this integration with --repo") if setting.agent_config: config_path = Path(setting.agent_config).expanduser() setting = replace(setting, agent_config=str((root / config_path).resolve())) diff --git a/src/search_swe/configuration.py b/src/search_swe/configuration.py index 81dc4a8..19e60d1 100644 --- a/src/search_swe/configuration.py +++ b/src/search_swe/configuration.py @@ -15,12 +15,21 @@ "SEARCH_SWE_PROVIDER": "openrouter", "AGENT_REASONING_EFFORT": "max", "VERIFIER_OPENAI_BASE_URL": "https://openrouter.ai/api/v1"} EFFORTS = ["low", "medium", "high", "xhigh", "max"] +AGY_OPENROUTER_EFFORTS = ("default", "low", "high") + + +def effort_levels(agent, provider): + if (agent, provider) == ("antigravity-cli", "openrouter"): + return list(AGY_OPENROUTER_EFFORTS) + return ["default", "off", "minimal", *EFFORTS] if agent == "pi" else ["default", *EFFORTS] def default_model(agent, provider): if provider == "native": return "" if provider == "openrouter": + if agent == "antigravity-cli": + return "google/gemini-4-argon" return "anthropic/claude-opus-5.5" if agent == "claude-code" else DEFAULTS["AGENT_MODEL"] if agent == "pi": return "deepseek/deepseek-flash" @@ -173,7 +182,7 @@ def validate(self): raise ValueError("--agent-config is used with --provider native") if self.agent == "pi" and self.provider != "native": pi_models.selection(pi_models.native_model(self.model, self.provider)) - levels = ["default", "off", "minimal", *EFFORTS] if self.agent == "pi" else ["default", *EFFORTS] + levels = effort_levels(self.agent, self.provider) if self.effort and self.effort not in levels: raise ValueError("Unsupported reasoning/thinking level: " + self.effort) if self.provider != "native": diff --git a/src/search_swe/harbor_native.py b/src/search_swe/harbor_native.py index a73c6e3..51cdee0 100644 --- a/src/search_swe/harbor_native.py +++ b/src/search_swe/harbor_native.py @@ -78,6 +78,8 @@ def agent_class(name): def routes(name): + if name == "antigravity-cli": + return ("native", "openrouter") return ("openrouter", "direct", "native") if name in ENHANCED else ("native",) diff --git a/src/search_swe/model_catalog.py b/src/search_swe/model_catalog.py index 06b5c79..8c326ef 100644 --- a/src/search_swe/model_catalog.py +++ b/src/search_swe/model_catalog.py @@ -110,7 +110,7 @@ def eligible(model, agent): return False if "supported_parameters" in model and "tools" not in model["supported_parameters"]: return False - endpoint = {"claude-code": "/api/v1/messages", "pi": "/api/v1/chat/completions", "codex": "/api/v1/responses"}[agent] + endpoint = {"claude-code": "/api/v1/messages", "codex": "/api/v1/responses"}.get(agent, "/api/v1/chat/completions") return "endpoints" not in model or endpoint in model["endpoints"] diff --git a/src/search_swe/wizard.py b/src/search_swe/wizard.py index f4a798a..62e6c0a 100644 --- a/src/search_swe/wizard.py +++ b/src/search_swe/wizard.py @@ -3,7 +3,7 @@ from pathlib import Path import questionary -from .configuration import EFFORTS, Setting, default_model +from .configuration import Setting, default_model, effort_levels from . import harbor_native, harnesses, pi_models from .model_catalog import MODEL_ID, ModelCompleter, eligible, load_models from .repository import clone_repo, is_repo, preference_path, remember_repo @@ -96,7 +96,7 @@ def select_setting(current, args, *, root=None): _, model_id, info = pi_models.selection(model) if not info["models"].get(model_id, {}).get("reasoning", True): metadata = {"supported_parameters": []} - levels = ["default", "off", "minimal", *EFFORTS] if agent == "pi" else ["default", *EFFORTS] + levels = effort_levels(agent, provider) effort = args.reasoning_effort if effort is None: parameters = metadata.get("supported_parameters") if metadata else None diff --git a/tests/unit/test_agy_openrouter.py b/tests/unit/test_agy_openrouter.py new file mode 100644 index 0000000..81e406d --- /dev/null +++ b/tests/unit/test_agy_openrouter.py @@ -0,0 +1,72 @@ +"""Protocol checks for the Antigravity OpenRouter bridge.""" + +from http.client import IncompleteRead +from io import BytesIO +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch + +from scripts.evaluation.agy_openrouter import GeminiHandler, gemini_response, openrouter_request + + +class AntigravityOpenRouterBridge(unittest.TestCase): + def test_tool_response_and_reasoning_are_replayed_together(self): + model = "google/gemini-4-argon" + details = [{"type": "reasoning.encrypted", "format": "google-gemini-v1", + "data": "opaque", "id": "call_example", "index": 0}] + call = {"id": "call_example", "type": "function", + "function": {"name": "read_file", "arguments": '{"Path":"fixture.txt"}'}} + response = gemini_response({ + "model": model, "choices": [{"finish_reason": "tool_calls", "message": { + "content": None, "tool_calls": [call], "reasoning_details": details, + }}], + "usage": {"prompt_tokens": 12, "completion_tokens": 10, "total_tokens": 22, + "completion_tokens_details": {"reasoning_tokens": 6}}, + }) + self.assertEqual(response["usageMetadata"]["candidatesTokenCount"], 4) + request = { + "systemInstruction": {"parts": [{"text": "Use the available tools."}]}, + "contents": [ + {"role": "user", "parts": [{"text": "Read the fixture."}]}, + response["candidates"][0]["content"], + # The actual agy built-in model path uses model for tool results. + {"role": "model", "parts": [{"functionResponse": { + "name": "read_file", "id": "call_example", "response": {"output": "fixture"}, + }}]}, + ], + "generationConfig": {"maxOutputTokens": 8192, "thinkingConfig": {"thinkingLevel": "HIGH"}}, + "tools": [{"functionDeclarations": [{"name": "read_file", "parametersJsonSchema": { + "type": "object", "properties": {"Path": {"type": "string"}}, + }}]}], + } + upstream = openrouter_request(request, model) + self.assertEqual(upstream["model"], model) + self.assertEqual(upstream["reasoning"], {"effort": "high"}) + self.assertEqual(upstream["max_tokens"], 8192) + self.assertEqual(upstream["tools"][0]["function"]["parameters"]["type"], "object") + self.assertEqual([m["role"] for m in upstream["messages"]], ["system", "user", "assistant", "tool"]) + self.assertEqual(upstream["messages"][2]["tool_calls"], [call]) + self.assertEqual(upstream["messages"][2]["reasoning_details"], details) + self.assertEqual(upstream["messages"][3], + {"role": "tool", "tool_call_id": "call_example", "content": '{"output":"fixture"}'}) + + def test_unmatched_tool_result_is_not_sent_as_a_user_message(self): + request = {"contents": [{"role": "user", "parts": [ + {"functionResponse": {"id": "missing", "name": "read_file", "response": {}}}, + ]}]} + with self.assertRaisesRegex(ValueError, "no matching call"): + openrouter_request(request, "google/gemini-4-argon") + + def test_truncated_upstream_response_reports_bad_gateway(self): + body = b'{"contents":[{"role":"user","parts":[{"text":"fixture"}]}]}' + handler = GeminiHandler.__new__(GeminiHandler) + handler.server = SimpleNamespace(model="google/gemini-4-argon", api_key="fixture", loopback_key="local") + handler.path = "/v1beta/models/gemini-4-argon:streamGenerateContent" + handler.headers = {"x-goog-api-key": "local", "Content-Length": str(len(body))} + handler.rfile = BytesIO(body) + handler.reply = Mock() + with patch("scripts.evaluation.agy_openrouter.call_openrouter", side_effect=IncompleteRead(b"{}")): + handler.do_POST() + handler.reply.assert_called_once_with(502, {"error": { + "code": 502, "message": "OpenRouter request or response failed (IncompleteRead)", "status": "BAD_GATEWAY", + }}) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index eb1b880..cff37ce 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -51,6 +51,15 @@ def test_compatible_patch_checkout_is_accepted(self): (self.root / "pyproject.toml").write_text('[project]\nname="search-swe"\nversion="0.3.1"\n') check_compatibility(self.root) + def test_agy_openrouter_requires_a_checkout_with_the_bridge(self): + (self.root / "pyproject.toml").write_text('[project]\nname="search-swe"\nversion="0.3.0"\n') + with patch("search_swe.runner.asset_plan", side_effect=AssertionError("assets prepared")), \ + redirect_stderr(io.StringIO()) as error: + self.assertEqual(main(["run", "first-search", "--repo", str(self.root), "--agent", "antigravity-cli", + "--provider", "openrouter", "--model", "google/gemini-4-argon", "--yes"]), 1) + self.assertIn("Antigravity CLI with OpenRouter requires an updated checkout", error.getvalue()) + self.assertFalse((self.root / ".env").exists()) + def test_new_pi_model_is_checked_before_credentials_and_backend_preflight(self): with patch("search_swe.model_catalog.load_models", return_value=({}, "unavailable")) as catalog, \ patch("search_swe.cli.runner.preflight", side_effect=AssertionError("late check")), \ diff --git a/tests/unit/test_harbor_native.py b/tests/unit/test_harbor_native.py index f2c8f33..1457481 100644 --- a/tests/unit/test_harbor_native.py +++ b/tests/unit/test_harbor_native.py @@ -84,8 +84,11 @@ def test_agent_route_and_credential_selection(self): self.assertIn("--native", setting.flags()) self.assertNotIn("--reasoning-effort", setting.flags()) self.assertEqual(Setting("nop", "native", "", "default").validate().required({}), set()) - with self.assertRaisesRegex(ValueError, "Unsupported"): - Setting("antigravity-cli", "openrouter", "google/model", "default").validate() + routed = Setting("antigravity-cli", "openrouter", "google/gemini-4-argon", "high").validate() + self.assertEqual(routed.required({}), {"AGENT_OPENROUTER_API_KEY"}) + self.assertIn("--openrouter", routed.flags()) + with self.assertRaisesRegex(ValueError, "Unsupported reasoning"): + Setting("antigravity-cli", "openrouter", "google/gemini-4-argon", "max").validate() plan = harbor_native.plan("aider", "openrouter/vendor/model", {"OPENROUTER_API_KEY": "submission-only"}) self.assertEqual(plan["env"], {"OPENROUTER_API_KEY": "${AGENT_OPENROUTER_API_KEY}"}) self.assertEqual(plan["required"], {"AGENT_OPENROUTER_API_KEY"}) @@ -395,6 +398,46 @@ def test_generated_job_keeps_kwargs_private_and_oracle_native(self): self.assertEqual(job.agents[0].kwargs["search_swe_native_name"], name) self.assertFalse(path.exists()) + def test_agy_openrouter_launch_keeps_credentials_and_provider_scoped(self): + launch_env = { + "AGENT_OPENROUTER_API_KEY": "coding-fixture", + "OPENROUTER_API_KEY": "submission-fixture", + "VERIFIER_OPENAI_BASE_URL": "https://openrouter.ai/api/v1", + "VERIFIER_OPENAI_API_KEY": "judge-fixture", + "ANSWER_JUDGE_BASE_URL": "https://openrouter.ai/api/v1", + "ANSWER_JUDGE_API_KEY": "answer-fixture", + "ANSWER_JUDGE_MODEL_NAME": "fixture/model", + } + + def inspect_job(argv, env, _repo): + path = Path(argv[argv.index("--config") + 1]) + self.assertNotIn("coding-fixture", path.read_text()) + self.assertNotIn("coding-fixture", argv) + job = JobConfig.model_validate_json(path.read_text()) + agent_config = job.agents[0] + self.assertEqual(agent_config.model_name, "google/gemini-4-argon") + self.assertEqual(agent_config.env, {"AGENT_OPENROUTER_API_KEY": "${AGENT_OPENROUTER_API_KEY}"}) + self.assertEqual(agent_config.extra_allowed_hosts, ["openrouter.ai"]) + self.assertEqual(agent_config.kwargs["reasoning_effort"], "high") + with patch.dict(os.environ, env, clear=True): + agent = AgentFactory.create_agent_from_config(agent_config, logs_dir=self.root) + self.assertEqual(agent.extra_env, {"AGENT_OPENROUTER_API_KEY": "coding-fixture"}) + self.assertIsInstance(agent, AntigravityCli) + self.assertEqual(env["OPENROUTER_API_KEY"], "submission-fixture") + self.assertEqual(env["VERIFIER_OPENAI_API_KEY"], "judge-fixture") + return 0 + + argv = ["run_task.py", "--task", "scientific-paper-qa", "--agent", "antigravity-cli", + "--openrouter", "--model", "google/gemini-4-argon", "--reasoning-effort", "high", "--no-env-file"] + with patch.object(sys, "argv", argv), patch.dict(os.environ, launch_env, clear=True), \ + patch.object(run_task, "read_manifest", return_value={"files": []}), \ + patch.object(run_task.shutil, "which", return_value="/fixture/harbor"), \ + patch.object(run_task, "prepare_task"), \ + patch.object(run_task, "run_command", side_effect=inspect_job) as launched, \ + redirect_stdout(io.StringIO()): + self.assertEqual(run_task.main(), 0) + launched.assert_called_once() + def test_each_registered_agent_reaches_native_launcher_preview(self): task = self.root / "tasks/example-search" task.mkdir(parents=True) From 760a57e1e16a191823e5b29383dce3eb777a7f53 Mon Sep 17 00:00:00 2001 From: hanhainebula <2512674094@qq.com> Date: Thu, 19 Nov 2026 14:19:30 +0800 Subject: [PATCH 3/5] Add verifier-only regrading for recorded Harbor trials --- docs/README.md | 2 +- docs/evaluation/cli.md | 25 +++ scripts/evaluation/run_task.py | 154 +++++++------- skills/run-searchswe-evaluation/SKILL.md | 8 +- src/search_swe/cli.py | 12 ++ src/search_swe/regrade.py | 248 +++++++++++++++++++++++ tests/unit/test_regrade.py | 159 +++++++++++++++ 7 files changed, 528 insertions(+), 80 deletions(-) create mode 100644 src/search_swe/regrade.py create mode 100644 tests/unit/test_regrade.py diff --git a/docs/README.md b/docs/README.md index d759ebc..48c94c3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,7 +9,7 @@ task with the CLI or your coding agent. | Guide | Use it to | | --- | --- | | [Quick start](evaluation/quickstart.md) | Install, choose CLI or agent operation, and inspect your first results | -| [CLI reference](evaluation/cli.md) | Configure settings, select tasks, and schedule sequential or parallel batches | +| [CLI reference](evaluation/cli.md) | Configure settings, run batches, and regrade recorded trials | | [Supported agents and providers](evaluation/agents.md) | Choose harnesses, versions and provider routes; configure API keys or Claude Code/Codex subscriptions | | [Submission model access](evaluation/submission-models.md) | Apply the required OpenRouter model allowlists to submission API keys | | [Data and models](evaluation/assets.md) | Plan disk space, download, verify, and restore fixed inputs | diff --git a/docs/evaluation/cli.md b/docs/evaluation/cli.md index 9429fdc..016abc5 100644 --- a/docs/evaluation/cli.md +++ b/docs/evaluation/cli.md @@ -59,6 +59,31 @@ New tasks are discovered from the checkout and normally need no CLI upgrade. The target checkout must match the installed CLI's release series. `update` updates repository files; package upgrades remain separate Python installations. +## Regrade a recorded trial + +Pass the directory of one completed local Harbor trial. The CLI discovers the +task from the recorded result and uses the verifier in the selected checkout. + +```bash +search-swe regrade /path/to/original-trial --dry-run +search-swe regrade /path/to/original-trial +``` + +Keep the original `config.json`, `result.json`, `agent/`, and +`artifacts/manifest.json` with its recorded artifact files. The preview checks +that the current verifier's declared inputs are present, including the ATIF +trajectory for tasks that require it. Run with the task's verifier credentials +and network settings in `.env` or `--env-file`. The verifier uses the task's +usual asset preparation and CPU/GPU requirements. Use `--offline` for cached +assets, `--cpu-set` for a CPU pool, or `--gpu-device` for a specific GPU. + +The CLI starts only the verifier and preserves the original trial. The new +trial and its `regrade.json` summary are written under `jobs/regrades/` or a +fresh `--output` directory. The summary records both trial IDs, the verifier +checkout revision, and rewards. Measurement validity is recorded when the +verifier reports it explicitly. Review the new verifier logs before using +the result, including when the reward is zero. + ## Interactive setup `search-swe` or `search-swe run` opens the wizard: diff --git a/scripts/evaluation/run_task.py b/scripts/evaluation/run_task.py index 1fd9b07..4404310 100755 --- a/scripts/evaluation/run_task.py +++ b/scripts/evaluation/run_task.py @@ -127,6 +127,64 @@ def host_is_allowed(host, allowed_hosts): ) +def verifier_settings(task_config, env): + """Return the variables and Harbor bindings for this task's verifier.""" + verifier_env = task_config.get("verifier", {}).get("env", {}) + mode, allowed_hosts = effective_network_policy(task_config, "verifier") + binds = {} + required = set() + endpoints = [] + if "OPENAI_API_KEY" in verifier_env: + for name in ("OPENAI_BASE_URL", "OPENAI_API_KEY"): + source = f"VERIFIER_{name}" + required.add(source) + binds[name] = f"${{{source}}}" + endpoints.append("VERIFIER_OPENAI_BASE_URL") + for name in ("ANSWER_JUDGE_MODEL_NAME", "ANSWER_JUDGE_BASE_URL", "ANSWER_JUDGE_API_KEY"): + required.update(re.findall(r"\$\{([A-Z][A-Z0-9_]*)(?::[-?][^}]*)?\}", verifier_env.get(name, ""))) + if "${ANSWER_JUDGE_BASE_URL" in verifier_env.get("ANSWER_JUDGE_BASE_URL", ""): + endpoints.append("ANSWER_JUDGE_BASE_URL") + for variable in endpoints: + if not env.get(variable): + continue + host = endpoint_hostname(env[variable], variable) + if mode == "no-network": + raise ValueError(f"{variable} cannot be used with verifier network_mode=no-network") + if mode == "allowlist" and not host_is_allowed(host, allowed_hosts): + raise ValueError(f"{variable} host {host!r} is not permitted by the task's [verifier].allowed_hosts") + return required, binds + + +def network_settings(task, env, agent_hosts, egress_path, dns_servers, egress_image): + """Select trusted Compose egress and service proxy bindings.""" + from scripts.egress.config import DEFAULT_IMAGE, direct_config, load_config, validate_task_networks + + if egress_path is not None: + if dns_servers or env.get("CONTAINER_PROXY") or egress_image: + raise ValueError("EGRESS_CONFIG cannot be combined with CONTAINER_DNS/--container-dns, CONTAINER_PROXY, or EGRESS_IMAGE/--egress-image; clear them explicitly") + egress = load_config(egress_path) + else: + egress = direct_config(egress_image or DEFAULT_IMAGE, dns_servers) + restricted = validate_task_networks(task, agent_hosts, egress.settings.get("upstream_host"), + allow_public=egress.settings.get("transport") == "direct") + kwargs = None + if egress_path is not None: + kwargs = {"egress_config": str(egress.path)} + elif restricted: + kwargs = {"egress_image": egress.image} + if dns_servers: + kwargs["egress_dns"] = ",".join(dns_servers) + proxy_bindings = {} + if env.get("CONTAINER_PROXY"): + if restricted: + raise ValueError("CONTAINER_PROXY is incompatible with this task's Harbor network " + "allowlists because a general proxy can bypass destination filtering") + env.setdefault("CONTAINER_NO_PROXY", "127.0.0.1,localhost") + proxy_bindings = {name: "${CONTAINER_PROXY}" for name in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY")} + proxy_bindings.update({name: "${CONTAINER_NO_PROXY}" for name in ("no_proxy", "NO_PROXY")}) + return egress, kwargs, proxy_bindings + + def main(): parser = argparse.ArgumentParser(description=__doc__) selection = parser.add_mutually_exclusive_group(required=True) @@ -264,7 +322,6 @@ def main(): validate_gpu_device(args.gpu_device) except ValueError as error: parser.error(str(error)) - verifier_env = task_config.get("verifier", {}).get("env", {}) output = args.output.resolve() if args.output else REPO / "jobs" / task_key(REPO, task) command = [ "harbor", "run", "--path", str(task), "--env", "docker", @@ -378,9 +435,6 @@ def main(): command.extend(["--ak", "model_config=" + json.dumps(model_config)]) agent_network_mode, _ = effective_network_policy(task_config, "agent") - verifier_network_mode, verifier_allowed_hosts = effective_network_policy( - task_config, "verifier" - ) agent_requires_host = any(step.get("agent", {}).get("network_mode", agent_network_mode) != "public" for step in task_config.get("steps") or [{}]) agent_hosts = native["hosts"] if native else [agent_host] if agent_host else [] @@ -393,86 +447,30 @@ def main(): for host in agent_hosts: command.extend(["--allow-agent-host", host]) - if "OPENAI_API_KEY" in verifier_env: - for name in ("OPENAI_BASE_URL", "OPENAI_API_KEY"): - source = f"VERIFIER_{name}" - required.append(source) - command.extend(["--verifier-env", f"{name}=${{{source}}}"]) - - # Task-specific judge settings are resolved by task.toml from Harbor's - # environment; require only the groups used by the selected task. - for name in ("ANSWER_JUDGE_MODEL_NAME", "ANSWER_JUDGE_BASE_URL", "ANSWER_JUDGE_API_KEY"): - required.extend(re.findall(r"\$\{([A-Z][A-Z0-9_]*)(?::[-?][^}]*)?\}", verifier_env.get(name, ""))) - task_setup = task_config.get("metadata", {}).get("configuration", {}) - required.extend(name for name, options in task_setup.items() if options.get("required")) - - endpoint_variables = [] - if "OPENAI_API_KEY" in verifier_env: - endpoint_variables.append("VERIFIER_OPENAI_BASE_URL") - if "${ANSWER_JUDGE_BASE_URL" in verifier_env.get("ANSWER_JUDGE_BASE_URL", ""): - endpoint_variables.append("ANSWER_JUDGE_BASE_URL") - for variable in endpoint_variables: - if not env.get(variable): - continue - try: - host = endpoint_hostname(env[variable], variable) - except ValueError as error: - parser.error(str(error)) - if verifier_network_mode == "no-network": - parser.error(f"{variable} cannot be used with verifier network_mode=no-network") - if verifier_network_mode == "allowlist" and not host_is_allowed( - host, verifier_allowed_hosts - ): - parser.error( - f"{variable} host {host!r} is not permitted by the task's " - "[verifier].allowed_hosts" - ) - - from scripts.egress.config import DEFAULT_IMAGE, direct_config, load_config, validate_task_networks - egress_image = args.egress_image or env.get("EGRESS_IMAGE") try: - if egress_path is not None: - if dns_servers or env.get("CONTAINER_PROXY") or egress_image: - parser.error("EGRESS_CONFIG cannot be combined with CONTAINER_DNS/--container-dns, CONTAINER_PROXY, or EGRESS_IMAGE/--egress-image; clear them explicitly") - egress = load_config(egress_path) - else: - egress = direct_config(egress_image or DEFAULT_IMAGE, dns_servers) - restricted = validate_task_networks(task, agent_hosts, egress.settings.get("upstream_host"), - allow_public=egress.settings.get("transport") == "direct") + required_verifier, verifier_bindings = verifier_settings(task_config, env) + egress, gateway_kwargs, proxy_bindings = network_settings( + task, env, agent_hosts, egress_path, dns_servers, args.egress_image or env.get("EGRESS_IMAGE") + ) except (OSError, ValueError) as error: parser.error(str(error)) - use_gateway = restricted or egress_path is not None - - if env.get("CONTAINER_PROXY"): - if restricted: - parser.error( - "CONTAINER_PROXY is incompatible with this task's Harbor network " - "allowlists because a general proxy can bypass destination filtering" - ) - env.setdefault("CONTAINER_NO_PROXY", "127.0.0.1,localhost") - for flag in ("--ae", "--verifier-env"): - for name in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY"): - command.extend([flag, f"{name}=${{CONTAINER_PROXY}}"]) - for name in ("no_proxy", "NO_PROXY"): - command.extend([flag, f"{name}=${{CONTAINER_NO_PROXY}}"]) - - if use_gateway: - command[command.index("--env") + 1] = "scripts.evaluation.harbor_environments:PhaseScopedDocker" - if egress_path is not None: - command.extend(["--environment-kwarg", f"egress_config={egress.path}"]) - else: - command.extend(["--environment-kwarg", f"egress_image={egress.image}"]) - if dns_servers: - command.extend(["--environment-kwarg", "egress_dns=" + ",".join(dns_servers)]) - + required.extend(required_verifier) + task_setup = task_config.get("metadata", {}).get("configuration", {}) + required.extend(name for name, options in task_setup.items() if options.get("required")) + for name, value in {**verifier_bindings, **proxy_bindings}.items(): + command.extend(["--verifier-env", f"{name}={value}"]) + for name, value in proxy_bindings.items(): + command.extend(["--ae", f"{name}={value}"]) + + use_gateway = gateway_kwargs is not None + if use_gateway or args.cpu_set is not None or args.gpu_device is not None: + adapter = "PhaseScopedDocker" if use_gateway else "GpuDocker" + command[command.index("--env") + 1] = f"scripts.evaluation.harbor_environments:{adapter}" + for name, value in (gateway_kwargs or {}).items(): + command.extend(["--environment-kwarg", f"{name}={value}"]) if args.cpu_set is not None: - if not use_gateway: - command[command.index("--env") + 1] = "scripts.evaluation.harbor_environments:GpuDocker" command.extend(["--environment-kwarg", f"cpu_set={args.cpu_set}"]) - if args.gpu_device is not None: - if not use_gateway: - command[command.index("--env") + 1] = "scripts.evaluation.harbor_environments:GpuDocker" command.extend(["--environment-kwarg", f"gpu_device={args.gpu_device}"]) # This launcher uses explicit API keys, without consulting a host auth.json. diff --git a/skills/run-searchswe-evaluation/SKILL.md b/skills/run-searchswe-evaluation/SKILL.md index eeb69f6..b9957ff 100644 --- a/skills/run-searchswe-evaluation/SKILL.md +++ b/skills/run-searchswe-evaluation/SKILL.md @@ -1,6 +1,6 @@ --- name: run-searchswe-evaluation -description: Configure, launch, monitor and summarize evaluations of existing Search-SWE tasks using the search-swe CLI. Use when the user specifies models, harnesses, providers, tasks or resource settings for benchmark runs, including preparation-only requests. +description: Configure, run and monitor Search-SWE evaluations or regrade a recorded trial using the search-swe CLI. Use for benchmark runs, their model and harness settings, resource allocation, and preparation-only requests. license: Apache-2.0 metadata: version: "0.3.1" @@ -55,6 +55,12 @@ Read `docs/evaluation/cli.md` in the selected checkout for its supported flags. `docs/evaluation/runtime.md` only for provider, network or bounded-storage setup details. These paths are relative to the benchmark checkout, not this installed skill. +For a verifier-only replay, use `search-swe regrade ORIGINAL_TRIAL --dry-run` +to check the recorded inputs, then run `search-swe regrade ORIGINAL_TRIAL` +with the verifier's service configuration. Report the new trial and its +`regrade.json`, including measurement validity when available. See the +regrade section of `docs/evaluation/cli.md`. + ## Resolve the requested experiment - Preserve the requested harness, provider, complete model ID, reasoning/thinking diff --git a/src/search_swe/cli.py b/src/search_swe/cli.py index a0f6819..691151b 100644 --- a/src/search_swe/cli.py +++ b/src/search_swe/cli.py @@ -48,6 +48,7 @@ def parser(): commands = root.add_subparsers(dest="command") descriptions = { "run": "Select tasks, check resources, restore assets and run evaluations.", + "regrade": "Run the current verifier against the recorded outputs of one local Harbor trial.", "configure": "Save model settings, API keys and default concurrency to .env. Select optional tasks to configure their services and check submission-key restrictions.", "list": "List available tasks and their resource requirements.", "doctor": "Check the local Docker host and evaluation prerequisites.", @@ -80,6 +81,14 @@ def parser(): command.add_argument("--env-file", type=Path, help="Environment file; defaults to the checkout .env") if name == "update": command.add_argument("--check", action="store_true", help="Fetch and report available updates without changing checkout files") + if name == "regrade": + command.add_argument("trial", type=Path, help="Original Harbor trial directory containing result.json, agent/ and artifacts/") + command.add_argument("--env-file", type=Path, help="Verifier and task environment; defaults to the checkout .env") + command.add_argument("--output", type=Path, help="Fresh result directory; defaults to jobs/regrades//") + command.add_argument("--cpu-set", help="Verifier CPU pool, e.g. 0-31") + command.add_argument("--gpu-device", help="Verifier GPU index or full UUID; defaults to an idle GPU") + command.add_argument("--offline", action="store_true", help="Restore task assets from cache only; verifier APIs may still need network") + command.add_argument("--dry-run", action="store_true", help="Check the trial's recorded inputs and preview without writing, downloading or starting Docker") if name not in ("run", "configure"): continue command.set_defaults(cpu_set=None, gpu_devices=None, memory_gb=None, output=None, edit_credentials=False) @@ -326,6 +335,9 @@ def main(argv=None): result = runner.doctor(root, env={**DEFAULTS, **read_env(path), **os.environ}) print(json.dumps(result, indent=2) if args.json else "\n".join(f"{key}: {value}" for key, value in result.items())) return 0 + if args.command == "regrade": + from .regrade import run + return run(args, root) return configure_run(args, root) except KeyboardInterrupt: print("Cancelled; inspect the batch summary if execution had started.", file=sys.stderr) diff --git a/src/search_swe/regrade.py b/src/search_swe/regrade.py new file mode 100644 index 0000000..48ed817 --- /dev/null +++ b/src/search_swe/regrade.py @@ -0,0 +1,248 @@ +"""Replay a local Harbor trial's recorded artifacts with the checkout's verifier.""" + +from contextlib import nullcontext +import json +import os +from pathlib import Path +import stat +import sys +import tempfile +from uuid import uuid4 + +from . import runner, updates +from .configuration import DEFAULTS, read_env +from .repository import VARIABLE, check_compatibility, discover +from .resources import Pool, cpu_ids +from .submission_checks import verify_submission_keys + + +def recorded_trial(source): + from harbor.models.trial.result import TrialResult + + for name in ("config.json", "result.json", "artifacts/manifest.json"): + path = source / name + if path.is_symlink() or not path.is_file(): + raise ValueError(f"Original Harbor trial is incomplete: missing regular file {path}") + if (source / "steps").exists(): + raise ValueError("Regrade requires a single-step Harbor trial.") + try: + result = TrialResult.model_validate_json((source / "result.json").read_text()) + except ValueError: + raise ValueError(f"Invalid Harbor trial result: {source / 'result.json'}") from None + if result.step_results: + raise ValueError("Regrade requires a single-step Harbor trial.") + return result + + +def recorded_inputs(source, task): + from harbor.constants import MAIN_SERVICE_NAME + from harbor.models.task.artifacts import convention_source_for_os, effective_artifact_service, with_convention_entry + from harbor.models.task.config import TaskConfig + from harbor.trial.artifact_handler import artifact_host_path + from harbor.trial.regrade import RegradeError, check_task_regradable, read_artifact_manifest + + if problem := check_task_regradable(task): + raise ValueError(problem) + config = TaskConfig.model_validate_toml((task / "task.toml").read_text()) + convention = convention_source_for_os(config.environment.os) + + # Harbor copies the two entire trees. Reject links and special files before + # it can follow links to unrelated host files or block reading a FIFO. + def walk_error(error): + raise error + + for directory in (source / "agent", source / "artifacts"): + if directory.is_symlink() or not directory.is_dir(): + raise ValueError(f"Original Harbor trial is incomplete: expected directory {directory}") + for base, directories, files in os.walk(directory, onerror=walk_error): + for name in directories + files: + path = Path(base) / name + mode = path.lstat().st_mode + if not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)): + raise ValueError(f"Only regular files and directories can be replayed: {path}") + + try: + manifest = read_artifact_manifest(source) + except RegradeError: + raise ValueError(f"Invalid Harbor artifact manifest: {source / 'artifacts/manifest.json'}") from None + entries = {(item.service or MAIN_SERVICE_NAME, item.source.rstrip("/")): item + for item in manifest} + if len(entries) != len(manifest): + raise ValueError("Original Harbor trial has duplicate manifest entries.") + missing = [] + for artifact in with_convention_entry(config.artifacts, convention_source=convention): + key = (effective_artifact_service(artifact), artifact.source.rstrip("/")) + entry = entries.get(key) + path = artifact_host_path(source / "artifacts", artifact) + if entry and entry.destination == path.relative_to(source).as_posix(): + if (entry.status == "empty" and entry.type == "directory" + and key == (MAIN_SERVICE_NAME, convention)): + continue + if entry.status == "ok" and ((entry.type == "file" and path.is_file()) + or (entry.type == "directory" and path.is_dir())): + continue + missing.append(artifact.source) + if missing: + raise ValueError(f"Original trial is missing usable recorded inputs for {task.name}: " + + ", ".join(missing)) + + +def measurement_valid(verifier): + health = [] + for name in ("reward-details.json", "judge-status.json"): + try: + data = json.loads((verifier / name).read_text()) + except (OSError, ValueError): + continue + if not isinstance(data, dict): + continue + for details in (data, data.get("final_gate")): + if not isinstance(details, dict): + continue + if type(details.get("measurement_valid")) is bool: + health.append(details["measurement_valid"]) + elif details.get("evaluation_valid") is True and details.get("judge_succeeded") is False: + # Some finalizers write zero when the audit could not complete. + health.append(False) + return all(health) if health else None + + +def result_summary(trial, report): + from harbor.models.trial.result import TrialResult + + try: + result = TrialResult.model_validate_json((trial / "result.json").read_text()) + except (OSError, ValueError): + return + rewards = result.verifier_result.rewards if result.verifier_result else None + report.update(trial_id=str(result.id), task_checksum=result.task_checksum, rewards=rewards) + if result.exception_info: + report.update(measurement_valid=False, status="verifier-error", + exception_type=result.exception_info.exception_type) + elif rewards: + valid = measurement_valid(trial / "verifier") + report.update(measurement_valid=valid, status="invalid-measurement" if valid is False else "completed") + + +def run(args, root): + from harbor.models.trial.config import AgentConfig, EnvironmentConfig, SourceTrialConfig, TaskConfig, TrialConfig, VerifierConfig + + from harbor.trial.regrade import local_task_name + + check_compatibility(root) + source = args.trial.expanduser().resolve() + original = recorded_trial(source) + name = original.task_name.removeprefix("search-swe/") + task = discover(root).get(name) + if task is None: + raise ValueError(f"Source task {original.task_name!r} is unavailable in this checkout.") + launcher = runner.backend(root, "evaluation/run_task") + if not hasattr(launcher, "verifier_settings"): + raise ValueError("This checkout is missing regrade support. Run search-swe update or select a newer checkout with --repo.") + task_path = launcher.select_task(root, f"tasks/{name}") + if original.task_name != local_task_name(task_path): + raise ValueError(f"Source task {original.task_name!r} does not match the current task metadata.") + recorded_inputs(source, task_path) + + trial_name = f"{name}__regrade_{uuid4().hex[:12]}" + output = args.output.expanduser().resolve() if args.output else root / "jobs" / "regrades" / trial_name + if output.exists() or output.is_relative_to(source) or output.is_relative_to(root / "tasks"): + raise ValueError(f"Choose a fresh --output outside the source trial and task packages: {output}") + if args.cpu_set: + cpu_ids(args.cpu_set) + if args.gpu_device: + if task.gpus != 1: + raise ValueError("--gpu-device requires a single-GPU task.") + launcher.validate_gpu_device(args.gpu_device) + env_file = (args.env_file or root / ".env").expanduser().resolve() + if args.env_file and not env_file.is_file(): + raise ValueError(f"Environment file does not exist: {env_file}") + env = {**DEFAULTS, **read_env(env_file), **os.environ} + env = {key: value for key, value in env.items() if value is not None} + required, verifier_env = launcher.verifier_settings(task.config, env) + references = {name for value in task.config.get("verifier", {}).get("env", {}).values() + for name in VARIABLE.findall(value)} + required.update(name for name, settings in task.configuration().items() + if name in references and settings.get("required")) + missing = sorted(name for name in required if not env.get(name, "").strip()) + egress_path = root / env["EGRESS_CONFIG"] if env.get("EGRESS_CONFIG") else None + dns_servers = launcher.parse_servers(env["CONTAINER_DNS"]) if env.get("CONTAINER_DNS") else None + _, gateway, proxy_env = launcher.network_settings(task_path, env, [], egress_path, dns_servers, + env.get("EGRESS_IMAGE")) + if dns_servers and gateway is None: + raise ValueError("CONTAINER_DNS requires a restricted task or an explicit direct EGRESS_CONFIG") + + print(f"Task: {name}\nSource: {source} ({original.id})\nVerifier checkout: {root}\nOutput: {output}", flush=True) + if args.dry_run: + print(f"Verifier resources: {task.cpus} CPUs, {task.memory_mb / 1024:g} GiB RAM, {task.gpus} GPUs.") + print(f"Preview only. Recorded inputs are present. Missing verifier settings: {', '.join(missing) or 'none'}.") + return 0 + if missing: + raise ValueError(f"Set {', '.join(missing)} in {env_file} or the shell.") + + runner.doctor(root, [task], env) + pool = Pool.host([task], cpu_set=args.cpu_set, gpu_devices=args.gpu_device) + pool.validate([task], env) + allocation = pool.acquire(task, env) + storage = launcher.StoragePlan.discover(task_path, task.config) + if storage: + env[storage.cpu_variable] = allocation.cpu_set + space = runner.backend(root, "assets/asset_space") + asset_plan = runner.asset_plan(root, [task], env) + space.describe(asset_plan, root / "tasks") + space.validate(asset_plan) + verify_submission_keys([task], env) + + adapter = "PhaseScopedDocker" if gateway is not None else "GpuDocker" + environment_kwargs = {**(gateway or {}), "cpu_set": allocation.cpu_set} + if allocation.gpu: + environment_kwargs["gpu_device"] = allocation.gpu + config = TrialConfig( + task=TaskConfig(path=task_path, source=original.source), + trial_name=trial_name, + trials_dir=output, + source_trial=SourceTrialConfig(action="regrade", type="local", trial_id=original.id, path=source), + agent=AgentConfig(name="nop"), + environment=EnvironmentConfig( + import_path=f"scripts.evaluation.harbor_environments:{adapter}", + override_gpus=0 if task.gpus else None, kwargs=environment_kwargs, + ), + verifier=VerifierConfig(env={**verifier_env, **proxy_env}), + ) + + env["PYTHONPATH"] = os.pathsep.join(filter(None, (str(root), env.get("PYTHONPATH")))) + git = updates.git_state(root) or {} + output.mkdir(parents=True, mode=0o700, exist_ok=False) + summary = output / "regrade.json" + trial = output / trial_name + report = { + "task": name, "source_trial": str(source), "source_trial_id": str(original.id), + "source_task_checksum": original.task_checksum, "trial": str(trial), + "verifier_checkout": str(root), "verifier_revision": git.get("revision"), + "verifier_checkout_dirty": git.get("dirty"), + "measurement_valid": None, "status": "preparing", + } + runner.private_json(summary, report) + try: + download = [sys.executable, str(root / "scripts/assets/download_assets.py"), "--task", name, "--force"] + if args.offline: + download.append("--local-files-only") + if launcher.run_command(download, env, root): + raise RuntimeError(f"Asset restoration failed for {name}.") + launcher.prepare_task(task_path, env, cpu_set=allocation.cpu_set) + with tempfile.TemporaryDirectory(prefix="search-swe-regrade-") as directory: + path = Path(directory) / "trial.json" + runner.private_json(path, config.model_dump(mode="json")) + report.update(status="verifier-error", measurement_valid=False) + with launcher.prepared_storage(storage, env) if storage else nullcontext(): + code = launcher.run_command(["harbor", "trial", "start", "--config", str(path)], env, root) + result_summary(trial, report) + if code: + report.update(status="verifier-error", measurement_valid=False, exit_code=code) + except BaseException: + report["status"] = "failed" + raise + finally: + runner.private_json(summary, report) + print(f"Regrade summary: {summary} ({report['status']})", flush=True) + return 0 if report["status"] == "completed" else report.get("exit_code", 1) diff --git a/tests/unit/test_regrade.py b/tests/unit/test_regrade.py new file mode 100644 index 0000000..085a8f8 --- /dev/null +++ b/tests/unit/test_regrade.py @@ -0,0 +1,159 @@ +from contextlib import redirect_stderr, redirect_stdout +import io +import json +import os +from pathlib import Path +import sys +from tempfile import TemporaryDirectory +import unittest +from unittest.mock import patch +from uuid import uuid4 + +from harbor.models.trial.config import TrialConfig + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "src")) + +from search_swe.cli import main +from search_swe.regrade import result_summary + + +def fixture(work): + checkout = work / "checkout" + checkout.mkdir() + for name in ("scripts", "environments", "src"): + (checkout / name).symlink_to(ROOT / name, target_is_directory=True) + (checkout / "pyproject.toml").write_text((ROOT / "pyproject.toml").read_text()) + task = checkout / "tasks/fixture-regrade" + (task / "environment").mkdir(parents=True) + (task / "tests").mkdir() + (task / "instruction.md").write_text("Fixture.") + (task / "assets.json").write_text('{"schema_version": 1, "files": []}') + (task / "task.toml").write_text(""" +artifacts = ["/app/submission", "/logs/agent/trajectory.json"] +[task] +name = "search-swe/fixture-regrade" +[environment] +cpus = 1 +memory_mb = 512 +network_mode = "no-network" +[verifier] +environment_mode = "separate" +""") + source = work / "original" + (source / "agent").mkdir(parents=True) + (source / "artifacts/app/submission").mkdir(parents=True) + (source / "artifacts/logs/agent").mkdir(parents=True) + (source / "artifacts/app/submission/answer.txt").write_text("recorded!") + (source / "artifacts/logs/agent/trajectory.json").write_text('{"recorded": true}') + (source / "artifacts/manifest.json").write_text(json.dumps([ + {"source": "/logs/artifacts", "destination": "artifacts/logs/artifacts", "type": "directory", "status": "empty"}, + {"source": "/app/submission", "destination": "artifacts/app/submission", "type": "directory", "status": "ok"}, + {"source": "/logs/agent/trajectory.json", "destination": "artifacts/logs/agent/trajectory.json", + "type": "file", "status": "ok"}, + ])) + config = {"task": {"path": str(task)}, "agent": {"name": "codex", "model_name": "provider/original"}} + result = {"id": str(uuid4()), "task_name": "search-swe/fixture-regrade", "trial_name": "original", + "trial_uri": source.as_uri(), "task_id": {"path": str(task)}, "task_checksum": "original-hash", + "config": config, "agent_info": {"name": "codex", "version": "test", + "model_info": {"name": "original", "provider": "provider"}}} + (source / "config.json").write_text(json.dumps(config)) + (source / "result.json").write_text(json.dumps(result)) + return checkout, task, source, result + + +class RegradeTests(unittest.TestCase): + def setUp(self): + directory = TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.checkout, self.task, self.source, self.original = fixture(Path(directory.name)) + self.output = Path(directory.name) / "new-output" + self.args = ["regrade", str(self.source), "--repo", str(self.checkout), "--output", str(self.output)] + environment = patch.dict(os.environ, {"PATH": os.environ["PATH"]}, clear=True) + environment.start() + self.addCleanup(environment.stop) + + def invoke(self, *args): + with redirect_stdout(io.StringIO()) as output, redirect_stderr(io.StringIO()) as error: + code = main([*self.args, *args]) + return code, output.getvalue() + error.getvalue() + + def test_preview_checks_inputs_without_side_effects(self): + with patch("search_swe.runner.doctor", side_effect=AssertionError("started preflight")): + code, output = self.invoke("--dry-run") + self.assertEqual(code, 0, output) + self.assertIn(self.original["id"], output) + self.assertFalse(self.output.exists()) + (self.source / "artifacts/logs/agent/trajectory.json").unlink() + code, output = self.invoke("--dry-run") + self.assertEqual(code, 1, output) + self.assertIn("/logs/agent/trajectory.json", output) + + def test_source_symlinks_are_rejected_before_replay(self): + (self.source / "agent/other").symlink_to(self.source / "result.json") + code, output = self.invoke("--dry-run") + self.assertEqual(code, 1, output) + self.assertIn("Only regular files and directories", output) + + def test_only_verifier_is_configured_and_zero_reward_is_preserved(self): + from search_swe import runner + launcher = runner.backend(self.checkout, "evaluation/run_task") + space = runner.backend(self.checkout, "assets/asset_space") + (self.task / "task.toml").write_text((self.task / "task.toml").read_text().replace( + 'network_mode = "no-network"', 'network_mode = "allowlist"\nallowed_hosts = ["openrouter.ai"]' + ) + '\n[verifier.env]\nOPENAI_API_KEY = "${VERIFIER_OPENAI_API_KEY:-}"\n') + task = self.task / "environment/docker-compose.yaml" + task.write_text("services: {}\n") + new_id = str(uuid4()) + original_source = (self.source / "result.json").read_bytes() + + def execute(command, env, cwd): + if command[0] == sys.executable: + self.assertIn("--local-files-only", command) + return 0 + self.assertEqual(command[:4], ["harbor", "trial", "start", "--config"]) + config_path = Path(command[4]) + raw_config = config_path.read_text() + self.assertNotIn("verifier-token", raw_config) + self.assertEqual(config_path.stat().st_mode & 0o777, 0o600) + config = TrialConfig.model_validate_json(raw_config) + self.assertEqual(config.agent.name, "nop") + self.assertEqual(config.source_trial.action, "regrade") + self.assertEqual(config.source_trial.path, self.source) + self.assertEqual(str(config.source_trial.trial_id), self.original["id"]) + self.assertEqual(config.verifier.env["OPENAI_API_KEY"], "${VERIFIER_OPENAI_API_KEY}") + self.assertTrue(config.environment.import_path.endswith(":PhaseScopedDocker")) + trial = config.trials_dir / config.trial_name + (trial / "verifier").mkdir(parents=True) + (trial / "verifier/judge-status.json").write_text('{"measurement_valid": true}') + (trial / "result.json").write_text(json.dumps({ + **self.original, "id": new_id, "trial_name": config.trial_name, "task_checksum": "current-hash", + "verifier_result": {"rewards": {"reward": 0}}, + })) + return 0 + + with patch.dict(os.environ, {"VERIFIER_OPENAI_API_KEY": "verifier-token"}), \ + patch.object(launcher, "run_command", side_effect=execute), \ + patch.object(runner, "backend", side_effect=[launcher, space]), \ + patch.object(runner, "doctor"), patch.object(runner, "asset_plan", return_value={}), \ + patch.object(space, "describe"), patch.object(space, "validate"): + code, output = self.invoke("--offline") + self.assertEqual(code, 0, output) + report = json.loads((self.output / "regrade.json").read_text()) + self.assertEqual(report["source_trial_id"], self.original["id"]) + self.assertEqual(report["trial_id"], new_id) + self.assertEqual(report["task_checksum"], "current-hash") + self.assertEqual(report["rewards"], {"reward": 0.0}) + self.assertTrue(report["measurement_valid"]) + self.assertEqual(original_source, (self.source / "result.json").read_bytes()) + + trial = Path(report["trial"]) + (trial / "verifier/judge-status.json").unlink() + result_summary(trial, report) + self.assertIsNone(report["measurement_valid"]) + for health in ({"measurement_valid": False}, + {"final_gate": {"evaluation_valid": True, "judge_succeeded": False}}): + (trial / "verifier/reward-details.json").write_text(json.dumps(health)) + result_summary(trial, report) + self.assertEqual(report["status"], "invalid-measurement") + self.assertFalse(report["measurement_valid"]) From cb4ebd7caaa2b2c417b623a0e14495a7ecc39323 Mon Sep 17 00:00:00 2001 From: hanhainebula <2512674094@qq.com> Date: Thu, 19 Nov 2026 14:38:19 +0800 Subject: [PATCH 4/5] Reject incomplete regrades and flag verifier timeouts --- docs/evaluation/cli.md | 17 ++++++++----- src/search_swe/regrade.py | 9 +++++++ tests/unit/test_regrade.py | 51 +++++++++++++++++++++++++++++++++++++- 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/docs/evaluation/cli.md b/docs/evaluation/cli.md index 016abc5..542e5e9 100644 --- a/docs/evaluation/cli.md +++ b/docs/evaluation/cli.md @@ -72,17 +72,22 @@ search-swe regrade /path/to/original-trial Keep the original `config.json`, `result.json`, `agent/`, and `artifacts/manifest.json` with its recorded artifact files. The preview checks that the current verifier's declared inputs are present, including the ATIF -trajectory for tasks that require it. Run with the task's verifier credentials -and network settings in `.env` or `--env-file`. The verifier uses the task's -usual asset preparation and CPU/GPU requirements. Use `--offline` for cached -assets, `--cpu-set` for a CPU pool, or `--gpu-device` for a specific GPU. +trajectory for tasks that require it. Regrade rejects +`large-scale-formula-retrieval` during preflight because its verifier needs +private agent-phase memory evidence outside the recorded trial. + +Run with the task's verifier credentials and network settings in `.env` +or `--env-file`. The verifier uses the task's usual asset preparation and +CPU/GPU requirements. Use `--offline` for cached assets, `--cpu-set` for a +CPU pool, or `--gpu-device` for a specific GPU. The CLI starts only the verifier and preserves the original trial. The new trial and its `regrade.json` summary are written under `jobs/regrades/` or a fresh `--output` directory. The summary records both trial IDs, the verifier checkout revision, and rewards. Measurement validity is recorded when the -verifier reports it explicitly. Review the new verifier logs before using -the result, including when the reward is zero. +verifier reports it explicitly. Verifier diagnostics that identify a failed +phase mark the measurement invalid and cause a nonzero exit status. Review the +new verifier logs before using the result, including when the reward is zero. ## Interactive setup diff --git a/src/search_swe/regrade.py b/src/search_swe/regrade.py index 48ed817..6cf412b 100644 --- a/src/search_swe/regrade.py +++ b/src/search_swe/regrade.py @@ -43,6 +43,11 @@ def recorded_inputs(source, task): if problem := check_task_regradable(task): raise ValueError(problem) + if task.name == "large-scale-formula-retrieval": + raise ValueError( + f"Regrade is unavailable for {task.name}: its verifier requires the original " + "agent's private memory evidence at /task-evidence/agent.json, outside the recorded trial." + ) config = TaskConfig.model_validate_toml((task / "task.toml").read_text()) convention = convention_source_for_os(config.environment.os) @@ -96,6 +101,10 @@ def measurement_valid(verifier): continue if not isinstance(data, dict): continue + guard = data.get("timeout_guard") + phase = guard.get("failed_phase") if isinstance(guard, dict) else None + if isinstance(phase, str) and phase.strip(): + return False for details in (data, data.get("final_gate")): if not isinstance(details, dict): continue diff --git a/tests/unit/test_regrade.py b/tests/unit/test_regrade.py index 085a8f8..6fa36fa 100644 --- a/tests/unit/test_regrade.py +++ b/tests/unit/test_regrade.py @@ -95,6 +95,44 @@ def test_source_symlinks_are_rejected_before_replay(self): self.assertEqual(code, 1, output) self.assertIn("Only regular files and directories", output) + def test_formula_requires_original_private_memory_evidence(self): + name = "large-scale-formula-retrieval" + formula = self.task.with_name(name) + self.task.rename(formula) + (formula / "task.toml").write_bytes((ROOT / "tasks" / name / "task.toml").read_bytes()) + (self.source / "result.json").write_text(json.dumps({ + **self.original, "task_name": f"search-swe/{name}", + })) + with patch("search_swe.runner.doctor", side_effect=AssertionError("started preflight")): + for flags in (("--dry-run",), ()): + with self.subTest(flags=flags): + code, output = self.invoke(*flags) + self.assertEqual(code, 1, output) + self.assertIn("/task-evidence/agent.json", output) + self.assertFalse(self.output.exists()) + + def test_timeout_guard_invalidates_a_numeric_reward(self): + new_trial = self.output / "trial" + (new_trial / "verifier").mkdir(parents=True) + (new_trial / "result.json").write_text(json.dumps({ + **self.original, "verifier_result": {"rewards": {"reward": 0}}, + })) + details = new_trial / "verifier/reward-details.json" + # Query Encoder includes an exit code. Older supervisors omit it. + for status in ({"exit_status": "124"}, {}): + with self.subTest(status=status): + details.write_text(json.dumps({"timeout_guard": { + "failed_phase": "primary_evaluation", + "failure_reason": "task-specific evaluation timed out", + "reward": 0.0, + **status, + }})) + report = {} + result_summary(new_trial, report) + self.assertEqual(report["rewards"], {"reward": 0.0}) + self.assertEqual(report["status"], "invalid-measurement") + self.assertIs(report["measurement_valid"], False) + def test_only_verifier_is_configured_and_zero_reward_is_preserved(self): from search_swe import runner launcher = runner.backend(self.checkout, "evaluation/run_task") @@ -106,6 +144,7 @@ def test_only_verifier_is_configured_and_zero_reward_is_preserved(self): task.write_text("services: {}\n") new_id = str(uuid4()) original_source = (self.source / "result.json").read_bytes() + timed_out = self.output.with_name("timed-out") def execute(command, env, cwd): if command[0] == sys.executable: @@ -126,6 +165,10 @@ def execute(command, env, cwd): trial = config.trials_dir / config.trial_name (trial / "verifier").mkdir(parents=True) (trial / "verifier/judge-status.json").write_text('{"measurement_valid": true}') + if config.trials_dir == timed_out: + (trial / "verifier/reward-details.json").write_text( + '{"timeout_guard": {"failed_phase": "primary_evaluation", "exit_status": "124"}}' + ) (trial / "result.json").write_text(json.dumps({ **self.original, "id": new_id, "trial_name": config.trial_name, "task_checksum": "current-hash", "verifier_result": {"rewards": {"reward": 0}}, @@ -134,11 +177,16 @@ def execute(command, env, cwd): with patch.dict(os.environ, {"VERIFIER_OPENAI_API_KEY": "verifier-token"}), \ patch.object(launcher, "run_command", side_effect=execute), \ - patch.object(runner, "backend", side_effect=[launcher, space]), \ + patch.object(runner, "backend", side_effect=[launcher, space, launcher, space]), \ patch.object(runner, "doctor"), patch.object(runner, "asset_plan", return_value={}), \ patch.object(space, "describe"), patch.object(space, "validate"): code, output = self.invoke("--offline") + timeout_code, timeout_output = self.invoke("--offline", "--output", str(timed_out)) self.assertEqual(code, 0, output) + self.assertEqual(timeout_code, 1, timeout_output) + timeout_report = json.loads((timed_out / "regrade.json").read_text()) + self.assertEqual(timeout_report["status"], "invalid-measurement") + self.assertIs(timeout_report["measurement_valid"], False) report = json.loads((self.output / "regrade.json").read_text()) self.assertEqual(report["source_trial_id"], self.original["id"]) self.assertEqual(report["trial_id"], new_id) @@ -150,6 +198,7 @@ def execute(command, env, cwd): trial = Path(report["trial"]) (trial / "verifier/judge-status.json").unlink() result_summary(trial, report) + self.assertEqual(report["status"], "completed") self.assertIsNone(report["measurement_valid"]) for health in ({"measurement_valid": False}, {"final_gate": {"evaluation_valid": True, "judge_succeeded": False}}): From 0220f339f940f0a8ebb729725f0128e913aa7cdb Mon Sep 17 00:00:00 2001 From: hanhainebula <2512674094@qq.com> Date: Thu, 19 Nov 2026 15:19:21 +0800 Subject: [PATCH 5/5] Flag failed answer judgments in regrade results --- docs/evaluation/cli.md | 6 +++--- src/search_swe/regrade.py | 14 ++++++++++--- tests/unit/test_regrade.py | 43 ++++++++++++++++++++++++++++++++------ 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/docs/evaluation/cli.md b/docs/evaluation/cli.md index 542e5e9..64a593a 100644 --- a/docs/evaluation/cli.md +++ b/docs/evaluation/cli.md @@ -85,9 +85,9 @@ The CLI starts only the verifier and preserves the original trial. The new trial and its `regrade.json` summary are written under `jobs/regrades/` or a fresh `--output` directory. The summary records both trial IDs, the verifier checkout revision, and rewards. Measurement validity is recorded when the -verifier reports it explicitly. Verifier diagnostics that identify a failed -phase mark the measurement invalid and cause a nonzero exit status. Review the -new verifier logs before using the result, including when the reward is zero. +verifier reports it explicitly. A failed verifier phase or a primary evaluation +invalidated by answer-judge errors causes a nonzero exit status. Review the new +verifier logs before using the result, including when the reward is zero. ## Interactive setup diff --git a/src/search_swe/regrade.py b/src/search_swe/regrade.py index 6cf412b..4c8b580 100644 --- a/src/search_swe/regrade.py +++ b/src/search_swe/regrade.py @@ -92,15 +92,23 @@ def walk_error(error): + ", ".join(missing)) -def measurement_valid(verifier): +def measurement_valid(verifier, task_name): + evaluation = f"{task_name.rsplit('/', 1)[-1]}-eval/evaluation.json" health = [] - for name in ("reward-details.json", "judge-status.json"): + for name in ("reward-details.json", "judge-status.json", evaluation): try: data = json.loads((verifier / name).read_text()) except (OSError, ValueError): continue if not isinstance(data, dict): continue + if name == evaluation: + # The primary grader records failed answer judgments separately + # from the final gate's trajectory judge status. + errors = data.get("judge_error_count") + if data.get("valid") is False and type(errors) is int and errors > 0: + return False + continue guard = data.get("timeout_guard") phase = guard.get("failed_phase") if isinstance(guard, dict) else None if isinstance(phase, str) and phase.strip(): @@ -129,7 +137,7 @@ def result_summary(trial, report): report.update(measurement_valid=False, status="verifier-error", exception_type=result.exception_info.exception_type) elif rewards: - valid = measurement_valid(trial / "verifier") + valid = measurement_valid(trial / "verifier", result.task_name) report.update(measurement_valid=valid, status="invalid-measurement" if valid is False else "completed") diff --git a/tests/unit/test_regrade.py b/tests/unit/test_regrade.py index 6fa36fa..dedf9db 100644 --- a/tests/unit/test_regrade.py +++ b/tests/unit/test_regrade.py @@ -133,6 +133,31 @@ def test_timeout_guard_invalidates_a_numeric_reward(self): self.assertEqual(report["status"], "invalid-measurement") self.assertIs(report["measurement_valid"], False) + def test_answer_judge_failure_is_distinct_from_a_completed_zero(self): + trial = self.output / "trial" + verifier = trial / "verifier" + for name in ("scientific-paper-qa", "long-pdf-evidence", "long-pdf-evidence-vectorless"): + evaluation = verifier / f"{name}-eval/evaluation.json" + evaluation.parent.mkdir(parents=True) + (trial / "result.json").write_text(json.dumps({ + **self.original, "task_name": f"search-swe/{name}", + "verifier_result": {"rewards": {"reward": 0}}, + })) + for judge_errors in (1, 0, None): + with self.subTest(name=name, judge_errors=judge_errors): + metrics = {"valid": judge_errors == 0} + if judge_errors is not None: + metrics["judge_error_count"] = judge_errors + evaluation.write_text(json.dumps(metrics)) + (verifier / "reward-details.json").write_text(json.dumps({ + "final_gate": {"evaluation_valid": metrics["valid"], "judge_succeeded": True}, + })) + report = {} + result_summary(trial, report) + self.assertEqual(report["rewards"], {"reward": 0.0}) + self.assertIs(report["measurement_valid"], False if judge_errors == 1 else None) + self.assertEqual(report["status"], "invalid-measurement" if judge_errors == 1 else "completed") + def test_only_verifier_is_configured_and_zero_reward_is_preserved(self): from search_swe import runner launcher = runner.backend(self.checkout, "evaluation/run_task") @@ -145,6 +170,7 @@ def test_only_verifier_is_configured_and_zero_reward_is_preserved(self): new_id = str(uuid4()) original_source = (self.source / "result.json").read_bytes() timed_out = self.output.with_name("timed-out") + judge_failed = self.output.with_name("judge-failed") def execute(command, env, cwd): if command[0] == sys.executable: @@ -169,6 +195,10 @@ def execute(command, env, cwd): (trial / "verifier/reward-details.json").write_text( '{"timeout_guard": {"failed_phase": "primary_evaluation", "exit_status": "124"}}' ) + if config.trials_dir == judge_failed: + evaluation = trial / "verifier/fixture-regrade-eval/evaluation.json" + evaluation.parent.mkdir() + evaluation.write_text('{"valid": false, "judge_error_count": 1}') (trial / "result.json").write_text(json.dumps({ **self.original, "id": new_id, "trial_name": config.trial_name, "task_checksum": "current-hash", "verifier_result": {"rewards": {"reward": 0}}, @@ -177,16 +207,17 @@ def execute(command, env, cwd): with patch.dict(os.environ, {"VERIFIER_OPENAI_API_KEY": "verifier-token"}), \ patch.object(launcher, "run_command", side_effect=execute), \ - patch.object(runner, "backend", side_effect=[launcher, space, launcher, space]), \ + patch.object(runner, "backend", side_effect=[launcher, space] * 3), \ patch.object(runner, "doctor"), patch.object(runner, "asset_plan", return_value={}), \ patch.object(space, "describe"), patch.object(space, "validate"): code, output = self.invoke("--offline") - timeout_code, timeout_output = self.invoke("--offline", "--output", str(timed_out)) + for invalid in (timed_out, judge_failed): + status, message = self.invoke("--offline", "--output", str(invalid)) + self.assertEqual(status, 1, message) + invalid_report = json.loads((invalid / "regrade.json").read_text()) + self.assertEqual(invalid_report["status"], "invalid-measurement") + self.assertIs(invalid_report["measurement_valid"], False) self.assertEqual(code, 0, output) - self.assertEqual(timeout_code, 1, timeout_output) - timeout_report = json.loads((timed_out / "regrade.json").read_text()) - self.assertEqual(timeout_report["status"], "invalid-measurement") - self.assertIs(timeout_report["measurement_valid"], False) report = json.loads((self.output / "regrade.json").read_text()) self.assertEqual(report["source_trial_id"], self.original["id"]) self.assertEqual(report["trial_id"], new_id)