From b08b5d89a6a4c51233137c0f0ceb3bd901724865 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Thu, 10 Sep 2026 07:40:23 +0000 Subject: [PATCH 01/20] feat: add CI harness smoke test pipeline with automated result verification --- .ci/harness.cloudbuild.yaml | 69 +++++++++++++++ .ci/harness_smoke.evalset.json | 31 +++++++ .ci/run_configs/agy_cli.yaml | 29 ++++++ .ci/run_configs/claude_code.yaml | 29 ++++++ .ci/run_configs/codex_cli.yaml | 29 ++++++ .ci/run_configs/gemini_cli.yaml | 29 ++++++ .ci/verify_harness.py | 147 +++++++++++++++++++++++++++++++ 7 files changed, 363 insertions(+) create mode 100644 .ci/harness.cloudbuild.yaml create mode 100644 .ci/harness_smoke.evalset.json create mode 100644 .ci/run_configs/agy_cli.yaml create mode 100644 .ci/run_configs/claude_code.yaml create mode 100644 .ci/run_configs/codex_cli.yaml create mode 100644 .ci/run_configs/gemini_cli.yaml create mode 100644 .ci/verify_harness.py diff --git a/.ci/harness.cloudbuild.yaml b/.ci/harness.cloudbuild.yaml new file mode 100644 index 00000000..37b20ac6 --- /dev/null +++ b/.ci/harness.cloudbuild.yaml @@ -0,0 +1,69 @@ +steps: + - id: build-image + name: 'gcr.io/cloud-builders/docker' + args: ['build', '-t', 'evalbench-harness-ci', '-f', 'evalbench_service/Dockerfile', '.'] + + # The four harness steps share a waitFor, so Cloud Build runs them + # concurrently. allowFailure keeps a single broken harness from cancelling + # the others -- verify-harnesses is the gate, so one run reports on all four. + - id: agy-cli + name: 'evalbench-harness-ci' + dir: '/evalbench' + waitFor: ['build-image'] + allowFailure: true + args: ['uv', 'run', '--no-sync', 'evalbench/evalbench.py', + '--experiment_config=.ci/run_configs/agy_cli.yaml'] + env: &harness_env + - 'EVAL_GCP_PROJECT_ID=${_EVAL_PROJECT}' + - 'CI_CSQL_INSTANCE=${_CI_CSQL_INSTANCE}' + - 'UV_CACHE_DIR=/tmp/uv-cache' + volumes: &results_volume + - name: 'eval_results' + path: '/evalbench/results' + + - id: claude-code + name: 'evalbench-harness-ci' + dir: '/evalbench' + waitFor: ['build-image'] + allowFailure: true + args: ['uv', 'run', '--no-sync', 'evalbench/evalbench.py', + '--experiment_config=.ci/run_configs/claude_code.yaml'] + env: *harness_env + volumes: *results_volume + + - id: codex-cli + name: 'evalbench-harness-ci' + dir: '/evalbench' + waitFor: ['build-image'] + allowFailure: true + args: ['uv', 'run', '--no-sync', 'evalbench/evalbench.py', + '--experiment_config=.ci/run_configs/codex_cli.yaml'] + env: *harness_env + volumes: *results_volume + + - id: gemini-cli + name: 'evalbench-harness-ci' + dir: '/evalbench' + waitFor: ['build-image'] + allowFailure: true + args: ['uv', 'run', '--no-sync', 'evalbench/evalbench.py', + '--experiment_config=.ci/run_configs/gemini_cli.yaml'] + env: *harness_env + volumes: *results_volume + + - id: verify-harnesses + name: 'evalbench-harness-ci' + dir: '/evalbench' + waitFor: ['agy-cli', 'claude-code', 'codex-cli', 'gemini-cli'] + args: ['uv', 'run', '--no-sync', 'python3', '.ci/verify_harness.py'] + volumes: *results_volume + +# _EVAL_PROJECT and _CI_CSQL_INSTANCE are deliberately left undefined so the +# target project and instance stay out of this public repo. The trigger +# supplies both; a build that omits either fails validation before any step +# runs. + +timeout: '3600s' + +options: + logging: CLOUD_LOGGING_ONLY diff --git a/.ci/harness_smoke.evalset.json b/.ci/harness_smoke.evalset.json new file mode 100644 index 00000000..9426523a --- /dev/null +++ b/.ci/harness_smoke.evalset.json @@ -0,0 +1,31 @@ +{ + "scenarios": [ + { + "id": "ci-csql-list-and-get", + "starting_prompt": "list all instances in project ${EVAL_GCP_PROJECT_ID}", + "conversation_plan": "Ask the agent to list instances in project ${EVAL_GCP_PROJECT_ID}. Once all instances are listed if ${CI_CSQL_INSTANCE} exist get its state and validate its RUNNABLE", + "expected_trajectory": [ + "cloud-sql__list_instances", + "cloud-sql__get_instance" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "${EVAL_GCP_PROJECT_ID}" + }, + "kind": "tools", + "max_turns": 3 + }, + { + "id": "ci-csql-get-instance-not-found", + "starting_prompt": "Get the details for the Cloud SQL instance named 'ci-missing-instance-do-not-create' in project '${EVAL_GCP_PROJECT_ID}'.", + "conversation_plan": "The user wants details for an instance that does not exist. The agent should call get_instance once, receive a not-found error, and report to the user that the instance could not be found. The user accepts that answer and ends the conversation. The user must not ask the agent to list instances, search for similar names, or otherwise recover.", + "expected_trajectory": [ + "cloud-sql__get_instance" + ], + "env": { + "GOOGLE_CLOUD_PROJECT": "${EVAL_GCP_PROJECT_ID}" + }, + "kind": "tools", + "max_turns": 2 + } + ] +} diff --git a/.ci/run_configs/agy_cli.yaml b/.ci/run_configs/agy_cli.yaml new file mode 100644 index 00000000..3819061d --- /dev/null +++ b/.ci/run_configs/agy_cli.yaml @@ -0,0 +1,29 @@ +dataset_config: .ci/harness_smoke.evalset.json +dataset_format: agent-format + +orchestrator: agent +model_config: datasets/model_configs/agy_cli_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + +# Only trajectory_matcher is thresholded. The rest are liveness-checked by +# .ci/verify_harness.py: they must run against real harness output without +# erroring. Judge verdicts are never gated on -- a wobbling gate gets ignored. +scorers: + trajectory_matcher: {} + turn_count: {} + agent_steps: {} + end_to_end_latency: {} + tool_call_latency: {} + token_consumption: {} + tokens_processed: {} + effective_billed_tokens: {} + goal_completion: + model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + behavioral_metrics: + model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + parameter_analysis: + model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + +reporting: + csv: + output_directory: 'results/ci/agy_cli' diff --git a/.ci/run_configs/claude_code.yaml b/.ci/run_configs/claude_code.yaml new file mode 100644 index 00000000..ebb2d5c6 --- /dev/null +++ b/.ci/run_configs/claude_code.yaml @@ -0,0 +1,29 @@ +dataset_config: .ci/harness_smoke.evalset.json +dataset_format: agent-format + +orchestrator: agent +model_config: datasets/model_configs/claude_code_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + +# Only trajectory_matcher is thresholded. The rest are liveness-checked by +# .ci/verify_harness.py: they must run against real harness output without +# erroring. Judge verdicts are never gated on -- a wobbling gate gets ignored. +scorers: + trajectory_matcher: {} + turn_count: {} + agent_steps: {} + end_to_end_latency: {} + tool_call_latency: {} + token_consumption: {} + tokens_processed: {} + effective_billed_tokens: {} + goal_completion: + model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + behavioral_metrics: + model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + parameter_analysis: + model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + +reporting: + csv: + output_directory: 'results/ci/claude_code' diff --git a/.ci/run_configs/codex_cli.yaml b/.ci/run_configs/codex_cli.yaml new file mode 100644 index 00000000..757d9ebf --- /dev/null +++ b/.ci/run_configs/codex_cli.yaml @@ -0,0 +1,29 @@ +dataset_config: .ci/harness_smoke.evalset.json +dataset_format: agent-format + +orchestrator: agent +model_config: datasets/model_configs/codex_cli_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + +# Only trajectory_matcher is thresholded. The rest are liveness-checked by +# .ci/verify_harness.py: they must run against real harness output without +# erroring. Judge verdicts are never gated on -- a wobbling gate gets ignored. +scorers: + trajectory_matcher: {} + turn_count: {} + agent_steps: {} + end_to_end_latency: {} + tool_call_latency: {} + token_consumption: {} + tokens_processed: {} + effective_billed_tokens: {} + goal_completion: + model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + behavioral_metrics: + model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + parameter_analysis: + model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + +reporting: + csv: + output_directory: 'results/ci/codex_cli' diff --git a/.ci/run_configs/gemini_cli.yaml b/.ci/run_configs/gemini_cli.yaml new file mode 100644 index 00000000..46fbcfff --- /dev/null +++ b/.ci/run_configs/gemini_cli.yaml @@ -0,0 +1,29 @@ +dataset_config: .ci/harness_smoke.evalset.json +dataset_format: agent-format + +orchestrator: agent +model_config: datasets/model_configs/gemini_cli_model.yaml +simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + +# Only trajectory_matcher is thresholded. The rest are liveness-checked by +# .ci/verify_harness.py: they must run against real harness output without +# erroring. Judge verdicts are never gated on -- a wobbling gate gets ignored. +scorers: + trajectory_matcher: {} + turn_count: {} + agent_steps: {} + end_to_end_latency: {} + tool_call_latency: {} + token_consumption: {} + tokens_processed: {} + effective_billed_tokens: {} + goal_completion: + model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + behavioral_metrics: + model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + parameter_analysis: + model_config: datasets/model_configs/gemini_3.1_pro_model.yaml + +reporting: + csv: + output_directory: 'results/ci/gemini_cli' diff --git a/.ci/verify_harness.py b/.ci/verify_harness.py new file mode 100644 index 00000000..6433cbca --- /dev/null +++ b/.ci/verify_harness.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Gates the harness smoke build on two tiers of check. + +evalbench.eval() exits 0 whenever a run completes, including when the agent +made no tool calls at all, so the exit code alone cannot gate CI. + +Tier 1 (threshold): trajectory_matcher must hit THRESHOLDS -- the agent really +reached the expected tools. + +Tier 2 (non-zero): telemetry scorers must report more than 0. They swallow +parse failures and return 0.0 with an explanation rather than raising, so a +plain liveness check passes even when a CLI renames a token field -- the exact +drift this build exists to catch. Every scenario makes at least one MCP call, +so 0 tokens or 0 latency can only mean the scorer failed to read the output. + +Tier 3 (liveness): every remaining scorer must emit a row per scenario, with no +comparison_error and a numeric score. This covers the LLM judges without ever +gating on their verdict, which would make the build flaky. +""" +import csv +import json +import os +import sys + +import yaml + +HARNESSES = ["agy_cli", "claude_code", "codex_cli", "gemini_cli"] +RUN_CONFIG_DIR = ".ci/run_configs" +EVALSET = ".ci/harness_smoke.evalset.json" +THRESHOLDS = {"trajectory_matcher": 100.0} +POSITIVE = { + "turn_count", + "agent_steps", + "end_to_end_latency", + "tool_call_latency", + "token_consumption", + "tokens_processed", + "effective_billed_tokens", +} + + +def expected_scenario_ids(): + with open(EVALSET) as f: + return sorted(s["id"] for s in json.load(f)["scenarios"]) + + +def run_config(harness): + with open(os.path.join(RUN_CONFIG_DIR, f"{harness}.yaml")) as f: + return yaml.safe_load(f) + + +def latest_job_dir(output_dir): + if not os.path.isdir(output_dir): + return None + jobs = [os.path.join(output_dir, d) for d in os.listdir(output_dir) + if os.path.isdir(os.path.join(output_dir, d))] + return max(jobs, key=os.path.getmtime) if jobs else None + + +def load_rows(job_dir): + """Returns {(comparator, scenario_id): (score_or_None, error)}.""" + path = os.path.join(job_dir, "scores.csv") + if not os.path.exists(path): + return None + rows = {} + with open(path, newline="") as f: + for row in csv.DictReader(f): + try: + score = float(row["score"]) + except (KeyError, TypeError, ValueError): + score = None + rows[(row.get("comparator"), row.get("id"))] = ( + score, (row.get("comparison_error") or "").strip() + ) + return rows + + +def check(harness, scenario_ids): + config = run_config(harness) + scorers = sorted(config.get("scorers") or {}) + output_dir = config["reporting"]["csv"]["output_directory"] + + job_dir = latest_job_dir(output_dir) + if job_dir is None: + return [f"no run output under {output_dir}"], 0, scorers + + rows = load_rows(job_dir) + if rows is None: + return [f"no scores.csv in {job_dir}"], 0, scorers + + problems = [] + checked = 0 + for scorer in scorers: + for sid in scenario_ids: + entry = rows.get((scorer, sid)) + if entry is None: + problems.append(f"{scorer}: no row for {sid}") + continue + score, error = entry + checked += 1 + if error: + problems.append(f"{scorer}: errored on {sid} -- {error[:120]}") + elif score is None: + problems.append(f"{scorer}: non-numeric score for {sid}") + elif scorer in THRESHOLDS and score < THRESHOLDS[scorer]: + problems.append( + f"{scorer}: {sid} scored {score:.1f}, " + f"need >= {THRESHOLDS[scorer]:.0f}" + ) + elif scorer in POSITIVE and score <= 0: + problems.append( + f"{scorer}: {sid} reported 0 -- scorer could not read " + f"the harness output" + ) + return problems, checked, scorers + + +def main(): + scenario_ids = expected_scenario_ids() + gated = ", ".join(f"{k} >= {v:.0f}" for k, v in THRESHOLDS.items()) + print(f"Scenarios: {len(scenario_ids)} | Harnesses: {len(HARNESSES)}") + print(f"Thresholded: {gated}") + print(f"Must be > 0: {', '.join(sorted(POSITIVE))}") + print("All other scorers are liveness-checked (ran, no error, " + "numeric score)\n") + + failed = [] + for harness in HARNESSES: + problems, checked, scorers = check(harness, scenario_ids) + if problems: + failed.append(harness) + print(f" [FAIL] {harness}") + for p in problems: + print(f" {p}") + else: + print(f" [PASS] {harness:<12} " + f"{checked} checks across {len(scorers)} scorers") + + if failed: + print(f"\nFAILED: {', '.join(failed)}") + return 1 + print("\nAll harnesses passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From f9f17b4229e14ddece635fdc8703a51dc6aa8f45 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 11 Sep 2026 05:52:05 +0000 Subject: [PATCH 02/20] feat: centralize model configurations in .ci/ directory and update model versions and environment settings --- .ci/harness.cloudbuild.yaml | 1 + .ci/model_configs/claude_code_model.yaml | 24 ++++++++++++++++++++ .ci/model_configs/codex_cli_model.yaml | 23 +++++++++++++++++++ .ci/model_configs/gemini_cli_model.yaml | 18 +++++++++++++++ .ci/run_configs/claude_code.yaml | 2 +- .ci/run_configs/codex_cli.yaml | 2 +- .ci/run_configs/gemini_cli.yaml | 2 +- datasets/model_configs/gemini_cli_model.yaml | 3 ++- 8 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 .ci/model_configs/claude_code_model.yaml create mode 100644 .ci/model_configs/codex_cli_model.yaml create mode 100644 .ci/model_configs/gemini_cli_model.yaml diff --git a/.ci/harness.cloudbuild.yaml b/.ci/harness.cloudbuild.yaml index 37b20ac6..fa6a2c50 100644 --- a/.ci/harness.cloudbuild.yaml +++ b/.ci/harness.cloudbuild.yaml @@ -15,6 +15,7 @@ steps: '--experiment_config=.ci/run_configs/agy_cli.yaml'] env: &harness_env - 'EVAL_GCP_PROJECT_ID=${_EVAL_PROJECT}' + - 'EVAL_GCP_PROJECT_REGION=us-central1' - 'CI_CSQL_INSTANCE=${_CI_CSQL_INSTANCE}' - 'UV_CACHE_DIR=/tmp/uv-cache' volumes: &results_volume diff --git a/.ci/model_configs/claude_code_model.yaml b/.ci/model_configs/claude_code_model.yaml new file mode 100644 index 00000000..ce39080a --- /dev/null +++ b/.ci/model_configs/claude_code_model.yaml @@ -0,0 +1,24 @@ +claude_code_version: "@anthropic-ai/claude-code@2.1.85" +generator: claude_code +model: "claude-opus-4-6" + +use_vertex: true +vertex_project_id: !ENV ${EVAL_GCP_PROJECT_ID} +# Claude on Vertex is only served from a few regions; us-east5 is the one +# Model Garden enables for these model ids. +vertex_region: "us-east5" + +env: + GOOGLE_CLOUD_PROJECT: !ENV ${EVAL_GCP_PROJECT_ID} + GOOGLE_CLOUD_LOCATION: "us-central1" + +setup: + mcp_servers: + "cloud-sql": + httpUrl: "https://sqladmin.googleapis.com/mcp" + authProviderType: google_credentials + oauth: + scopes: + - https://www.googleapis.com/auth/cloud-platform + headers: + X-Goog-User-Project: !ENV ${EVAL_GCP_PROJECT_ID} diff --git a/.ci/model_configs/codex_cli_model.yaml b/.ci/model_configs/codex_cli_model.yaml new file mode 100644 index 00000000..b2316367 --- /dev/null +++ b/.ci/model_configs/codex_cli_model.yaml @@ -0,0 +1,23 @@ +codex_cli_version: "@openai/codex@latest" +generator: codex_cli +model: "gpt-5.5" + +openai_api_key_secret: "projects/393137573/secrets/OPENAI_API_KEY/versions/1" + +# Codex reports token counts but no cost; these rates turn them into cost_usd. +pricing: + input_per_million_usd: 1.25 + cached_input_per_million_usd: 0.125 + output_per_million_usd: 10.0 + +env: + GOOGLE_CLOUD_PROJECT: !ENV ${EVAL_GCP_PROJECT_ID} + GOOGLE_CLOUD_LOCATION: "us-central1" + +setup: + mcp_servers: + "cloud-sql": + httpUrl: "https://sqladmin.googleapis.com/mcp" + authProviderType: google_credentials + headers: + X-Goog-User-Project: !ENV ${EVAL_GCP_PROJECT_ID} diff --git a/.ci/model_configs/gemini_cli_model.yaml b/.ci/model_configs/gemini_cli_model.yaml new file mode 100644 index 00000000..104749e6 --- /dev/null +++ b/.ci/model_configs/gemini_cli_model.yaml @@ -0,0 +1,18 @@ +gemini_cli_version: "@google/gemini-cli@0.25.1" +generator: gemini_cli + +env: + GOOGLE_CLOUD_PROJECT: !ENV ${EVAL_GCP_PROJECT_ID} + GOOGLE_CLOUD_LOCATION: "us-central1" + GOOGLE_GENAI_USE_VERTEXAI: "true" + +setup: + mcp_servers: + "cloud-sql": + httpUrl: "https://sqladmin.googleapis.com/mcp" + authProviderType: google_credentials + oauth: + scopes: + - https://www.googleapis.com/auth/cloud-platform + headers: + X-Goog-User-Project: !ENV ${EVAL_GCP_PROJECT_ID} diff --git a/.ci/run_configs/claude_code.yaml b/.ci/run_configs/claude_code.yaml index ebb2d5c6..ab0f9b9e 100644 --- a/.ci/run_configs/claude_code.yaml +++ b/.ci/run_configs/claude_code.yaml @@ -2,7 +2,7 @@ dataset_config: .ci/harness_smoke.evalset.json dataset_format: agent-format orchestrator: agent -model_config: datasets/model_configs/claude_code_model.yaml +model_config: .ci/model_configs/claude_code_model.yaml simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml # Only trajectory_matcher is thresholded. The rest are liveness-checked by diff --git a/.ci/run_configs/codex_cli.yaml b/.ci/run_configs/codex_cli.yaml index 757d9ebf..49bf7ae4 100644 --- a/.ci/run_configs/codex_cli.yaml +++ b/.ci/run_configs/codex_cli.yaml @@ -2,7 +2,7 @@ dataset_config: .ci/harness_smoke.evalset.json dataset_format: agent-format orchestrator: agent -model_config: datasets/model_configs/codex_cli_model.yaml +model_config: .ci/model_configs/codex_cli_model.yaml simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml # Only trajectory_matcher is thresholded. The rest are liveness-checked by diff --git a/.ci/run_configs/gemini_cli.yaml b/.ci/run_configs/gemini_cli.yaml index 46fbcfff..ae59b7d7 100644 --- a/.ci/run_configs/gemini_cli.yaml +++ b/.ci/run_configs/gemini_cli.yaml @@ -2,7 +2,7 @@ dataset_config: .ci/harness_smoke.evalset.json dataset_format: agent-format orchestrator: agent -model_config: datasets/model_configs/gemini_cli_model.yaml +model_config: .ci/model_configs/gemini_cli_model.yaml simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml # Only trajectory_matcher is thresholded. The rest are liveness-checked by diff --git a/datasets/model_configs/gemini_cli_model.yaml b/datasets/model_configs/gemini_cli_model.yaml index 5d1ac0e6..e322395b 100644 --- a/datasets/model_configs/gemini_cli_model.yaml +++ b/datasets/model_configs/gemini_cli_model.yaml @@ -1,9 +1,10 @@ -gemini_cli_version: "@google/gemini-cli@0.25.1" +gemini_cli_version: "@google/gemini-cli@0.55.1" generator: gemini_cli env: GOOGLE_CLOUD_PROJECT: "astana-evaluation" GOOGLE_CLOUD_LOCATION: "us-central1" GOOGLE_GENAI_USE_VERTEXAI: "true" + GEMINI_CLI_TRUST_WORKSPACE: "true" setup: mcp_servers: "cloud-sql": From f8082d5e48dc766b12cd097d2d8351145e9c6bf2 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 11 Sep 2026 06:18:03 +0000 Subject: [PATCH 03/20] chore: update gemini-cli version, improve CI build caching, and remove redundant config comments --- .ci/harness.cloudbuild.yaml | 32 ++++++++++++++------ .ci/model_configs/claude_code_model.yaml | 3 +- .ci/model_configs/gemini_cli_model.yaml | 3 +- .ci/run_configs/agy_cli.yaml | 3 -- .ci/run_configs/claude_code.yaml | 3 -- .ci/run_configs/codex_cli.yaml | 3 -- .ci/run_configs/gemini_cli.yaml | 3 -- datasets/model_configs/gemini_cli_model.yaml | 3 +- 8 files changed, 27 insertions(+), 26 deletions(-) diff --git a/.ci/harness.cloudbuild.yaml b/.ci/harness.cloudbuild.yaml index fa6a2c50..6a9745a2 100644 --- a/.ci/harness.cloudbuild.yaml +++ b/.ci/harness.cloudbuild.yaml @@ -1,11 +1,24 @@ +# _EVAL_PROJECT and _CI_CSQL_INSTANCE are left undefined so the target project +# and instance stay out of this public repo; the trigger supplies both. + steps: + # Primes the layer cache. The pull fails on the first build, before the tag exists. + - id: pull-cache + name: 'gcr.io/cloud-builders/docker' + entrypoint: 'bash' + args: ['-c', 'docker pull us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache || exit 0'] + - id: build-image name: 'gcr.io/cloud-builders/docker' - args: ['build', '-t', 'evalbench-harness-ci', '-f', 'evalbench_service/Dockerfile', '.'] + waitFor: ['pull-cache'] + args: ['build', + '--cache-from', 'us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache', + '-t', 'evalbench-harness-ci', + '-t', 'us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache', + '-f', 'evalbench_service/Dockerfile', '.'] - # The four harness steps share a waitFor, so Cloud Build runs them - # concurrently. allowFailure keeps a single broken harness from cancelling - # the others -- verify-harnesses is the gate, so one run reports on all four. + # allowFailure on all four: verify-harnesses is the gate, so one broken + # harness must not cancel the others before they report. - id: agy-cli name: 'evalbench-harness-ci' dir: '/evalbench' @@ -59,12 +72,13 @@ steps: args: ['uv', 'run', '--no-sync', 'python3', '.ci/verify_harness.py'] volumes: *results_volume -# _EVAL_PROJECT and _CI_CSQL_INSTANCE are deliberately left undefined so the -# target project and instance stay out of this public repo. The trigger -# supplies both; a build that omits either fails validation before any step -# runs. - timeout: '3600s' +# Republished every build so the next one starts warm. +images: + - 'us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache' + options: logging: CLOUD_LOGGING_ONLY + # Sized for the four concurrent harness steps, not for the build. + machineType: 'E2_HIGHCPU_8' diff --git a/.ci/model_configs/claude_code_model.yaml b/.ci/model_configs/claude_code_model.yaml index ce39080a..9125871a 100644 --- a/.ci/model_configs/claude_code_model.yaml +++ b/.ci/model_configs/claude_code_model.yaml @@ -4,8 +4,7 @@ model: "claude-opus-4-6" use_vertex: true vertex_project_id: !ENV ${EVAL_GCP_PROJECT_ID} -# Claude on Vertex is only served from a few regions; us-east5 is the one -# Model Garden enables for these model ids. +# Not us-central1: Vertex serves Claude from only a few regions. vertex_region: "us-east5" env: diff --git a/.ci/model_configs/gemini_cli_model.yaml b/.ci/model_configs/gemini_cli_model.yaml index 104749e6..3231d03e 100644 --- a/.ci/model_configs/gemini_cli_model.yaml +++ b/.ci/model_configs/gemini_cli_model.yaml @@ -1,10 +1,11 @@ -gemini_cli_version: "@google/gemini-cli@0.25.1" +gemini_cli_version: "@google/gemini-cli@0.55.1" generator: gemini_cli env: GOOGLE_CLOUD_PROJECT: !ENV ${EVAL_GCP_PROJECT_ID} GOOGLE_CLOUD_LOCATION: "us-central1" GOOGLE_GENAI_USE_VERTEXAI: "true" + GEMINI_CLI_TRUST_WORKSPACE: "true" setup: mcp_servers: diff --git a/.ci/run_configs/agy_cli.yaml b/.ci/run_configs/agy_cli.yaml index 3819061d..9af79f52 100644 --- a/.ci/run_configs/agy_cli.yaml +++ b/.ci/run_configs/agy_cli.yaml @@ -5,9 +5,6 @@ orchestrator: agent model_config: datasets/model_configs/agy_cli_model.yaml simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml -# Only trajectory_matcher is thresholded. The rest are liveness-checked by -# .ci/verify_harness.py: they must run against real harness output without -# erroring. Judge verdicts are never gated on -- a wobbling gate gets ignored. scorers: trajectory_matcher: {} turn_count: {} diff --git a/.ci/run_configs/claude_code.yaml b/.ci/run_configs/claude_code.yaml index ab0f9b9e..8b4f3636 100644 --- a/.ci/run_configs/claude_code.yaml +++ b/.ci/run_configs/claude_code.yaml @@ -5,9 +5,6 @@ orchestrator: agent model_config: .ci/model_configs/claude_code_model.yaml simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml -# Only trajectory_matcher is thresholded. The rest are liveness-checked by -# .ci/verify_harness.py: they must run against real harness output without -# erroring. Judge verdicts are never gated on -- a wobbling gate gets ignored. scorers: trajectory_matcher: {} turn_count: {} diff --git a/.ci/run_configs/codex_cli.yaml b/.ci/run_configs/codex_cli.yaml index 49bf7ae4..a4a5f389 100644 --- a/.ci/run_configs/codex_cli.yaml +++ b/.ci/run_configs/codex_cli.yaml @@ -5,9 +5,6 @@ orchestrator: agent model_config: .ci/model_configs/codex_cli_model.yaml simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml -# Only trajectory_matcher is thresholded. The rest are liveness-checked by -# .ci/verify_harness.py: they must run against real harness output without -# erroring. Judge verdicts are never gated on -- a wobbling gate gets ignored. scorers: trajectory_matcher: {} turn_count: {} diff --git a/.ci/run_configs/gemini_cli.yaml b/.ci/run_configs/gemini_cli.yaml index ae59b7d7..e6373fdc 100644 --- a/.ci/run_configs/gemini_cli.yaml +++ b/.ci/run_configs/gemini_cli.yaml @@ -5,9 +5,6 @@ orchestrator: agent model_config: .ci/model_configs/gemini_cli_model.yaml simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml -# Only trajectory_matcher is thresholded. The rest are liveness-checked by -# .ci/verify_harness.py: they must run against real harness output without -# erroring. Judge verdicts are never gated on -- a wobbling gate gets ignored. scorers: trajectory_matcher: {} turn_count: {} diff --git a/datasets/model_configs/gemini_cli_model.yaml b/datasets/model_configs/gemini_cli_model.yaml index e322395b..5d1ac0e6 100644 --- a/datasets/model_configs/gemini_cli_model.yaml +++ b/datasets/model_configs/gemini_cli_model.yaml @@ -1,10 +1,9 @@ -gemini_cli_version: "@google/gemini-cli@0.55.1" +gemini_cli_version: "@google/gemini-cli@0.25.1" generator: gemini_cli env: GOOGLE_CLOUD_PROJECT: "astana-evaluation" GOOGLE_CLOUD_LOCATION: "us-central1" GOOGLE_GENAI_USE_VERTEXAI: "true" - GEMINI_CLI_TRUST_WORKSPACE: "true" setup: mcp_servers: "cloud-sql": From 9f478d0c93106898b19142f50983d4fc644cfa78 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 11 Sep 2026 06:26:41 +0000 Subject: [PATCH 04/20] chore: update EVAL_GCP_PROJECT_REGION to global in cloudbuild configuration --- .ci/harness.cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/harness.cloudbuild.yaml b/.ci/harness.cloudbuild.yaml index 6a9745a2..5e1de6bc 100644 --- a/.ci/harness.cloudbuild.yaml +++ b/.ci/harness.cloudbuild.yaml @@ -28,7 +28,7 @@ steps: '--experiment_config=.ci/run_configs/agy_cli.yaml'] env: &harness_env - 'EVAL_GCP_PROJECT_ID=${_EVAL_PROJECT}' - - 'EVAL_GCP_PROJECT_REGION=us-central1' + - 'EVAL_GCP_PROJECT_REGION=global' - 'CI_CSQL_INSTANCE=${_CI_CSQL_INSTANCE}' - 'UV_CACHE_DIR=/tmp/uv-cache' volumes: &results_volume From 8dbd9763e069238c899caaad60ba70918c35e19c Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 11 Sep 2026 07:02:31 +0000 Subject: [PATCH 05/20] chore: parameterize secret project ID and update Gemini cloud location to global --- .ci/model_configs/codex_cli_model.yaml | 2 +- .ci/model_configs/gemini_cli_model.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/model_configs/codex_cli_model.yaml b/.ci/model_configs/codex_cli_model.yaml index b2316367..ed7dff4d 100644 --- a/.ci/model_configs/codex_cli_model.yaml +++ b/.ci/model_configs/codex_cli_model.yaml @@ -2,7 +2,7 @@ codex_cli_version: "@openai/codex@latest" generator: codex_cli model: "gpt-5.5" -openai_api_key_secret: "projects/393137573/secrets/OPENAI_API_KEY/versions/1" +openai_api_key_secret: !ENV projects/${EVAL_GCP_PROJECT_ID}/secrets/OPENAI_API_KEY/versions/1 # Codex reports token counts but no cost; these rates turn them into cost_usd. pricing: diff --git a/.ci/model_configs/gemini_cli_model.yaml b/.ci/model_configs/gemini_cli_model.yaml index 3231d03e..b294f7f2 100644 --- a/.ci/model_configs/gemini_cli_model.yaml +++ b/.ci/model_configs/gemini_cli_model.yaml @@ -3,7 +3,7 @@ generator: gemini_cli env: GOOGLE_CLOUD_PROJECT: !ENV ${EVAL_GCP_PROJECT_ID} - GOOGLE_CLOUD_LOCATION: "us-central1" + GOOGLE_CLOUD_LOCATION: "global" GOOGLE_GENAI_USE_VERTEXAI: "true" GEMINI_CLI_TRUST_WORKSPACE: "true" From ffb7f7f3ee882fa82d76e34c3bc6f4c4b9ed5b17 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 11 Sep 2026 07:42:18 +0000 Subject: [PATCH 06/20] feat: add step to fetch service account key from Secret Manager and mount it to harness containers --- .ci/harness.cloudbuild.yaml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/.ci/harness.cloudbuild.yaml b/.ci/harness.cloudbuild.yaml index 5e1de6bc..6fe7a8cb 100644 --- a/.ci/harness.cloudbuild.yaml +++ b/.ci/harness.cloudbuild.yaml @@ -17,12 +17,24 @@ steps: '-t', 'us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache', '-f', 'evalbench_service/Dockerfile', '.'] - # allowFailure on all four: verify-harnesses is the gate, so one broken - # harness must not cancel the others before they report. + # Fetch the Secret Manager key into a shared volume + - id: fetch-sa-key + name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' + entrypoint: 'bash' + args: + - '-c' + - | + mkdir -p /etc/evalbench-sa-key + gcloud secrets versions access latest --secret="evalbench-sa-key" --project="$_EVAL_PROJECT" > /etc/evalbench-sa-key/key.json + volumes: + - name: 'evalbench_sa_key_volume' + path: '/etc/evalbench-sa-key' + + # agy-cli step (mounts /etc/evalbench-sa-key) - id: agy-cli name: 'evalbench-harness-ci' dir: '/evalbench' - waitFor: ['build-image'] + waitFor: ['build-image', 'fetch-sa-key'] allowFailure: true args: ['uv', 'run', '--no-sync', 'evalbench/evalbench.py', '--experiment_config=.ci/run_configs/agy_cli.yaml'] @@ -31,9 +43,11 @@ steps: - 'EVAL_GCP_PROJECT_REGION=global' - 'CI_CSQL_INSTANCE=${_CI_CSQL_INSTANCE}' - 'UV_CACHE_DIR=/tmp/uv-cache' - volumes: &results_volume + volumes: &all_volumes - name: 'eval_results' path: '/evalbench/results' + - name: 'evalbench_sa_key_volume' + path: '/etc/evalbench-sa-key' - id: claude-code name: 'evalbench-harness-ci' From 4a78c937888295983daaa750c6b9b0769a34a632 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 11 Sep 2026 07:51:10 +0000 Subject: [PATCH 07/20] refactor: clarify cloudbuild steps and optimize volume mounts for individual harness tasks --- .ci/harness.cloudbuild.yaml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/.ci/harness.cloudbuild.yaml b/.ci/harness.cloudbuild.yaml index 6fe7a8cb..2ee1af4b 100644 --- a/.ci/harness.cloudbuild.yaml +++ b/.ci/harness.cloudbuild.yaml @@ -1,13 +1,11 @@ -# _EVAL_PROJECT and _CI_CSQL_INSTANCE are left undefined so the target project -# and instance stay out of this public repo; the trigger supplies both. - steps: - # Primes the layer cache. The pull fails on the first build, before the tag exists. + # 1. Pull cached Docker image - id: pull-cache name: 'gcr.io/cloud-builders/docker' entrypoint: 'bash' args: ['-c', 'docker pull us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache || exit 0'] + # 2. Build Docker image - id: build-image name: 'gcr.io/cloud-builders/docker' waitFor: ['pull-cache'] @@ -17,7 +15,7 @@ steps: '-t', 'us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache', '-f', 'evalbench_service/Dockerfile', '.'] - # Fetch the Secret Manager key into a shared volume + # 3. Fetch SA key ONLY for agy-cli - id: fetch-sa-key name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' entrypoint: 'bash' @@ -27,10 +25,10 @@ steps: mkdir -p /etc/evalbench-sa-key gcloud secrets versions access latest --secret="evalbench-sa-key" --project="$_EVAL_PROJECT" > /etc/evalbench-sa-key/key.json volumes: - - name: 'evalbench_sa_key_volume' + - name: 'sa_key_volume' path: '/etc/evalbench-sa-key' - # agy-cli step (mounts /etc/evalbench-sa-key) + # 4. agy-cli: mounts results_volume + sa_key_volume - id: agy-cli name: 'evalbench-harness-ci' dir: '/evalbench' @@ -43,12 +41,13 @@ steps: - 'EVAL_GCP_PROJECT_REGION=global' - 'CI_CSQL_INSTANCE=${_CI_CSQL_INSTANCE}' - 'UV_CACHE_DIR=/tmp/uv-cache' - volumes: &all_volumes + volumes: - name: 'eval_results' path: '/evalbench/results' - - name: 'evalbench_sa_key_volume' + - name: 'sa_key_volume' path: '/etc/evalbench-sa-key' + # 5. claude-code: only requires results_volume - id: claude-code name: 'evalbench-harness-ci' dir: '/evalbench' @@ -57,8 +56,11 @@ steps: args: ['uv', 'run', '--no-sync', 'evalbench/evalbench.py', '--experiment_config=.ci/run_configs/claude_code.yaml'] env: *harness_env - volumes: *results_volume + volumes: &results_volume + - name: 'eval_results' + path: '/evalbench/results' + # 6. codex-cli: only requires results_volume - id: codex-cli name: 'evalbench-harness-ci' dir: '/evalbench' @@ -69,6 +71,7 @@ steps: env: *harness_env volumes: *results_volume + # 7. gemini-cli: only requires results_volume - id: gemini-cli name: 'evalbench-harness-ci' dir: '/evalbench' @@ -79,6 +82,7 @@ steps: env: *harness_env volumes: *results_volume + # 8. verify-harnesses - id: verify-harnesses name: 'evalbench-harness-ci' dir: '/evalbench' @@ -95,4 +99,4 @@ images: options: logging: CLOUD_LOGGING_ONLY # Sized for the four concurrent harness steps, not for the build. - machineType: 'E2_HIGHCPU_8' + machineType: 'E2_HIGHCPU_8' \ No newline at end of file From d65ff56a6b27cbd9f92cd711b3f1975a1ef56404 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 11 Sep 2026 09:18:33 +0000 Subject: [PATCH 08/20] feat: capture and log probe process output and exit code in agy_cli error messages --- evalbench/generators/models/agy_cli.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/evalbench/generators/models/agy_cli.py b/evalbench/generators/models/agy_cli.py index 98ebe127..c89f4d7a 100644 --- a/evalbench/generators/models/agy_cli.py +++ b/evalbench/generators/models/agy_cli.py @@ -511,7 +511,7 @@ def _verify_runtime(self, configured_servers: list): self.agy_bin, "ping", model=self.model ) try: - subprocess.run( + probe = subprocess.run( cmd, env=env, cwd=self.fake_home, stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=120, check=False, @@ -585,6 +585,11 @@ def _verify_runtime(self, configured_servers: list): "MCP tools are missing." ) msg += f"\nADC in use: {self.adc_path}" + msg += ( + f"\nProbe exit code: {probe.returncode}" + f"\nProbe STDOUT:\n{probe.stdout}" + f"\nProbe STDERR:\n{probe.stderr}" + ) if marker_hits: msg += "\nProbe log fatal markers:\n" + "\n".join( f" {h}" for h in marker_hits From b59399c44c95001289a724a3c3030f02ea0d548c Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 11 Sep 2026 09:27:25 +0000 Subject: [PATCH 09/20] refactor: remove trajectory_matcher threshold gating and update CI cache strategy to push on failure --- .ci/harness.cloudbuild.yaml | 13 +++++++++---- .ci/verify_harness.py | 19 ++++++------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/.ci/harness.cloudbuild.yaml b/.ci/harness.cloudbuild.yaml index 2ee1af4b..2f4db1d1 100644 --- a/.ci/harness.cloudbuild.yaml +++ b/.ci/harness.cloudbuild.yaml @@ -15,10 +15,19 @@ steps: '-t', 'us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache', '-f', 'evalbench_service/Dockerfile', '.'] + # Pushed here rather than via top-level `images:`, which only runs when the + # whole build succeeds -- the failing builds would never warm the cache. + - id: push-cache + name: 'gcr.io/cloud-builders/docker' + waitFor: ['build-image'] + allowFailure: true + args: ['push', 'us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache'] + # 3. Fetch SA key ONLY for agy-cli - id: fetch-sa-key name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' entrypoint: 'bash' + waitFor: ['-'] args: - '-c' - | @@ -92,10 +101,6 @@ steps: timeout: '3600s' -# Republished every build so the next one starts warm. -images: - - 'us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache' - options: logging: CLOUD_LOGGING_ONLY # Sized for the four concurrent harness steps, not for the build. diff --git a/.ci/verify_harness.py b/.ci/verify_harness.py index 6433cbca..a4c03abd 100644 --- a/.ci/verify_harness.py +++ b/.ci/verify_harness.py @@ -4,18 +4,19 @@ evalbench.eval() exits 0 whenever a run completes, including when the agent made no tool calls at all, so the exit code alone cannot gate CI. -Tier 1 (threshold): trajectory_matcher must hit THRESHOLDS -- the agent really -reached the expected tools. - -Tier 2 (non-zero): telemetry scorers must report more than 0. They swallow +Tier 1 (non-zero): telemetry scorers must report more than 0. They swallow parse failures and return 0.0 with an explanation rather than raising, so a plain liveness check passes even when a CLI renames a token field -- the exact drift this build exists to catch. Every scenario makes at least one MCP call, so 0 tokens or 0 latency can only mean the scorer failed to read the output. -Tier 3 (liveness): every remaining scorer must emit a row per scenario, with no +Tier 2 (liveness): every remaining scorer must emit a row per scenario, with no comparison_error and a numeric score. This covers the LLM judges without ever gating on their verdict, which would make the build flaky. + +trajectory_matcher is deliberately liveness-only, not thresholded: which tools +an agent reaches for varies run to run, and a harness may shell out instead of +calling the MCP tool. tool_call_latency > 0 is what proves tools were used. """ import csv import json @@ -27,7 +28,6 @@ HARNESSES = ["agy_cli", "claude_code", "codex_cli", "gemini_cli"] RUN_CONFIG_DIR = ".ci/run_configs" EVALSET = ".ci/harness_smoke.evalset.json" -THRESHOLDS = {"trajectory_matcher": 100.0} POSITIVE = { "turn_count", "agent_steps", @@ -102,11 +102,6 @@ def check(harness, scenario_ids): problems.append(f"{scorer}: errored on {sid} -- {error[:120]}") elif score is None: problems.append(f"{scorer}: non-numeric score for {sid}") - elif scorer in THRESHOLDS and score < THRESHOLDS[scorer]: - problems.append( - f"{scorer}: {sid} scored {score:.1f}, " - f"need >= {THRESHOLDS[scorer]:.0f}" - ) elif scorer in POSITIVE and score <= 0: problems.append( f"{scorer}: {sid} reported 0 -- scorer could not read " @@ -117,9 +112,7 @@ def check(harness, scenario_ids): def main(): scenario_ids = expected_scenario_ids() - gated = ", ".join(f"{k} >= {v:.0f}" for k, v in THRESHOLDS.items()) print(f"Scenarios: {len(scenario_ids)} | Harnesses: {len(HARNESSES)}") - print(f"Thresholded: {gated}") print(f"Must be > 0: {', '.join(sorted(POSITIVE))}") print("All other scorers are liveness-checked (ran, no error, " "numeric score)\n") From 455462d3fd951c886b18107d89d5b4f5371e37a5 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 11 Sep 2026 10:31:45 +0000 Subject: [PATCH 10/20] feat: update secret key name to be consistent with other key names --- .ci/model_configs/codex_cli_model.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/model_configs/codex_cli_model.yaml b/.ci/model_configs/codex_cli_model.yaml index ed7dff4d..d6349474 100644 --- a/.ci/model_configs/codex_cli_model.yaml +++ b/.ci/model_configs/codex_cli_model.yaml @@ -2,7 +2,7 @@ codex_cli_version: "@openai/codex@latest" generator: codex_cli model: "gpt-5.5" -openai_api_key_secret: !ENV projects/${EVAL_GCP_PROJECT_ID}/secrets/OPENAI_API_KEY/versions/1 +openai_api_key_secret: !ENV projects/${EVAL_GCP_PROJECT_ID}/secrets/open-api-key/versions/latest # Codex reports token counts but no cost; these rates turn them into cost_usd. pricing: From 97658934abbcb0872eaa50fef88e325deb3a8052 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 11 Sep 2026 11:21:41 +0000 Subject: [PATCH 11/20] ci: llm key name --- .ci/model_configs/codex_cli_model.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/model_configs/codex_cli_model.yaml b/.ci/model_configs/codex_cli_model.yaml index d6349474..13a885cb 100644 --- a/.ci/model_configs/codex_cli_model.yaml +++ b/.ci/model_configs/codex_cli_model.yaml @@ -2,7 +2,7 @@ codex_cli_version: "@openai/codex@latest" generator: codex_cli model: "gpt-5.5" -openai_api_key_secret: !ENV projects/${EVAL_GCP_PROJECT_ID}/secrets/open-api-key/versions/latest +openai_api_key_secret: !ENV projects/${EVAL_GCP_PROJECT_ID}/secrets/openai-api-key/versions/1 # Codex reports token counts but no cost; these rates turn them into cost_usd. pricing: From ec1a3923c33bab897d273fb2a9519910b5dfd2eb Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad <74178334+omkargaikwad23@users.noreply.github.com> Date: Fri, 11 Sep 2026 18:07:33 +0530 Subject: [PATCH 12/20] Change OpenAI API key secret version to latest Updated the OpenAI API key secret version to 'latest'. --- .ci/model_configs/codex_cli_model.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/model_configs/codex_cli_model.yaml b/.ci/model_configs/codex_cli_model.yaml index 13a885cb..0a051679 100644 --- a/.ci/model_configs/codex_cli_model.yaml +++ b/.ci/model_configs/codex_cli_model.yaml @@ -2,7 +2,7 @@ codex_cli_version: "@openai/codex@latest" generator: codex_cli model: "gpt-5.5" -openai_api_key_secret: !ENV projects/${EVAL_GCP_PROJECT_ID}/secrets/openai-api-key/versions/1 +openai_api_key_secret: !ENV projects/${EVAL_GCP_PROJECT_ID}/secrets/openai-api-key/versions/latest # Codex reports token counts but no cost; these rates turn them into cost_usd. pricing: From b41f45ef6118bdae04e88e0f303e9c59550834ee Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad <74178334+omkargaikwad23@users.noreply.github.com> Date: Fri, 11 Sep 2026 18:19:12 +0530 Subject: [PATCH 13/20] Change OpenAI API key secret version to 2 Updated the version of the OpenAI API key secret in the configuration file. --- .ci/model_configs/codex_cli_model.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/model_configs/codex_cli_model.yaml b/.ci/model_configs/codex_cli_model.yaml index 0a051679..0061cfb9 100644 --- a/.ci/model_configs/codex_cli_model.yaml +++ b/.ci/model_configs/codex_cli_model.yaml @@ -2,7 +2,7 @@ codex_cli_version: "@openai/codex@latest" generator: codex_cli model: "gpt-5.5" -openai_api_key_secret: !ENV projects/${EVAL_GCP_PROJECT_ID}/secrets/openai-api-key/versions/latest +openai_api_key_secret: !ENV projects/${EVAL_GCP_PROJECT_ID}/secrets/openai-api-key/versions/2 # Codex reports token counts but no cost; these rates turn them into cost_usd. pricing: From 2c6ef86ba4e8fbf4aacf16456742bf46001975f5 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 11 Sep 2026 15:53:08 +0000 Subject: [PATCH 14/20] refactor: remove inline comments from CI configuration and update cloud SQL smoke test conversation plan --- .ci/harness.cloudbuild.yaml | 8 -------- .ci/harness_smoke.evalset.json | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.ci/harness.cloudbuild.yaml b/.ci/harness.cloudbuild.yaml index 2f4db1d1..4d5c7b39 100644 --- a/.ci/harness.cloudbuild.yaml +++ b/.ci/harness.cloudbuild.yaml @@ -1,11 +1,9 @@ steps: - # 1. Pull cached Docker image - id: pull-cache name: 'gcr.io/cloud-builders/docker' entrypoint: 'bash' args: ['-c', 'docker pull us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache || exit 0'] - # 2. Build Docker image - id: build-image name: 'gcr.io/cloud-builders/docker' waitFor: ['pull-cache'] @@ -23,7 +21,6 @@ steps: allowFailure: true args: ['push', 'us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache'] - # 3. Fetch SA key ONLY for agy-cli - id: fetch-sa-key name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' entrypoint: 'bash' @@ -37,7 +34,6 @@ steps: - name: 'sa_key_volume' path: '/etc/evalbench-sa-key' - # 4. agy-cli: mounts results_volume + sa_key_volume - id: agy-cli name: 'evalbench-harness-ci' dir: '/evalbench' @@ -56,7 +52,6 @@ steps: - name: 'sa_key_volume' path: '/etc/evalbench-sa-key' - # 5. claude-code: only requires results_volume - id: claude-code name: 'evalbench-harness-ci' dir: '/evalbench' @@ -69,7 +64,6 @@ steps: - name: 'eval_results' path: '/evalbench/results' - # 6. codex-cli: only requires results_volume - id: codex-cli name: 'evalbench-harness-ci' dir: '/evalbench' @@ -80,7 +74,6 @@ steps: env: *harness_env volumes: *results_volume - # 7. gemini-cli: only requires results_volume - id: gemini-cli name: 'evalbench-harness-ci' dir: '/evalbench' @@ -91,7 +84,6 @@ steps: env: *harness_env volumes: *results_volume - # 8. verify-harnesses - id: verify-harnesses name: 'evalbench-harness-ci' dir: '/evalbench' diff --git a/.ci/harness_smoke.evalset.json b/.ci/harness_smoke.evalset.json index 9426523a..9712e895 100644 --- a/.ci/harness_smoke.evalset.json +++ b/.ci/harness_smoke.evalset.json @@ -3,7 +3,7 @@ { "id": "ci-csql-list-and-get", "starting_prompt": "list all instances in project ${EVAL_GCP_PROJECT_ID}", - "conversation_plan": "Ask the agent to list instances in project ${EVAL_GCP_PROJECT_ID}. Once all instances are listed if ${CI_CSQL_INSTANCE} exist get its state and validate its RUNNABLE", + "conversation_plan": "The user wants to see every Cloud SQL instance in project ${EVAL_GCP_PROJECT_ID}, then check the health of one of them. Once the agent has listed the instances, the user asks for the current state of ${CI_CSQL_INSTANCE} and expects the agent to report whether it is RUNNABLE. The user accepts that answer and ends the conversation.", "expected_trajectory": [ "cloud-sql__list_instances", "cloud-sql__get_instance" From 1d42c705f3587605ba983c58bdb35193f60a4863 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Mon, 14 Sep 2026 06:16:36 +0000 Subject: [PATCH 15/20] revert: drop agy_cli probe output logging Out of scope for the CI harness smoke test PR. --- evalbench/generators/models/agy_cli.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/evalbench/generators/models/agy_cli.py b/evalbench/generators/models/agy_cli.py index c89f4d7a..98ebe127 100644 --- a/evalbench/generators/models/agy_cli.py +++ b/evalbench/generators/models/agy_cli.py @@ -511,7 +511,7 @@ def _verify_runtime(self, configured_servers: list): self.agy_bin, "ping", model=self.model ) try: - probe = subprocess.run( + subprocess.run( cmd, env=env, cwd=self.fake_home, stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=120, check=False, @@ -585,11 +585,6 @@ def _verify_runtime(self, configured_servers: list): "MCP tools are missing." ) msg += f"\nADC in use: {self.adc_path}" - msg += ( - f"\nProbe exit code: {probe.returncode}" - f"\nProbe STDOUT:\n{probe.stdout}" - f"\nProbe STDERR:\n{probe.stderr}" - ) if marker_hits: msg += "\nProbe log fatal markers:\n" + "\n".join( f" {h}" for h in marker_hits From e5043fa1f56e26fe15f66640ff4b58c30561e3c7 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Mon, 14 Sep 2026 06:35:05 +0000 Subject: [PATCH 16/20] fix: enforce non-zero trajectory_matcher scores and handle non-finite telemetry values in verify_harness.py --- .ci/verify_harness.py | 46 +++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/.ci/verify_harness.py b/.ci/verify_harness.py index a4c03abd..b988f559 100644 --- a/.ci/verify_harness.py +++ b/.ci/verify_harness.py @@ -1,25 +1,25 @@ #!/usr/bin/env python3 """Gates the harness smoke build on two tiers of check. -evalbench.eval() exits 0 whenever a run completes, including when the agent -made no tool calls at all, so the exit code alone cannot gate CI. - -Tier 1 (non-zero): telemetry scorers must report more than 0. They swallow -parse failures and return 0.0 with an explanation rather than raising, so a -plain liveness check passes even when a CLI renames a token field -- the exact -drift this build exists to catch. Every scenario makes at least one MCP call, -so 0 tokens or 0 latency can only mean the scorer failed to read the output. - -Tier 2 (liveness): every remaining scorer must emit a row per scenario, with no -comparison_error and a numeric score. This covers the LLM judges without ever -gating on their verdict, which would make the build flaky. - -trajectory_matcher is deliberately liveness-only, not thresholded: which tools -an agent reaches for varies run to run, and a harness may shell out instead of -calling the MCP tool. tool_call_latency > 0 is what proves tools were used. +evalbench.eval() exits 0 whenever a run completes, so the exit code alone +cannot gate CI. + +Tier 1 (non-zero): telemetry scorers and trajectory_matcher must report more +than 0. They swallow parse failures and return 0.0 with an explanation rather +than raising, so a plain liveness check passes even when a CLI renames a token +field -- the exact drift this build exists to catch. Every scenario requires at +least one tool call, so these can never legitimately be 0. + +Tier 2 (liveness) covers the LLM judges without ever gating on their verdict, +which would make the build flaky. + +trajectory_matcher scores Jaccard overlap, so > 0 means at least one expected +tool was called -- which is what catches a harness that silently shells out +instead of reaching for MCP. """ import csv import json +import math import os import sys @@ -29,6 +29,7 @@ RUN_CONFIG_DIR = ".ci/run_configs" EVALSET = ".ci/harness_smoke.evalset.json" POSITIVE = { + "trajectory_matcher", "turn_count", "agent_steps", "end_to_end_latency", @@ -67,6 +68,10 @@ def load_rows(job_dir): for row in csv.DictReader(f): try: score = float(row["score"]) + # nan <= 0 is False, so an unguarded nan would clear the + # Tier 1 gate. + if not math.isfinite(score): + score = None except (KeyError, TypeError, ValueError): score = None rows[(row.get("comparator"), row.get("id"))] = ( @@ -103,10 +108,13 @@ def check(harness, scenario_ids): elif score is None: problems.append(f"{scorer}: non-numeric score for {sid}") elif scorer in POSITIVE and score <= 0: - problems.append( - f"{scorer}: {sid} reported 0 -- scorer could not read " - f"the harness output" + reason = ( + "none of the expected tools were called" + if scorer == "trajectory_matcher" + else "the agent called no tools, or the scorer could not " + "read the harness output" ) + problems.append(f"{scorer}: {sid} reported 0 -- {reason}") return problems, checked, scorers From 7385d026583fd004c560bdaefa10591bdad19846 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Mon, 14 Sep 2026 07:00:05 +0000 Subject: [PATCH 17/20] feat: add agy_cli model configuration and update run config path --- .ci/model_configs/agy_cli_model.yaml | 20 ++++++++++++++++++++ .ci/run_configs/agy_cli.yaml | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 .ci/model_configs/agy_cli_model.yaml diff --git a/.ci/model_configs/agy_cli_model.yaml b/.ci/model_configs/agy_cli_model.yaml new file mode 100644 index 00000000..1aba462f --- /dev/null +++ b/.ci/model_configs/agy_cli_model.yaml @@ -0,0 +1,20 @@ +generator: agy_cli +# ADC does not authorize the Pro (High) tier. +model: "Gemini 3.1 Pro (Low)" +timeout: "20m" + +env: + GOOGLE_CLOUD_PROJECT: !ENV ${EVAL_GCP_PROJECT_ID} + GOOGLE_CLOUD_LOCATION: "global" + GOOGLE_GENAI_USE_VERTEXAI: "true" + +setup: + mcp_servers: + "cloud-sql": + httpUrl: "https://sqladmin.googleapis.com/mcp" + authProviderType: google_credentials + oauth: + scopes: + - https://www.googleapis.com/auth/cloud-platform + headers: + X-Goog-User-Project: !ENV ${EVAL_GCP_PROJECT_ID} diff --git a/.ci/run_configs/agy_cli.yaml b/.ci/run_configs/agy_cli.yaml index 9af79f52..0800e5de 100644 --- a/.ci/run_configs/agy_cli.yaml +++ b/.ci/run_configs/agy_cli.yaml @@ -2,7 +2,7 @@ dataset_config: .ci/harness_smoke.evalset.json dataset_format: agent-format orchestrator: agent -model_config: datasets/model_configs/agy_cli_model.yaml +model_config: .ci/model_configs/agy_cli_model.yaml simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml scorers: From 61f5d8988d6794337f866a66fd2a1d0988ef10e4 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Mon, 14 Sep 2026 07:29:12 +0000 Subject: [PATCH 18/20] refactor: consolidate individual harness run configs into a single parameterized YAML file using environment variables --- .ci/harness.cloudbuild.yaml | 16 ++++++------ .../agy_cli.yaml => run_config.yaml} | 6 +++-- .ci/run_configs/claude_code.yaml | 26 ------------------- .ci/run_configs/codex_cli.yaml | 26 ------------------- .ci/run_configs/gemini_cli.yaml | 26 ------------------- .ci/verify_harness.py | 10 ++++--- 6 files changed, 18 insertions(+), 92 deletions(-) rename .ci/{run_configs/agy_cli.yaml => run_config.yaml} (72%) delete mode 100644 .ci/run_configs/claude_code.yaml delete mode 100644 .ci/run_configs/codex_cli.yaml delete mode 100644 .ci/run_configs/gemini_cli.yaml diff --git a/.ci/harness.cloudbuild.yaml b/.ci/harness.cloudbuild.yaml index 4d5c7b39..1bc9b281 100644 --- a/.ci/harness.cloudbuild.yaml +++ b/.ci/harness.cloudbuild.yaml @@ -39,8 +39,8 @@ steps: dir: '/evalbench' waitFor: ['build-image', 'fetch-sa-key'] allowFailure: true - args: ['uv', 'run', '--no-sync', 'evalbench/evalbench.py', - '--experiment_config=.ci/run_configs/agy_cli.yaml'] + entrypoint: 'bash' + args: ['-c', 'CI_HARNESS=agy_cli uv run --no-sync evalbench/evalbench.py --experiment_config=.ci/run_config.yaml'] env: &harness_env - 'EVAL_GCP_PROJECT_ID=${_EVAL_PROJECT}' - 'EVAL_GCP_PROJECT_REGION=global' @@ -57,8 +57,8 @@ steps: dir: '/evalbench' waitFor: ['build-image'] allowFailure: true - args: ['uv', 'run', '--no-sync', 'evalbench/evalbench.py', - '--experiment_config=.ci/run_configs/claude_code.yaml'] + entrypoint: 'bash' + args: ['-c', 'CI_HARNESS=claude_code uv run --no-sync evalbench/evalbench.py --experiment_config=.ci/run_config.yaml'] env: *harness_env volumes: &results_volume - name: 'eval_results' @@ -69,8 +69,8 @@ steps: dir: '/evalbench' waitFor: ['build-image'] allowFailure: true - args: ['uv', 'run', '--no-sync', 'evalbench/evalbench.py', - '--experiment_config=.ci/run_configs/codex_cli.yaml'] + entrypoint: 'bash' + args: ['-c', 'CI_HARNESS=codex_cli uv run --no-sync evalbench/evalbench.py --experiment_config=.ci/run_config.yaml'] env: *harness_env volumes: *results_volume @@ -79,8 +79,8 @@ steps: dir: '/evalbench' waitFor: ['build-image'] allowFailure: true - args: ['uv', 'run', '--no-sync', 'evalbench/evalbench.py', - '--experiment_config=.ci/run_configs/gemini_cli.yaml'] + entrypoint: 'bash' + args: ['-c', 'CI_HARNESS=gemini_cli uv run --no-sync evalbench/evalbench.py --experiment_config=.ci/run_config.yaml'] env: *harness_env volumes: *results_volume diff --git a/.ci/run_configs/agy_cli.yaml b/.ci/run_config.yaml similarity index 72% rename from .ci/run_configs/agy_cli.yaml rename to .ci/run_config.yaml index 0800e5de..584092c6 100644 --- a/.ci/run_configs/agy_cli.yaml +++ b/.ci/run_config.yaml @@ -1,8 +1,10 @@ +# Shared by all four harnesses. CI_HARNESS selects which one runs, so every +# harness is scored identically and stays comparable. dataset_config: .ci/harness_smoke.evalset.json dataset_format: agent-format orchestrator: agent -model_config: .ci/model_configs/agy_cli_model.yaml +model_config: !ENV .ci/model_configs/${CI_HARNESS}_model.yaml simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml scorers: @@ -23,4 +25,4 @@ scorers: reporting: csv: - output_directory: 'results/ci/agy_cli' + output_directory: !ENV results/ci/${CI_HARNESS} diff --git a/.ci/run_configs/claude_code.yaml b/.ci/run_configs/claude_code.yaml deleted file mode 100644 index 8b4f3636..00000000 --- a/.ci/run_configs/claude_code.yaml +++ /dev/null @@ -1,26 +0,0 @@ -dataset_config: .ci/harness_smoke.evalset.json -dataset_format: agent-format - -orchestrator: agent -model_config: .ci/model_configs/claude_code_model.yaml -simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml - -scorers: - trajectory_matcher: {} - turn_count: {} - agent_steps: {} - end_to_end_latency: {} - tool_call_latency: {} - token_consumption: {} - tokens_processed: {} - effective_billed_tokens: {} - goal_completion: - model_config: datasets/model_configs/gemini_3.1_pro_model.yaml - behavioral_metrics: - model_config: datasets/model_configs/gemini_3.1_pro_model.yaml - parameter_analysis: - model_config: datasets/model_configs/gemini_3.1_pro_model.yaml - -reporting: - csv: - output_directory: 'results/ci/claude_code' diff --git a/.ci/run_configs/codex_cli.yaml b/.ci/run_configs/codex_cli.yaml deleted file mode 100644 index a4a5f389..00000000 --- a/.ci/run_configs/codex_cli.yaml +++ /dev/null @@ -1,26 +0,0 @@ -dataset_config: .ci/harness_smoke.evalset.json -dataset_format: agent-format - -orchestrator: agent -model_config: .ci/model_configs/codex_cli_model.yaml -simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml - -scorers: - trajectory_matcher: {} - turn_count: {} - agent_steps: {} - end_to_end_latency: {} - tool_call_latency: {} - token_consumption: {} - tokens_processed: {} - effective_billed_tokens: {} - goal_completion: - model_config: datasets/model_configs/gemini_3.1_pro_model.yaml - behavioral_metrics: - model_config: datasets/model_configs/gemini_3.1_pro_model.yaml - parameter_analysis: - model_config: datasets/model_configs/gemini_3.1_pro_model.yaml - -reporting: - csv: - output_directory: 'results/ci/codex_cli' diff --git a/.ci/run_configs/gemini_cli.yaml b/.ci/run_configs/gemini_cli.yaml deleted file mode 100644 index e6373fdc..00000000 --- a/.ci/run_configs/gemini_cli.yaml +++ /dev/null @@ -1,26 +0,0 @@ -dataset_config: .ci/harness_smoke.evalset.json -dataset_format: agent-format - -orchestrator: agent -model_config: .ci/model_configs/gemini_cli_model.yaml -simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml - -scorers: - trajectory_matcher: {} - turn_count: {} - agent_steps: {} - end_to_end_latency: {} - tool_call_latency: {} - token_consumption: {} - tokens_processed: {} - effective_billed_tokens: {} - goal_completion: - model_config: datasets/model_configs/gemini_3.1_pro_model.yaml - behavioral_metrics: - model_config: datasets/model_configs/gemini_3.1_pro_model.yaml - parameter_analysis: - model_config: datasets/model_configs/gemini_3.1_pro_model.yaml - -reporting: - csv: - output_directory: 'results/ci/gemini_cli' diff --git a/.ci/verify_harness.py b/.ci/verify_harness.py index b988f559..2d464e72 100644 --- a/.ci/verify_harness.py +++ b/.ci/verify_harness.py @@ -23,10 +23,10 @@ import os import sys -import yaml +from pyaml_env import parse_config HARNESSES = ["agy_cli", "claude_code", "codex_cli", "gemini_cli"] -RUN_CONFIG_DIR = ".ci/run_configs" +RUN_CONFIG = ".ci/run_config.yaml" EVALSET = ".ci/harness_smoke.evalset.json" POSITIVE = { "trajectory_matcher", @@ -46,8 +46,10 @@ def expected_scenario_ids(): def run_config(harness): - with open(os.path.join(RUN_CONFIG_DIR, f"{harness}.yaml")) as f: - return yaml.safe_load(f) + # The run config resolves ${CI_HARNESS} into the model config and output + # paths, so it has to be re-parsed per harness. + os.environ["CI_HARNESS"] = harness + return parse_config(RUN_CONFIG) def latest_job_dir(output_dir): From d0edc766c67c47fe1792d6cebae99c4c107841d8 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Mon, 14 Sep 2026 09:58:48 +0000 Subject: [PATCH 19/20] chore: update model versions and reduce max turns for smoke harness --- .ci/harness_smoke.evalset.json | 2 +- .ci/model_configs/claude_code_model.yaml | 4 ++-- .ci/model_configs/gemini_cli_model.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/harness_smoke.evalset.json b/.ci/harness_smoke.evalset.json index 9712e895..2a976d19 100644 --- a/.ci/harness_smoke.evalset.json +++ b/.ci/harness_smoke.evalset.json @@ -12,7 +12,7 @@ "GOOGLE_CLOUD_PROJECT": "${EVAL_GCP_PROJECT_ID}" }, "kind": "tools", - "max_turns": 3 + "max_turns": 2 }, { "id": "ci-csql-get-instance-not-found", diff --git a/.ci/model_configs/claude_code_model.yaml b/.ci/model_configs/claude_code_model.yaml index 9125871a..c2606979 100644 --- a/.ci/model_configs/claude_code_model.yaml +++ b/.ci/model_configs/claude_code_model.yaml @@ -1,6 +1,6 @@ -claude_code_version: "@anthropic-ai/claude-code@2.1.85" +claude_code_version: "@anthropic-ai/claude-code@latest" generator: claude_code -model: "claude-opus-4-6" +model: "claude-opus-5" use_vertex: true vertex_project_id: !ENV ${EVAL_GCP_PROJECT_ID} diff --git a/.ci/model_configs/gemini_cli_model.yaml b/.ci/model_configs/gemini_cli_model.yaml index b294f7f2..67bd122e 100644 --- a/.ci/model_configs/gemini_cli_model.yaml +++ b/.ci/model_configs/gemini_cli_model.yaml @@ -1,4 +1,4 @@ -gemini_cli_version: "@google/gemini-cli@0.55.1" +gemini_cli_version: "@google/gemini-cli@latest" generator: gemini_cli env: From 7f08d5d76193b0258c601e98af1150bee38ff3f3 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad <74178334+omkargaikwad23@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:25:41 +0530 Subject: [PATCH 20/20] Update vertex_region to 'global' in model config Changed vertex_region from 'us-east5' to 'global' for broader accessibility. --- .ci/model_configs/claude_code_model.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.ci/model_configs/claude_code_model.yaml b/.ci/model_configs/claude_code_model.yaml index c2606979..4d4350e3 100644 --- a/.ci/model_configs/claude_code_model.yaml +++ b/.ci/model_configs/claude_code_model.yaml @@ -4,8 +4,7 @@ model: "claude-opus-5" use_vertex: true vertex_project_id: !ENV ${EVAL_GCP_PROJECT_ID} -# Not us-central1: Vertex serves Claude from only a few regions. -vertex_region: "us-east5" +vertex_region: "global" env: GOOGLE_CLOUD_PROJECT: !ENV ${EVAL_GCP_PROJECT_ID}