From b84c98b3dfc448d816770e24a3257eaf19a42325 Mon Sep 17 00:00:00 2001 From: Yifan Lu Date: Fri, 30 Oct 2026 17:08:14 -0500 Subject: [PATCH] /evaluate: grade with main's evaluator and keep submission code away from the write token The workflow ran automation/process_pr_submission.py from the base branch, but the script then checked out the PR head and ran that branch's validator, evaluator and lockfile. Fixes merged to main didn't apply until a contributor rebased, and code in a submission branch ran in a job whose token could post PR comments and update checks. - The script now creates its worktree from the base branch and copies in only the submission's data files from the PR: its prediction, robustness and config files, prices for models it adds (models already priced keep main's price), and the names it adds to universal_model_names.py, which is parsed, not imported. Other files the PR changes are listed in the log. - The workflow is split into three jobs. Only the evaluate job runs submission code (LiveCodeBench answers are executed), with a read-only token and no stored credentials; it hands metrics.json to the report job, which accepts only numbers from it. - The router name is checked against [A-Za-z0-9._-] and passed through env instead of being expanded into scripts. - The results comment names the evaluator commit. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/pr-evaluation.yml | 182 +++++++++++++++------ README.md | 2 +- automation/process_pr_submission.py | 244 ++++++++++++++++++++++++++-- 3 files changed, 360 insertions(+), 68 deletions(-) diff --git a/.github/workflows/pr-evaluation.yml b/.github/workflows/pr-evaluation.yml index 72fd3868..139bd717 100644 --- a/.github/workflows/pr-evaluation.yml +++ b/.github/workflows/pr-evaluation.yml @@ -4,12 +4,13 @@ on: issue_comment: types: [created] +# Grading runs code from the submission (LiveCodeBench answers are executed), so the +# evaluate job gets a read-only token and stores no credentials. The jobs that can +# comment on the PR or update its check never run submission code. +permissions: {} + jobs: - evaluate-router: - concurrency: - group: evaluate-router-pr-${{ github.event.issue.number }} - cancel-in-progress: false - timeout-minutes: 120 + start: if: >- github.event.issue.pull_request && startsWith(github.event.comment.body, '/evaluate') && @@ -22,10 +23,13 @@ jobs: runs-on: ubuntu-latest permissions: checks: write - contents: read pull-requests: write - env: - UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv + outputs: + head_sha: ${{ steps.pr.outputs.head_sha }} + base_ref: ${{ steps.pr.outputs.base_ref }} + base_sha: ${{ steps.pr.outputs.base_sha }} + number: ${{ steps.pr.outputs.number }} + check_id: ${{ steps.checkrun.outputs.id }} steps: - name: Acknowledge /evaluate command uses: actions/github-script@v8 @@ -73,32 +77,52 @@ jobs: }); core.setOutput('id', String(result.data.id)); + evaluate: + needs: start + concurrency: + group: evaluate-router-pr-${{ github.event.issue.number }} + cancel-in-progress: false + timeout-minutes: 120 + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + router: ${{ steps.detect.outputs.router }} + split: ${{ steps.detect.outputs.split }} + evaluator_commit: ${{ steps.base.outputs.commit }} + env: + UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv + steps: - name: Checkout base repository (for evaluation scripts) + id: base uses: actions/checkout@v5 with: - ref: ${{ steps.pr.outputs.base_ref }} + ref: ${{ needs.start.outputs.base_ref }} path: base clean: false fetch-depth: 0 + persist-credentials: false - - name: Checkout PR branch (for prediction file only) + - name: Checkout PR branch (to detect the router; never executed) uses: actions/checkout@v5 with: - ref: ${{ steps.pr.outputs.head_sha }} + ref: ${{ needs.start.outputs.head_sha }} path: pr clean: false fetch-depth: 0 + persist-credentials: false - name: Detect changed prediction file id: detect shell: bash working-directory: pr + env: + BASE_REF: ${{ needs.start.outputs.base_ref }} + BASE_SHA: ${{ needs.start.outputs.base_sha }} run: | set -euo pipefail # Compare against the upstream base branch # This ensures each router submission is evaluated independently - BASE_REF="${{ steps.pr.outputs.base_ref }}" - BASE_SHA="${{ steps.pr.outputs.base_sha }}" if [[ -z "$BASE_SHA" ]]; then echo "Error: Could not determine PR base SHA" >&2 @@ -158,6 +182,12 @@ jobs: exit 1 fi + # The name reaches later shell and script steps, so keep it to safe characters. + if [[ ! "$router_name" =~ ^[A-Za-z0-9._-]+$ ]]; then + echo "Router name may only use letters, digits, '.', '_' and '-': $router_name" >&2 + exit 1 + fi + ROUTER_NAME="$router_name" echo "router=$ROUTER_NAME" >> "$GITHUB_OUTPUT" @@ -167,7 +197,7 @@ jobs: echo "Error: Prediction file not found at $PREDICTION_FILE" >&2 exit 1 fi - ENTRY_COUNT=$(python3 -c "import json; print(len(json.load(open('$PREDICTION_FILE'))))") + ENTRY_COUNT=$(python3 -I -c "import json; print(len(json.load(open('$PREDICTION_FILE'))))") echo "Prediction file contains $ENTRY_COUNT entries" if [[ "$ENTRY_COUNT" -eq 8400 ]]; then @@ -182,10 +212,13 @@ jobs: - name: Show detected router if: ${{ steps.detect.outputs.router != '' }} + env: + ROUTER: ${{ steps.detect.outputs.router }} + SPLIT: ${{ steps.detect.outputs.split }} run: | set -euo pipefail - echo "Detected router submission: ${{ steps.detect.outputs.router }}" - echo "Detected split: ${{ steps.detect.outputs.split }}" + echo "Detected router submission: $ROUTER" + echo "Detected split: $SPLIT" - name: Free up disk space on the runner if: ${{ steps.detect.outputs.router != '' }} @@ -227,19 +260,6 @@ jobs: mkdir -p dataset uv run python scripts/process_datasets/prep_datasets.py - - name: Copy PR prediction file to base workspace - if: ${{ steps.detect.outputs.router != '' }} - run: | - set -euo pipefail - ROUTER_NAME="${{ steps.detect.outputs.router }}" - # Copy prediction file from PR to base workspace - mkdir -p base/router_inference/predictions - cp "pr/router_inference/predictions/${ROUTER_NAME}.json" \ - "base/router_inference/predictions/${ROUTER_NAME}.json" - cp "pr/router_inference/predictions/${ROUTER_NAME}-robustness.json" \ - "base/router_inference/predictions/${ROUTER_NAME}-robustness.json" - echo "Copied prediction file from PR to base workspace" - - name: Enable Swap Space run: | # Create a new 10GB swap file (using a unique name to avoid conflicts) @@ -255,35 +275,89 @@ jobs: working-directory: base env: ROUTERARENA_DATASET_DIR: ${{ github.workspace }}/base/dataset + PR_NUMBER: ${{ needs.start.outputs.number }} + ROUTER: ${{ steps.detect.outputs.router }} + SPLIT: ${{ steps.detect.outputs.split }} + BASE_SHA: ${{ needs.start.outputs.base_sha }} run: | set -euo pipefail - # Uses base repo's evaluation script (safe - not from PR) - BASE_SHA="${{ steps.pr.outputs.base_sha }}" + # Grades with this checkout's validator and evaluator; the script copies only + # the submission's prediction, config and new model files from the PR. uv run python automation/process_pr_submission.py \ - --pr "${{ steps.pr.outputs.number }}" \ - --router "${{ steps.detect.outputs.router }}" \ - --split "${{ steps.detect.outputs.split }}" \ - --base-ref "$BASE_SHA" 2>&1 | tee evaluation_output.txt + --pr "$PR_NUMBER" \ + --router "$ROUTER" \ + --split "$SPLIT" \ + --base-ref "$BASE_SHA" \ + --eval-ref HEAD 2>&1 | tee evaluation_output.txt + + - name: Upload metrics + if: ${{ steps.evaluate.outcome == 'success' }} + uses: actions/upload-artifact@v4 + with: + name: evaluation-metrics + path: base/metrics.json + if-no-files-found: error + retention-days: 7 + + - name: Cleanup workspace + if: always() + run: | + rm -rf base/ pr/ + + report: + needs: [start, evaluate] + if: ${{ always() && needs.start.result == 'success' }} + runs-on: ubuntu-latest + permissions: + checks: write + pull-requests: write + steps: + - name: Download metrics + if: ${{ needs.evaluate.result == 'success' && needs.evaluate.outputs.router != '' }} + uses: actions/download-artifact@v4 + with: + name: evaluation-metrics + path: metrics - name: Post evaluation results as PR comment - if: ${{ steps.detect.outputs.router != '' && steps.evaluate.outcome == 'success' }} + id: post + if: ${{ needs.evaluate.result == 'success' && needs.evaluate.outputs.router != '' }} uses: actions/github-script@v8 + env: + ROUTER: ${{ needs.evaluate.outputs.router }} + SPLIT: ${{ needs.evaluate.outputs.split }} + BASE_REF: ${{ needs.start.outputs.base_ref }} + EVALUATOR_COMMIT: ${{ needs.evaluate.outputs.evaluator_commit }} with: script: | const fs = require('fs'); const path = require('path'); let comment = '## Router Evaluation Results\n\n'; - comment += `**Router:** \`${{ steps.detect.outputs.router }}\`\n`; - comment += `**Dataset Split:** \`${{ steps.detect.outputs.split }}\`\n\n`; + comment += `**Router:** \`${process.env.ROUTER}\`\n`; + comment += `**Dataset Split:** \`${process.env.SPLIT}\`\n`; + comment += `**Evaluator:** \`${process.env.BASE_REF}\` @ ${process.env.EVALUATOR_COMMIT.slice(0, 7)}\n\n`; // Read metrics from metrics.json file (required - no fallback) - const metricsPath = path.join('base', 'metrics.json'); + const metricsPath = path.join('metrics', 'metrics.json'); if (!fs.existsSync(metricsPath)) { throw new Error(`metrics.json not found at ${metricsPath}. Evaluation must produce metrics.json file.`); } + // The evaluate job ran submission code, so accept only numbers from its output. const metrics = JSON.parse(fs.readFileSync(metricsPath, 'utf8')); + const checkNumbers = (obj, keys) => { + for (const key of keys) { + if (obj[key] !== undefined && !Number.isFinite(obj[key])) { + throw new Error(`metrics.json: ${key} is not a number`); + } + } + }; + checkNumbers(metrics, ['arena_score', 'accuracy', 'total_cost', 'avg_cost_per_query', + 'avg_cost_per_1000', 'num_queries', 'abnormal_count', 'robustness_score']); + if (metrics.optimality) { + checkNumbers(metrics.optimality, ['opt_sel', 'opt_cost', 'opt_acc']); + } comment += '### RouterArena Metrics\n\n'; comment += '| Metric | Value |\n'; comment += '|--------|-------|\n'; @@ -333,7 +407,7 @@ jobs: console.log('Successfully posted evaluation results as PR comment'); - name: React with success - if: ${{ steps.detect.outputs.router != '' && steps.evaluate.outcome == 'success' }} + if: ${{ steps.post.outcome == 'success' }} uses: actions/github-script@v8 with: script: | @@ -345,12 +419,13 @@ jobs: }); - name: Complete PR check - if: ${{ always() && steps.checkrun.outputs.id != '' }} + if: ${{ always() && needs.start.outputs.check_id != '' }} uses: actions/github-script@v8 env: - DETECTED_ROUTER: ${{ steps.detect.outputs.router }} - DETECT_OUTCOME: ${{ steps.detect.outcome }} - EVALUATE_OUTCOME: ${{ steps.evaluate.outcome }} + DETECTED_ROUTER: ${{ needs.evaluate.outputs.router }} + EVALUATE_RESULT: ${{ needs.evaluate.result }} + POST_OUTCOME: ${{ steps.post.outcome }} + CHECK_ID: ${{ needs.start.outputs.check_id }} with: script: | let conclusion = 'success'; @@ -358,28 +433,27 @@ jobs: let summary = 'Router evaluation finished successfully.'; if (!process.env.DETECTED_ROUTER) { - conclusion = process.env.DETECT_OUTCOME === 'success' ? 'neutral' : 'failure'; - title = process.env.DETECT_OUTCOME === 'success' ? 'No router file detected' : 'Evaluation setup failed'; - summary = process.env.DETECT_OUTCOME === 'success' + conclusion = process.env.EVALUATE_RESULT === 'success' ? 'neutral' : 'failure'; + title = process.env.EVALUATE_RESULT === 'success' ? 'No router file detected' : 'Evaluation setup failed'; + summary = process.env.EVALUATE_RESULT === 'success' ? 'No changed prediction file was detected for this PR, so evaluation was skipped.' : 'Failed while detecting prediction files for this PR.'; - } else if (process.env.EVALUATE_OUTCOME !== 'success') { + } else if (process.env.EVALUATE_RESULT !== 'success') { conclusion = 'failure'; title = 'Evaluation failed'; summary = 'The evaluation step failed. Check this workflow run logs for details.'; + } else if (process.env.POST_OUTCOME !== 'success') { + conclusion = 'failure'; + title = 'Could not report results'; + summary = 'The evaluation finished, but its results could not be posted. Check this workflow run logs for details.'; } await github.rest.checks.update({ owner: context.repo.owner, repo: context.repo.repo, - check_run_id: Number('${{ steps.checkrun.outputs.id }}'), + check_run_id: Number(process.env.CHECK_ID), status: 'completed', conclusion, completed_at: new Date().toISOString(), output: { title, summary } }); - - - name: Cleanup workspace - if: always() - run: | - rm -rf base/ pr/ diff --git a/README.md b/README.md index d5ca75b9..ff4d77b4 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ To get your router on the leaderboard, you can open a Pull Request with your rou - When the PR is ready for evaluation, call `/evaluate` in the PR comment to trigger the evaluation workflow. See an example [here](https://github.com/RouteWorks/RouterArena/pull/71#issuecomment-3904936480). - The automated workflow will: - Validate your submission - - Run evaluation on the full dataset + - Run evaluation on the full dataset with the evaluation code on `main`. Only the three files above are taken from your branch, plus any new models you add to `model_cost/model_cost.json` and `universal_model_names.py` (models already listed keep their price on `main`) - Post results as a comment on your PR - Update the leaderboard upon approval diff --git a/automation/process_pr_submission.py b/automation/process_pr_submission.py index 32213614..2b4f20d8 100644 --- a/automation/process_pr_submission.py +++ b/automation/process_pr_submission.py @@ -18,8 +18,10 @@ Key features: -* Fetches the pull request head and creates an isolated git worktree so that the - current working copy stays untouched. +* Grades with the base repository's code, never the pull request's: it creates an + isolated git worktree of ``/main`` (so the current working copy stays + untouched) and copies in only the submission's data files from the pull request + (prediction, robustness and config files, plus the model names and prices it adds). * Runs the existing validation and evaluation scripts via ``uv run``. * Saves evaluation summaries and the evaluated prediction file under ``pr_evaluations/pr-/`` for auditing. @@ -33,6 +35,7 @@ from __future__ import annotations +import ast import math import argparse import json @@ -52,6 +55,15 @@ WORKTREES_DIR = REPO_ROOT / ".pr_worktrees" ARTIFACTS_DIR = REPO_ROOT / "pr_evaluations" +# The only files taken from a pull request; everything else comes from the base branch. +SUBMISSION_FILES = ( + "router_inference/predictions/{router}.json", + "router_inference/predictions/{router}-robustness.json", + "router_inference/config/{router}.json", +) +MODEL_COST_PATH = "model_cost/model_cost.json" +MODEL_NAMES_PATH = "universal_model_names.py" + class CommandError(RuntimeError): """Raised when a subprocess fails and we want a cleaner error message.""" @@ -105,12 +117,19 @@ def run_command( def ensure_worktree( - pr_number: int, remote: str, *, keep_existing: bool = False + pr_number: int, remote: str, eval_ref: str, *, keep_existing: bool = False ) -> tuple[Path, str]: - """Fetch a PR head and materialise it as a git worktree.""" + """Fetch a PR head and create a worktree of the evaluation code to grade it in. + + The worktree is checked out at ``eval_ref``, so the validator, evaluator and + dependencies come from the base repository, never from the pull request. The PR + head is only fetched to a local branch, from which ``stage_submission`` copies + the submission's data files. + """ branch_name = f"pr-{pr_number}" - fetch_ref = f"pull/{pr_number}/head:{branch_name}" + # "+" lets the fetch replace a branch left from an earlier run after a force-push. + fetch_ref = f"+pull/{pr_number}/head:{branch_name}" # Ensure the base branch is up-to-date for later diffs. run_command(["git", "fetch", remote, "main"], cwd=REPO_ROOT) run_command(["git", "fetch", remote, fetch_ref], cwd=REPO_ROOT) @@ -129,8 +148,9 @@ def ensure_worktree( "worktree", "add", "--force", + "--detach", str(worktree_path), - branch_name, + eval_ref, ], cwd=REPO_ROOT, ) @@ -155,7 +175,7 @@ def cleanup_worktree(worktree_path: Path, branch_name: str, *, keep: bool) -> No def ensure_prediction_file_added( - worktree_path: Path, base_ref: str, router_name: str, *, robustness: bool = False + pr_ref: str, base_ref: str, router_name: str, *, robustness: bool = False ) -> None: """Verify the PR adds or modifies a prediction file for the specified router.""" @@ -168,14 +188,14 @@ def ensure_prediction_file_added( "git", "diff", "--name-status", - f"{base_ref}...HEAD", + f"{base_ref}...{pr_ref}", "--", str(target_path), ] completed = subprocess.run( diff_cmd, - cwd=worktree_path, + cwd=REPO_ROOT, check=True, text=True, capture_output=True, @@ -197,6 +217,187 @@ def ensure_prediction_file_added( ) +def read_file_at(ref: str, path: str) -> Optional[bytes]: + """Return a file's contents at ``ref``, or None if it doesn't exist there. + + Reads the git blob directly, so nothing from the pull request is checked out, + and refuses anything other than a regular file (a symlink, for example). + """ + + listing = subprocess.run( + ["git", "ls-tree", ref, "--", path], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ).stdout.split() + if not listing: + return None + mode, kind = listing[0], listing[1] + if kind != "blob" or mode not in ("100644", "100755"): + raise RuntimeError(f"{path} at {ref} is not a regular file.") + return subprocess.run( + ["git", "cat-file", "blob", f"{ref}:{path}"], + cwd=REPO_ROOT, + check=True, + capture_output=True, + ).stdout + + +def literal_assignments(source: str, names: tuple[str, ...]) -> dict[str, object]: + """Evaluate top-level ``name = `` assignments without running the code.""" + + values: dict[str, object] = {} + for node in ast.parse(source).body: + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target, value = node.targets[0], node.value + elif isinstance(node, ast.AnnAssign) and node.value is not None: + target, value = node.target, node.value + else: + continue + if isinstance(target, ast.Name) and target.id in names: + values[target.id] = ast.literal_eval(value) + return values + + +def stage_model_prices(worktree_path: Path, pr_ref: str, base_ref: str) -> None: + """Add prices for the models the PR introduces; priced models keep their price. + + Only entries the PR itself adds or edits count, so a PR branched from an older + base doesn't undo price changes made on the base branch since. + """ + + content = read_file_at(pr_ref, MODEL_COST_PATH) + if content is None: + return + pr_prices = json.loads(content) + if not isinstance(pr_prices, dict): + raise ValueError(f"{MODEL_COST_PATH} in the pull request is not a JSON object.") + merge_base = subprocess.run( + ["git", "merge-base", base_ref, pr_ref], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ).stdout.strip() + before = json.loads(read_file_at(merge_base, MODEL_COST_PATH) or b"{}") + pr_edits = { + model: price for model, price in pr_prices.items() if before.get(model) != price + } + + cost_path = worktree_path / MODEL_COST_PATH + prices = json.loads(cost_path.read_text(encoding="utf-8")) + added = {model: price for model, price in pr_edits.items() if model not in prices} + changed = sorted( + model + for model, price in pr_edits.items() + if model in prices and price != prices[model] + ) + if changed: + print( + f"⚠ The PR changes the price of {len(changed)} existing model(s); " + f"evaluating with the base branch's price: {', '.join(changed)}" + ) + if added: + prices.update(added) + cost_path.write_text(json.dumps(prices, indent=2) + "\n", encoding="utf-8") + print( + f" Added prices for {len(added)} new model(s): {', '.join(sorted(added))}" + ) + + +def stage_model_names(worktree_path: Path, pr_ref: str) -> None: + """Register the model names and name mappings the PR adds. + + The PR's ``universal_model_names.py`` is parsed, not imported, so none of its + code runs; ``universal_names`` and ``mapping`` must be plain literals. + """ + + content = read_file_at(pr_ref, MODEL_NAMES_PATH) + if content is None: + return + keys = ("universal_names", "mapping") + try: + pr_values = literal_assignments(content.decode("utf-8"), keys) + except (SyntaxError, ValueError, UnicodeDecodeError) as error: + raise RuntimeError( + f"Could not read {MODEL_NAMES_PATH} from the pull request; universal_names " + f"and mapping must be plain literals ({error})." + ) from error + pr_names = pr_values.get("universal_names", []) + pr_mapping = pr_values.get("mapping", {}) + if not ( + isinstance(pr_names, list) + and all(isinstance(name, str) for name in pr_names) + and isinstance(pr_mapping, dict) + and all( + isinstance(k, str) and isinstance(v, str) for k, v in pr_mapping.items() + ) + ): + raise RuntimeError( + f"{MODEL_NAMES_PATH} in the pull request must define universal_names as a " + "list of strings and mapping as a dict of strings." + ) + + names_path = worktree_path / MODEL_NAMES_PATH + base_values = literal_assignments(names_path.read_text(encoding="utf-8"), keys) + base_names = base_values.get("universal_names", []) + base_mapping = base_values.get("mapping", {}) + assert isinstance(base_names, list) and isinstance(base_mapping, dict) + new_names = [name for name in pr_names if name not in base_names] + new_mapping = {k: v for k, v in pr_mapping.items() if k not in base_mapping} + if not (new_names or new_mapping): + return + with names_path.open("a", encoding="utf-8") as handle: + handle.write("\n# Added from the pull request under evaluation.\n") + handle.write(f"universal_names.extend({new_names!r})\n") + handle.write(f"mapping.update({new_mapping!r})\n") + print( + f" Added {len(new_names)} model name(s) and {len(new_mapping)} name mapping(s)" + ) + + +def stage_submission( + worktree_path: Path, pr_ref: str, base_ref: str, router_name: str +) -> None: + """Copy a submission's data files from the pull request into the worktree. + + Only the router's prediction, robustness and config files are copied as they + are, plus the model names and prices the PR adds. Every other file the PR + changes is evaluated with the base branch's version and listed in the log. + """ + + staged = {template.format(router=router_name) for template in SUBMISSION_FILES} + for path in sorted(staged): + content = read_file_at(pr_ref, path) + if content is None: + continue + destination = worktree_path / path + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(content) + print(f" Copied {path}") + + stage_model_prices(worktree_path, pr_ref, base_ref) + stage_model_names(worktree_path, pr_ref) + + changed = subprocess.run( + ["git", "diff", "--name-only", f"{base_ref}...{pr_ref}"], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ).stdout.splitlines() + ignored = [ + path + for path in changed + if path not in staged | {MODEL_COST_PATH, MODEL_NAMES_PATH} + ] + if ignored: + print("⚠ Changed in the PR but not used for evaluation:") + for path in ignored: + print(f" {path}") + + def find_dataset_source() -> Optional[Path]: """Locate a dataset directory accessible from the current environment.""" @@ -449,6 +650,12 @@ def main(argv: Optional[list[str]] = None) -> int: default=None, help="Git ref to diff against when validating submissions (default: /main).", ) + parser.add_argument( + "--eval-ref", + default=None, + help="Commit whose validator and evaluator grade the submission " + "(default: /main).", + ) parser.add_argument( "--allow-existing-prediction", action="store_true", @@ -461,21 +668,31 @@ def main(argv: Optional[list[str]] = None) -> int: branch_name = "" base_ref = args.base_ref or f"{args.remote}/main" + eval_ref = args.eval_ref or f"{args.remote}/main" try: worktree_path, branch_name = ensure_worktree( - args.pr, args.remote, keep_existing=args.keep_worktree + args.pr, args.remote, eval_ref, keep_existing=args.keep_worktree + ) + evaluator_commit = run_command( + ["git", "rev-parse", "HEAD"], cwd=worktree_path, capture=True + ).stdout.strip() + print( + f"✔ Created worktree at {worktree_path} " + f"(evaluator: {eval_ref} @ {evaluator_commit})" ) - print(f"✔ Created worktree at {worktree_path}") sync_dataset_into_worktree(worktree_path) if not args.allow_existing_prediction: - ensure_prediction_file_added(worktree_path, base_ref, args.router) + ensure_prediction_file_added(branch_name, base_ref, args.router) ensure_prediction_file_added( - worktree_path, base_ref, args.router, robustness=True + branch_name, base_ref, args.router, robustness=True ) + print("▶ Copying the submission's files from the pull request...", flush=True) + stage_submission(worktree_path, branch_name, base_ref, args.router) + if not args.skip_sync: print("▶ Syncing dependencies with uv...", flush=True) run_command(["uv", "sync", "--locked"], cwd=worktree_path, capture=False) @@ -598,6 +815,7 @@ def main(argv: Optional[list[str]] = None) -> int: "router": args.router, "split": args.split, "worktree": str(worktree_path), + "evaluator_commit": evaluator_commit, "validation_stdout": validation_result.stdout if validation_result.stdout else "",