Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 45 additions & 6 deletions .github/workflows/annotation-layout-cost.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,26 @@ name: Annotation layout cost evidence

on:
workflow_dispatch:
inputs:
cohort:
description: Cost cohort (fixed15 is the existing default)
type: choice
options: [fixed15, ftc09]
default: fixed15
pull_request:
types: [labeled]

# This runs only by manual dispatch or when a maintainer adds the
# annotation-layout-cost label to a PR. The 15 fixed-sheet CAD cases are
# expensive on macOS and Windows; ordinary PR updates must not rerun them.
# annotation-layout-cost or annotation-layout-cost-ftc09 label to a PR.
# The CAD cases are expensive on macOS and Windows; ordinary PR updates must
# not rerun them. FTC-09 is a distinct public real part, not large user CAD.
# Cost evidence is not a substitute for semantic or safety admission.
permissions:
contents: read

jobs:
candidate-first-cost:
if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'annotation-layout-cost'
if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'annotation-layout-cost' || github.event.label.name == 'annotation-layout-cost-ftc09'
strategy:
fail-fast: false
matrix:
Expand All @@ -34,15 +41,44 @@ jobs:
python-version: "3.12"
- name: Install pinned development dependencies
run: uv sync --group dev --locked
- name: Select allowlisted cost cohort
id: cohort
shell: bash
env:
REQUESTED_COHORT: ${{ inputs.cohort }}
PR_LABEL: ${{ github.event.label.name }}
run: |
if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then
cohort="$REQUESTED_COHORT"
elif [ "$PR_LABEL" = annotation-layout-cost-ftc09 ]; then
cohort=ftc09
else
cohort=fixed15
fi
case "$cohort" in
fixed15) manifest=tests/fixtures/annotation-layout-corpus-v2.json ;;
ftc09) manifest=tests/fixtures/annotation-layout-holdout-ftc09-v1.json ;;
*) echo "Unsupported cost cohort: $cohort" >&2; exit 2 ;;
esac
printf 'cohort=%s\n' "$cohort" >> "$GITHUB_OUTPUT"
printf 'LAYOUT_COST_COHORT=%s\n' "$cohort" >> "$GITHUB_ENV"
printf 'LAYOUT_COST_MANIFEST=%s\n' "$manifest" >> "$GITHUB_ENV"
- name: Record source provenance before CAD work
shell: bash
run: |
python - <<'PY'
import json
import os
import subprocess
from hashlib import sha256
from pathlib import Path

manifest = Path(os.environ["LAYOUT_COST_MANIFEST"])
# Windows checkout may rewrite text line endings. Hash the committed
# manifest blob so provenance is comparable across all three runners.
manifest_blob = subprocess.check_output(
["git", "show", f"HEAD:{manifest.as_posix()}"]
)
Path("layout-cost-provenance.json").write_text(json.dumps({
"source_sha": subprocess.check_output(
["git", "rev-parse", "HEAD"], text=True
Expand All @@ -52,14 +88,17 @@ jobs:
"event_name": os.environ["GITHUB_EVENT_NAME"],
"run_id": os.environ["GITHUB_RUN_ID"],
"runner_os": os.environ["RUNNER_OS"],
"cohort": os.environ["LAYOUT_COST_COHORT"],
"manifest": str(manifest),
"manifest_sha256": sha256(manifest_blob).hexdigest(),
}, indent=2) + "\n", encoding="utf-8")
PY
- name: Measure candidate-first and offline baseline workers
shell: bash
run: |
echo "source_sha=$(git rev-parse HEAD)"
echo "source_sha=$(git rev-parse HEAD) cohort=$LAYOUT_COST_COHORT"
uv run --no-sync python scripts/annotation-scheme-corpus \
--manifest tests/fixtures/annotation-layout-corpus-v2.json \
--manifest "$LAYOUT_COST_MANIFEST" \
--output layout-cost-cases --candidate-first --jobs 1 \
--worker-timeout-seconds 900 \
> layout-cost-report.json
Expand All @@ -78,7 +117,7 @@ jobs:
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: annotation-layout-cost-${{ matrix.os }}
name: annotation-layout-cost-${{ steps.cohort.outputs.cohort }}-${{ matrix.os }}
path: |
layout-cost-report.json
layout-cost-provenance.json
Expand Down
14 changes: 14 additions & 0 deletions tests/test_workflows.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,20 @@ def test_opt_in_annotation_cost_run_bounds_each_isolated_worker():
assert "--worker-timeout-seconds 900" in job


def test_annotation_cost_real_part_cohort_is_explicit_and_does_not_rerun_fixed15():
workflow = _workflow("annotation-layout-cost.yml")
job = _job(workflow, "candidate-first-cost")

assert "annotation-layout-cost-ftc09" in job
assert "REQUESTED_COHORT: ${{ inputs.cohort }}" in job
assert "fixed15) manifest=tests/fixtures/annotation-layout-corpus-v2.json" in job
assert "ftc09) manifest=tests/fixtures/annotation-layout-holdout-ftc09-v1.json" in job
assert '--manifest "$LAYOUT_COST_MANIFEST"' in job
assert '["git", "show", f"HEAD:{manifest.as_posix()}"]' in job
assert '"manifest_sha256": sha256(manifest_blob).hexdigest()' in job
assert "annotation-layout-cost-${{ steps.cohort.outputs.cohort }}-${{ matrix.os }}" in job


def test_compatibility_jobs_use_the_pr_manifest_and_keep_the_full_tier_reachable():
test_job = _job(_workflow("ci.yml"), "test")

Expand Down
Loading