benchmark.yml

name: Benchmarks

on:
  pull_request:
    branches:
      - master

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  benchmark:
    name: Benchmarks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - name: Install uv and Python 3.14
        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
        with:
          python-version: "3.14"
          enable-cache: true
          cache-dependency-glob: "pyproject.toml"

      - name: Install dependencies
        run: uv sync --group observ

      # Single runs are too noisy to gate on (see benchmarks/compare_runs.py),
      # so master and PR code are benchmarked interleaved, three runs each,
      # with hash randomization pinned so dict/set timings are comparable
      # between processes. The master runs are `|| true` so a baseline that
      # can't run (e.g. benchmarks exercising new APIs) skips the compare
      # instead of failing the job.
      - name: Run benchmarks
        env:
          PYTHONHASHSEED: "0"
        run: |
          git fetch origin master

          for i in 1 2 3; do
            git checkout origin/master -- patchdiff/
            uv run --no-sync pytest benchmarks/benchmark.py \
                --benchmark-only \
                --benchmark-save=master$i \
                --benchmark-sort=mean || true

            git checkout HEAD -- patchdiff/
            uv run --no-sync pytest benchmarks/benchmark.py \
                --benchmark-only \
                --benchmark-save=branch$i \
                --benchmark-sort=mean
          done

      # The 25% threshold is a tripwire for gross accidental regressions,
      # not a precision instrument: CI noise between runs of identical code
      # reaches 10-30%. Smaller deltas should be measured with deliberate
      # repeated local runs.
      - name: Compare benchmarks
        run: |
          uv run --no-sync python benchmarks/compare_runs.py \
              --baseline '.benchmarks/*/*_master?.json' \
              --branch '.benchmarks/*/*_branch?.json' \
              --threshold 0.25