Measure benchmarks by counted instructions with CodSpeed instead of phpbench

.agents/setup

@@ -75,7 +75,7 @@ log "Using $(composer --version 2>/dev/null | head -1)"
 # the same lockfile. Patches are applied by cweagans/composer-patches on install.
 log "composer install (root)"
 composer install --no-interaction --no-progress
-log "composer install (tests: paratest, phpbench, phpunit)"
+log "composer install (tests: paratest, phpunit)"
 composer install --working-dir tests --no-interaction --no-progress
 
 log "setup complete"

.github/workflows/benchmarks.yml

@@ -0,0 +1,168 @@
+name: "Benchmarks"
+
+on:
+  pull_request:
+    paths:
+      - 'src/**'
+      - 'conf/**'
+      - 'stubs/**'
+      - 'composer.lock'
+      - 'tests/bench/**'
+      - '.github/workflows/benchmarks.yml'
+  push:
+    branches:
+      - "2.3.x"
+    paths:
+      - 'src/**'
+      - 'conf/**'
+      - 'stubs/**'
+      - 'composer.lock'
+      - 'tests/bench/**'
+      - '.github/workflows/benchmarks.yml'
+
+concurrency:
+  group: benchmarks-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  CODSPEED_VERSION: "5.6.3"
+  # tests/bench/measure.php calls it over FFI to measure only the analysis of the benchmark
+  INSTRUMENT_HOOKS_COMMIT: "8fd5b878499f819561b33219cdafaa6f8c28b4e4"
+  # Each shard is one runner, see tests/bench/shard.php
+  BENCHMARK_SHARDS: "4"
+
+jobs:
+  shards:
+    name: "Split benchmarks into shards"
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    outputs:
+      matrix: ${{ steps.shards.outputs.matrix }}
+
+    steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3
+        with:
+          egress-policy: audit
+
+      - name: "Checkout"
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962" # 2.40.0
+        with:
+          coverage: "none"
+          php-version: "8.5"
+
+      - uses: "ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda" # v4.0.0
+
+      - name: "Split tests/bench/data"
+        id: shards
+        run: echo "matrix=$(php tests/bench/shard.php "$BENCHMARK_SHARDS")" >> "$GITHUB_OUTPUT"
+
+  run:
+    name: "Benchmarks shard ${{ matrix.shard }}"
+    needs: shards
+    # Pinned because the instruction counts depend on the PHP build and the system libraries
+    runs-on: ubuntu-24.04
+    # The slowest shard takes about 20 minutes, and run.php stops a benchmark after 25
+    timeout-minutes: 40
+
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.shards.outputs.matrix) }}
+
+    steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3
+        with:
+          egress-policy: audit
+
+      - name: "Checkout"
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962" # 2.40.0
+        with:
+          coverage: "none"
+          php-version: "8.5"
+          extensions: ds,ffi,mbstring
+          ini-file: production
+          ini-values: memory_limit=-1
+
+      - uses: "ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda" # v4.0.0
+
+      - name: "Install CodSpeed"
+        run: |
+          curl -fsSL "https://github.com/CodSpeedHQ/codspeed/releases/download/v${CODSPEED_VERSION}/codspeed-runner-installer.sh" | bash
+          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
+          "$HOME/.cargo/bin/codspeed" setup --mode=simulation
+
+      - name: "Build CodSpeed's instrument-hooks"
+        run: |
+          git init -q "$RUNNER_TEMP/instrument-hooks"
+          cd "$RUNNER_TEMP/instrument-hooks"
+          git fetch -q --depth=1 https://github.com/CodSpeedHQ/instrument-hooks.git "$INSTRUMENT_HOOKS_COMMIT"
+          git checkout -q FETCH_HEAD
+          gcc -shared -fPIC -O2 -Iincludes -w dist/core.c -o libinstrument_hooks.so
+          echo "INSTRUMENT_HOOKS_LIBRARY=$PWD/libinstrument_hooks.so" >> "$GITHUB_ENV"
+
+      - name: "Run benchmarks"
+        env:
+          BENCHMARKS: ${{ matrix.benchmarks }}
+        # Valgrind runs each benchmark on one CPU, so the shard runs as many at once as the runner has
+        run: |
+          mkdir -p tmp/bench-results
+          tr ' ' '\n' <<< "$BENCHMARKS" | xargs -P "$(nproc)" -I {} php tests/bench/run.php "tests/bench/data/{}" "tmp/bench-results/{}.neon"
+
+      - name: "Upload results"
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: benchmark-results-${{ matrix.shard }}
+          path: tmp/bench-results/*.neon
+          if-no-files-found: ignore
+
+  compare:
+    name: "Compare with expected.neon"
+    needs: run
+    if: ${{ !cancelled() }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3
+        with:
+          egress-policy: audit
+
+      - name: "Checkout"
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: "Install PHP"
+        uses: "shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962" # 2.40.0
+        with:
+          coverage: "none"
+          php-version: "8.5"
+
+      - uses: "ramsey/composer-install@65e4f84970763564f46a70b8a54b90d033b3bdda" # v4.0.0
+
+      - name: "Download results"
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          pattern: benchmark-results-*
+          merge-multiple: true
+          path: tmp/bench-results
+
+      - name: "Compare"
+        run: "php tests/bench/compare.php tmp/bench-results tmp/expected.neon"
+
+      - name: "Upload measured metrics"
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: benchmark-measured
+          path: tmp/expected.neon

.github/workflows/tests.yml

@@ -260,7 +260,7 @@ jobs:
 
       - name: "Downgrade PHPUnit with Paratest"
         shell: bash
-        run: "composer require --dev phpbench/phpbench:^1.2.15 phpunit/phpunit:^9.6 brianium/paratest:^6.5 symfony/console:^5.4 symfony/process:^5.4 doctrine/instantiator:^1.0 --update-with-dependencies --ignore-platform-reqs --working-dir=tests"
+        run: "composer require --dev phpunit/phpunit:^9.6 brianium/paratest:^6.5 symfony/console:^5.4 symfony/process:^5.4 doctrine/instantiator:^1.0 --update-with-dependencies --ignore-platform-reqs --working-dir=tests"
 
       - uses: ./.github/actions/downgrade-code
         with:

.gitignore

@@ -11,10 +11,8 @@
 /tests/tmp
 /tests/.phpunit.result.cache
 /tests/PHPStan/Reflection/data/golden/
-/tests/bench/storage/local-baseline.xml
 tmp/.memory_limit
 e2e/bashunit
-/.phpbench
 /.amp/portals/
 /e2e/turbo-arena/tmp
 /e2e/fork-unsafe-extension/tmp

CLAUDE.md

@@ -294,6 +294,7 @@ New introduced classes in these categories should be shadowed with a native C++
 - **Always verify the test fails before the fix.** Stating "tests pass" is not enough — that only proves the fix doesn't break things. `git stash` the source change, run the test, see it fail for the right reason; `git stash pop`, run again, see it pass. Only then is the fix confirmed. The `regression-test` skill bakes this in for bug-issue workflows.
 - **Reproducers come from the OP's playground link** in the issue body, not from later comments. Copy the snippet as close to verbatim as practical.
 - **Integration tests**: `AnalyserIntegrationTest` runs full analysis on test files and checks error output.
+- **Benchmarks**: every file in `tests/bench/data` is analysed in the Benchmarks workflow under CodSpeed's CPU simulation, which counts instructions instead of measuring time, so the numbers are the same on every runner. `tests/bench/measure.php` boots PHPStan and analyses a warm-up file first, so only the analysis of the benchmark file itself is counted. `tests/bench/compare.php` compares them against `tests/bench/expected.neon`; when a change moves them on purpose or adds a benchmark, replace that file with the workflow's `benchmark-measured` artifact.
 
 ### Adding support for new PHP versions
 
@@ -311,7 +312,7 @@ Recent work on PHP 8.5 support shows the pattern:
 
 - **One fix, one commit (or one PR)**. Bench scripts and tests for a fix go in the same commit as the source change, not separately.
 - **Commit messages and PR titles describe the change, not the bug they close.** Prefer "Do not subtract TemplateType from TemplateType" over "Fix #14459: type subtraction". Issue closure goes in the PR body via `Closes https://github.com/phpstan/phpstan/issues/<n>`.
-- **Standard reproducer command**: `bin/phpstan analyse -l 8 <file> --debug` (add `-vvv` for hangs and infinite loops). Files under `tests/bench/data` are not directly runnable as PHPStan inputs — copy the reproducing snippet into a `test.php` at the repo root first, then analyse that.
+- **Standard reproducer command**: `bin/phpstan analyse -l 8 <file> --debug` (add `-vvv` for hangs and infinite loops). Files under `tests/bench/data` are excluded by the default config — analyse them with the benchmarks' own config: `bin/phpstan analyse -c tests/bench/phpstan.neon --debug <file>`.
 - **Debug helpers in analysed code**: `\PHPStan\dumpType($expr)` prints the inferred type of an expression at that point; `\PHPStan\debugScope()` prints the current scope. Inside `NodeScopeResolver` itself, `var_dump($scope->debug())` is the canonical inspection point. Reach for these before guessing what PHPStan is doing.
 - **When the user says "I don't understand why X is needed"** about existing code, treat it as a request to investigate whether X is actually needed — not a request to defend the status quo. The answer is often that two paths can be unified or one block deleted.
 

Makefile

@@ -15,7 +15,6 @@ lint:
 	XDEBUG_MODE=off php vendor/bin/parallel-lint --colors -e php,php3,php4,php5,phtml,phpt,fixed \
 		--exclude tests/PHPStan/Analyser/data \
 		--exclude tests/bench/data \
-		--exclude tests/bench/RegressionBench.php \
 		--exclude tests/PHPStan/Analyser/nsrt \
 		--exclude tests/PHPStan/Rules/Methods/data \
 		--exclude tests/PHPStan/Rules/Functions/data \
@@ -212,15 +211,6 @@ infection:
 		--ignore-msi-with-no-mutations \
 		--logger-text=php://stdout
 
-.PHONY: phpbench
-phpbench:
-	@find tests/bench/storage/local-baseline.xml -mtime -60m | grep . || (echo "PHPBench baseline file does not exist or is too old. Regenerate it using 'make phpbench-baseline'." && exit 1)
-	XDEBUG_MODE=off tests/vendor/bin/phpbench run --file=tests/bench/storage/local-baseline.xml --report=my-report
-
-.PHONY: phpbench-baseline
-phpbench-baseline:
-	XDEBUG_MODE=off tests/vendor/bin/phpbench run --dump-file=tests/bench/storage/local-baseline.xml
-
 download-coverage:
 	rm -rf tmp/coverage-download
 	mkdir tmp/coverage-download

build/downgrade.php

@@ -6,6 +6,7 @@
 		__DIR__ . '/../build/PHPStan',
 		__DIR__ . '/../src',
 		__DIR__ . '/../tests/PHPStan',
+		__DIR__ . '/../tests/bench',
 		__DIR__ . '/../tests/e2e',
 	],
 	'excludePaths' => [

build/phpstan.neon

@@ -28,12 +28,12 @@ parameters:
 	reportPossiblyNonexistentConstantArrayOffset: true
 	excludePaths:
 		- ../tests/*/data/*
+		- ../tests/bench/Warmup.php
 		- ../tests/tmp/*
 		- ../tests/vendor/*
 		- ../tests/PHPStan/Analyser/nsrt/*
 		- ../tests/PHPStan/Analyser/traits/*
 		- ../tests/notAutoloaded/*
-		- ../tests/bench/*
 		- ../tests/PHPStan/Reflection/CircularReferenceClassReflectionTest.php
 		- ../tests/PHPStan/Reflection/UnionTypesTest.php
 		- ../tests/PHPStan/Reflection/MixedTypeTest.php
@@ -90,6 +90,7 @@ parameters:
 			paths:
 				- ../src/File/FsEventsFileMonitor.php
 				- ../src/File/InotifyFileMonitor.php
+				- ../tests/bench/measure.php
 		-
 			identifier: shipmonk.deadMethod
 			message: '#^Unused .*?::__construct#' # likely used in DIC