ci: generate the linux engine definitions natively instead of through docker

.github/workflows/ci.yml

@@ -470,7 +470,7 @@ jobs:
       # restore/save rather than the combined action: on an exact hit every layer
       # in the directory is still valid, so there is nothing to write back and the
       # `--cache-to` export below can be skipped outright. That export is not free
-      # - the header-drift job measures it at 13-15s per target - and today it is
+      # - the generator's docker path measured it at 13-15s per target - and today it is
       # paid on every run only to be thrown away. The prefix keys still give a warm
       # start when the base image moved, which is what keeps that rebuild
       # incremental instead of a compile from zero.
@@ -534,98 +534,62 @@ jobs:
           docker run --rm -v "$PWD:/app" -w /app z-engine-php:debug \
             vendor/bin/phpunit ${{ matrix.opcache_args }}
 
+  # Regenerates the linux-x64-{nts,zts} artifacts natively on the runner - the
+  # same setup-php build the tests jobs run on - and diffs the committed ones
+  # against the result, exactly like the darwin and windows jobs below.
+  #
+  # Docker (`composer gen-headers`, the local developer path) cannot be the CI
+  # oracle while the minor is pre-release: docker-library ships the
+  # `php:<minor>-rc-cli` image as a ZTS build (there is no separate nts
+  # variant yet), so the nts target came out as a ZTS artifact and this job
+  # was red with a diff nobody could commit. generate.php now refuses such a
+  # mismatched image, and CI generates for the interpreter it actually runs.
+  #
+  # The nts leg is the canonical build: it also publishes stubs/ and
+  # .phpstorm.meta.php, so only that leg diffs them.
   header-drift:
-    name: Generated headers up to date
+    name: Generated linux headers up to date (${{ matrix.ts }})
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ts: [nts, zts]
     steps:
       - uses: actions/checkout@v7
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v4
-      # Caches the generator image's toolchain layers (apt/clang/ext-ffi); the
-      # emit stage itself always re-runs - it is the step that produces the
-      # artifacts being drift-checked. generate.php rotates per-target subdirs.
-      #
-      # Both base images go into the key: this job builds one image per target
-      # and nts starts from `php:<minor>-cli` while zts starts from
-      # `php:<minor>-zts`, so either tag moving invalidates the toolchain layer.
-      # Without them the entry freezes at whatever the first run produced and can
-      # never be rewritten - which is the state this job was already in, paying
-      # ~16s to reinstall clang and ~15s to export a cache nobody would store, on
-      # every single run. See the same reasoning in tests-internal-debug.
-      - name: Resolve the base image digests
-        id: gen-cache-key
-        run: |
-          # Same helper as tests-internal-debug: prints the raw manifest only on
-          # success, so a failed attempt cannot feed half a response into the hash.
-          resolve() {
-            local raw
-            for attempt in 1 2 3; do
-              if raw="$(docker buildx imagetools inspect "$1" --raw)" && [ -n "$raw" ]; then
-                printf '%s' "$raw"
-                return 0
-              fi
-              sleep "$((attempt * 3))"
-            done
-            return 1
-          }
-          # While the minor is pre-release Docker publishes only the -rc family
-          # (which also covers betas); generate.php falls back the same way, so
-          # the key hashes whichever manifests the build will actually use
-          resolve_variant() {
-            resolve "php:${PHP_MINOR}-$1" || resolve "php:${PHP_MINOR}-rc-$1"
-          }
-          if cli="$(resolve_variant cli)" && zts="$(resolve_variant zts)"; then
-            digest="$(printf '%s%s' "$cli" "$zts" | sha256sum | cut -d' ' -f1)"
-            echo "php:${PHP_MINOR}[-rc]-cli + php:${PHP_MINOR}[-rc]-zts hash to ${digest}"
-          else
-            # As above: a registry hiccup costs an exact key, never the job.
-            digest="unresolved-${{ github.run_id }}"
-            echo "::warning::Could not resolve the php:${PHP_MINOR} base image digests - building without an exact cache key"
-          fi
-          prefix="docker-gen-${PHP_MINOR}-${{ runner.os }}-${{ runner.arch }}-"
-          {
-            echo "prefix=${prefix}"
-            echo "recipe=${prefix}${{ hashFiles('tools/generator/Dockerfile') }}-"
-            echo "key=${prefix}${{ hashFiles('tools/generator/Dockerfile') }}-${digest}"
-          } >> "$GITHUB_OUTPUT"
 
-      - name: Restore the generator image layer cache
-        id: gen-layer-cache
-        uses: actions/cache/restore@v6
-        with:
-          path: /tmp/.buildx-gen-cache
-          key: ${{ steps.gen-cache-key.outputs.key }}
-          restore-keys: |
-            ${{ steps.gen-cache-key.outputs.recipe }}
-            ${{ steps.gen-cache-key.outputs.prefix }}
-            docker-gen-${{ env.PHP_MINOR }}-
-
-      # On an exact hit the restored layers are still valid, so generate.php is
-      # told to read the cache without exporting it again - the export is the
-      # expensive half and there would be nowhere to put the result.
-      - name: Regenerate engine definitions
-        run: composer gen-headers
+      - name: Set up PHP
+        uses: shivammathur/setup-php@v2
         env:
-          Z_ENGINE_BUILDX_CACHE_DIR: /tmp/.buildx-gen-cache
-          Z_ENGINE_BUILDX_CACHE_READONLY: ${{ steps.gen-layer-cache.outputs.cache-hit == 'true' && '1' || '' }}
-
-      - name: Save the generator image layer cache
-        if: steps.gen-layer-cache.outputs.cache-hit != 'true'
-        uses: actions/cache/save@v6
+          phpts: ${{ matrix.ts == 'zts' && 'ts' || 'nts' }}
         with:
-          path: /tmp/.buildx-gen-cache
-          key: ${{ steps.gen-cache-key.outputs.key }}
+          php-version: ${{ env.PHP_MINOR }}
+          extensions: ffi
+          ini-values: ffi.enable=1
+          coverage: none
+
+      - name: Assert the thread-safety mode matches the matrix leg
+        run: |
+          php -r 'exit(ZEND_THREAD_SAFE === ("'"${{ matrix.ts }}"'" === "zts") ? 0 : 1);' \
+            || { echo "::error::PHP thread-safety mode does not match the ${{ matrix.ts }} matrix leg"; exit 1; }
+
+      - name: Regenerate engine definitions natively
+        run: php tools/generator/generate.php --native --php=${{ env.PHP_MINOR }} --ts=${{ matrix.ts }}
+
       - name: Fail if the committed artifacts are stale
         run: |
-          if ! git diff --exit-code -- include/ stubs/ .phpstorm.meta.php; then
-            echo "::error::Generated engine definitions (or the struct stubs / phpstorm meta) are out of date. Run 'composer gen-headers' and commit the result."
+          paths="include/${PHP_MINOR}/linux-x64-${{ matrix.ts }}"
+          if [ "${{ matrix.ts }}" = nts ]; then
+            paths="${paths} stubs/ .phpstorm.meta.php"
+          fi
+          # shellcheck disable=SC2086 # the variable is a deliberate word list
+          if ! git diff --exit-code -- ${paths}; then
+            echo "::error::Generated linux engine definitions (or the struct stubs / phpstorm meta) are out of date. Run the 'Generate linux headers' workflow and merge its commit, or 'composer gen-headers' locally."
             exit 1
           fi
 
   # The darwin counterpart of header-drift: regenerates natively on a macOS
   # runner (docker cannot produce darwin artifacts) and diffs only the dir the
-  # runner's own architecture can regenerate. The linux job above stays safe
-  # unscoped - linux regeneration never touches the darwin directories.
+  # runner's own architecture can regenerate.
   header-drift-darwin:
     name: Generated darwin headers up to date (${{ matrix.runner-arch.arch }}, ${{ matrix.ts }})
     runs-on: ${{ matrix.runner-arch.runner }}

.github/workflows/generate-linux-headers.yml

@@ -0,0 +1,173 @@
+name: Generate linux headers
+
+# Generates the linux-x64-{nts,zts} FFI engine definitions natively on Linux
+# runners (the setup-php build of this branch's PHP minor - the same one the
+# tests jobs run on) and commits them back to the branch, together with the
+# branch-level stubs the canonical nts build publishes (stubs/ and
+# .phpstorm.meta.php).
+#
+# The local developer path is `composer gen-headers`, which builds inside the
+# official docker image. That path is not available to CI while the minor is
+# pre-release: docker-library ships the `php:<minor>-rc-cli` image as a ZTS
+# build (no separate nts variant), so the nts target cannot come out of it -
+# generate.php refuses such an image. This workflow is the Linux counterpart
+# of the darwin/windows ones and the canonical way to refresh the linux
+# artifacts without Docker.
+#
+# Triggers:
+# - pull_request touching the generator: refreshes the artifacts on the PR
+#   branch itself (same-repo PRs only - fork PRs get the uploaded artifacts
+#   but no push).
+# - workflow_dispatch: manual refresh; run it against the branch whose
+#   artifacts need regenerating.
+#
+# A push made with GITHUB_TOKEN does not retrigger workflows, so the commit
+# job cannot loop; the next human push re-runs full CI against the artifacts.
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'tools/generator/**'
+      - '.github/workflows/generate-linux-headers.yml'
+
+permissions:
+  contents: write
+
+# This workflow commits and pushes to the target branch, so two runs against
+# the same branch must never overlap (they would race on the push and rebase).
+# Queue them instead of cancelling: a cancelled run can leave the artifacts
+# half-refreshed, and the commit job is the only thing that publishes them.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}
+  cancel-in-progress: false
+
+# This branch targets PHP 8.6 only - engine structures are version-specific.
+env:
+  PHP_MINOR: '8.6'
+
+jobs:
+  generate:
+    name: Generate linux-x64-${{ matrix.ts }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ts: [nts, zts]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}
+
+      - name: Set up PHP
+        uses: shivammathur/setup-php@v2
+        env:
+          phpts: ${{ matrix.ts == 'zts' && 'ts' || 'nts' }}
+        with:
+          php-version: ${{ env.PHP_MINOR }}
+          extensions: ffi, opcache
+          ini-values: ffi.enable=1
+          coverage: none
+
+      - name: Assert the build matches the matrix leg (Linux x64, ${{ matrix.ts }})
+        run: |
+          php -r 'exit(PHP_OS_FAMILY === "Linux" && php_uname("m") === "x86_64" ? 0 : 1);' \
+            || { echo "::error::expected a Linux x86_64 build"; exit 1; }
+          php -r 'exit(ZEND_THREAD_SAFE === ("${{ matrix.ts }}" === "zts") ? 0 : 1);' \
+            || { echo "::error::PHP thread-safety mode does not match the ${{ matrix.ts }} matrix leg"; exit 1; }
+
+      # engine.h declares no FFI_LIB: symbols must resolve from the running
+      # php process image. Prove that up front so a resolution problem fails
+      # here with a clear message instead of deep inside the validation stage.
+      - name: FFI symbol-resolution preflight
+        run: |
+          php -d ffi.enable=1 -r '$e = FFI::cdef("const char *get_zend_version(void);"); echo $e->get_zend_version(), PHP_EOL;' \
+            || { echo "::error::FFI cannot resolve engine symbols from the php process image on this build"; exit 1; }
+
+      - name: Generate linux artifacts
+        run: php tools/generator/generate.php --native --php=${{ env.PHP_MINOR }} --ts=${{ matrix.ts }}
+
+      # On failure, surface the intermediate build dir (emitted engine.h,
+      # preprocessed source, probe) so the broken declaration can be diagnosed
+      # from the run page.
+      - name: Stash the build directory for diagnosis
+        if: failure()
+        run: |
+          mkdir -p "$RUNNER_TEMP/gen-debug"
+          cp -R "${TMPDIR:-/tmp}/"z-engine-generator-* "$RUNNER_TEMP/gen-debug/" 2>/dev/null || true
+      - uses: actions/upload-artifact@v7
+        if: failure()
+        with:
+          name: gen-debug-linux-x64-${{ matrix.ts }}
+          path: ${{ runner.temp }}/gen-debug
+          if-no-files-found: ignore
+
+      - name: Smoke-check the artifacts
+        run: |
+          dir="include/${PHP_MINOR}/linux-x64-${{ matrix.ts }}"
+          for file in engine.h constants.php layouts.json probe.c; do
+            test -s "${dir}/${file}" || { echo "::error::${dir}/${file} is missing or empty"; exit 1; }
+          done
+          grep -q 'FFI_SCOPE "ZEngine"' "${dir}/engine.h" \
+            || { echo "::error::${dir}/engine.h has no FFI_SCOPE declaration"; exit 1; }
+          if [ "${{ matrix.ts }}" = nts ]; then
+            for file in stubs/zend-engine-structs.php .phpstorm.meta.php; do
+              test -s "${file}" || { echo "::error::the canonical nts build did not publish ${file}"; exit 1; }
+            done
+          fi
+
+      - uses: actions/upload-artifact@v7
+        with:
+          name: linux-x64-${{ matrix.ts }}
+          path: include/${{ env.PHP_MINOR }}/linux-x64-${{ matrix.ts }}
+          if-no-files-found: error
+
+      # The canonical nts build also owns the branch-level stubs; they travel
+      # as one artifact rooted at the repository (their least common ancestor)
+      # so the commit job can drop them straight into place.
+      - uses: actions/upload-artifact@v7
+        if: matrix.ts == 'nts'
+        with:
+          name: linux-stubs
+          path: |
+            stubs/zend-engine-structs.php
+            .phpstorm.meta.php
+          if-no-files-found: error
+
+  # A single commit job (instead of each runner pushing) serializes both
+  # matrix legs into one commit and eliminates the push race between them.
+  commit:
+    name: Commit generated artifacts
+    needs: generate
+    # Pushing requires same-repo credentials; fork PRs stop at the uploads.
+    if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}
+
+      - uses: actions/download-artifact@v8
+        with:
+          pattern: linux-x64-*
+          path: include/${{ env.PHP_MINOR }}/
+
+      - uses: actions/download-artifact@v8
+        with:
+          name: linux-stubs
+          path: .
+
+      - name: Commit and push if the artifacts changed
+        env:
+          TARGET_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref_name }}
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add -- "include/${PHP_MINOR}" stubs .phpstorm.meta.php
+          if git diff --cached --quiet; then
+            echo "Generated artifacts are identical to the committed ones - nothing to do."
+            exit 0
+          fi
+          git commit -m "Generate linux FFI engine definitions for PHP ${PHP_MINOR}"
+          git pull --rebase origin "${TARGET_REF}"
+          git push origin "HEAD:${TARGET_REF}"

AGENTS.md

@@ -71,7 +71,16 @@ composer gen-headers          # all targets for this branch (needs Docker on Lin
 ```
 
 The generator (`tools/generator/`) runs inside the official `php:<minor>` Docker
-image so the artifacts always match a real build. Regenerate whenever you:
+image so the artifacts always match a real build. CI does not use that path,
+though: its `header-drift` job regenerates the linux artifacts natively on the
+runner's setup-php build (the same one the tests run on), and the
+**"Generate linux headers"** workflow (`.github/workflows/generate-linux-headers.yml`)
+refreshes and commits them the same way the darwin/windows workflows do - run it
+against the branch whose artifacts are stale when you have no Docker at hand.
+While a minor is pre-release the docker path may not even be able to produce
+the nts target: docker-library has shipped the `php:<minor>-rc-cli` image as a
+ZTS build, which `generate.php` detects and refuses (it never writes a ZTS
+layout under an nts name). Regenerate whenever you:
 
 - bump the branch to a new PHP minor,
 - add or remove an engine symbol in `tools/generator/symbols.php`,
@@ -115,8 +124,9 @@ the output must be byte-identical to the Docker pipeline's (verified on
 Ubuntu clang-18 vs the trixie image: the emitter normalizes declarations from
 the clang AST, so compiler version does not leak into the artifacts). Only
 then apply the manifest change and regenerate for real. The `header-drift`
-CI jobs re-run the pipeline (Docker for linux, native for darwin and windows)
-and fail on any divergence, so never skip the pre-check.
+CI jobs re-run the pipeline natively on every platform (linux, darwin and
+windows, each on its runner's setup-php build) and fail on any divergence, so
+never skip the pre-check.
 
 ### macOS (darwin) artifacts
 
@@ -224,9 +234,11 @@ clean at level max — do not add to the baseline without good reason.
 
 ### A docker cache key must name the base image, not only the recipe
 
-Two CI jobs build docker images from a `php:<minor>-*` tag — `tests-internal-debug`
-(a full `--enable-debug` PHP compile) and `header-drift` (the generator toolchain) —
-and both wrap the buildx layer cache in `actions/cache`. Those tags **move**: every
+`tests-internal-debug` builds a docker image from a `php:<minor>-*` tag (a full
+`--enable-debug` PHP compile) and wraps the buildx layer cache in `actions/cache`;
+`header-drift` did the same for the generator toolchain until the linux artifacts
+moved to native generation, and its history is the cautionary tale below. Those
+tags **move**: every
 PHP patch release and every Debian security rebuild republishes them under a new
 digest, and buildx correctly invalidates every layer built on top.
 

CONTRIBUTING.md

@@ -35,7 +35,8 @@ composer test:internal
    `AGENTS.md`) — fixes cascade upward, never downward.
 2. Keep new code clean at PHPStan level max and passing `composer cs:check`.
 3. If you change the set of engine symbols the library uses, update
-   `tools/generator/symbols.php` and run `composer gen-headers` (needs Docker).
+   `tools/generator/symbols.php` and run `composer gen-headers` (needs Docker; without it, run the
+   "Generate linux headers" workflow against your branch - see AGENTS.md).
    Never hand-edit anything under `include/`.
 4. Add or update tests. If a struct is dereferenced by your code, make sure it
    is listed in `layout_structs` so its layout is verified against the C

include/8.6/darwin-arm64-nts/constants.php

@@ -138,7 +138,7 @@
     'HASH_ADD_NEXT' => 16,
     'HT_MIN_MASK' => 4294967294,
     'HT_MIN_SIZE' => 8,
-    'ZEND_MODULE_API_NO' => 20250926,
+    'ZEND_MODULE_API_NO' => 20260924,
     'MODULE_PERSISTENT' => 1,
     'MODULE_TEMPORARY' => 2,
     'CONST_CS' => 0,

include/8.6/darwin-arm64-nts/engine.h

@@ -864,6 +864,10 @@ struct _zend_executor_globals {
  HashTable callable_convert_cache;
  HashTable partial_function_application_cache;
  zend_stack lambda_cache;
+ zend_vm_stack vm_stack_page_cache;
+ uint32_t vm_stack_page_cache_count;
+ zend_vm_stack fiber_vm_stack_page_cache;
+ uint32_t fiber_vm_stack_page_cache_count;
  void *reserved[6];
 };
 typedef enum {