the nightly benchmark stand: one box, every [benchmark] in the tree, plotted over commits

.github/workflows/nightly_bench.yml

@@ -0,0 +1,149 @@
+name: nightly benchmark stand
+
+# Drives the benchmark stand on the web box (dasweb-1, the daslang.io origin) and
+# publishes the night at https://daslang.io/bench/. The box does the work - checkout,
+# Release build, every `[benchmark]` file in benchmarks/ in the interpreter and JIT
+# lanes, the report - so the numbers come from ONE machine night after night; this
+# runner only kicks it off over ssh, streams the log, and posts the summary.
+#
+# Red means the night is not a usable data point: the build failed, or at least one
+# benchmark file failed (compile error, failed assertion, crash, timeout, memory cap).
+# A regression is a warning in the step summary and on the site, not a red - flip
+# `fail_on_regression` on a dispatch to make it one.
+#
+# No self-hosted runner: this repository is public, and a self-hosted runner would
+# execute pull-request code on the production web box. The key reaches only the
+# stand's gate verbs - start, follow, status, summary - never a shell (a forced
+# command in the box's authorized_keys).
+
+on:
+  schedule:
+    # 05:00 UTC - after nightly_playground's 04:30, while the box is otherwise idle.
+    - cron: '0 5 * * *'
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'commit or branch to benchmark (default: master)'
+        required: false
+        default: 'master'
+      fail_on_regression:
+        description: 'turn a regression into a red run'
+        required: false
+        default: 'false'
+
+defaults:
+  run:
+    shell: bash
+
+# One night at a time: the box holds a run lock, so a second start would only be refused.
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
+
+# Empty on purpose: every step of the one job talks to the box over ssh. Nothing checks the
+# repository out and nothing calls the GitHub API, so the token needs no scope at all.
+permissions: {}
+
+jobs:
+  bench:
+    # The cron only runs on the canonical repo: it drives daslang.io's box, which a fork
+    # neither owns nor holds the key to, and a red nightly emails the fork owner.
+    if: github.event_name != 'schedule' || github.repository == 'GaijinEntertainment/daScript'
+    runs-on: ubuntu-latest
+    # Build (~20-40 min on the box) + two lanes of the suite (~1-2 h) + report.
+    timeout-minutes: 360
+    # SECRET SCOPE, like pages.yml: DASWEB_BENCH_KEY is an environment secret on
+    # `github-pages` (the repository has no repo-level secrets).
+    environment:
+      name: github-pages
+    env:
+      BOX: 89.167.63.131
+      BOX_USER: bench
+    steps:
+      - name: "Set up the ssh key"
+        id: key
+        env:
+          BENCH_KEY: ${{ secrets.DASWEB_BENCH_KEY }}
+        run: |
+          set -euo pipefail
+          # A fork has no access to the secret and legitimately skips, the way pages.yml's deploy
+          # step does; on the canonical repo an empty key means the nightly silently stopped
+          # measuring, so it is loud there.
+          if [ -z "$BENCH_KEY" ]; then
+              if [ "$GITHUB_REPOSITORY" = "GaijinEntertainment/daScript" ]; then
+                  echo "::error::DASWEB_BENCH_KEY is empty on the canonical repo - the stand was not driven."
+                  echo "It is an ENVIRONMENT secret on 'github-pages'; check the job still declares that environment."
+                  exit 1
+              fi
+              echo "DASWEB_BENCH_KEY not set (fork) - skipping the run"
+              echo "skipped=1" >> "$GITHUB_OUTPUT"
+              exit 0
+          fi
+          mkdir -p ~/.ssh
+          printf '%s\n' "$BENCH_KEY" > ~/.ssh/bench_key
+          chmod 600 ~/.ssh/bench_key
+          # Pinned host key - the same box and key pages.yml deploys to; no trust-on-first-use.
+          echo "$BOX ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILP2wzOeEGD2r7apldQlTe2gwR1dVTCXCfrD+pJooMmY" >> ~/.ssh/known_hosts
+          echo "RSH=ssh -i $HOME/.ssh/bench_key -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ConnectTimeout=30" >> "$GITHUB_ENV"
+
+      - name: "Start the night on the box"
+        id: start
+        if: steps.key.outputs.skipped != '1'
+        env:
+          REF: ${{ github.event.inputs.ref || 'master' }}
+          FAIL_ON_REGRESSION: ${{ github.event.inputs.fail_on_regression || 'false' }}
+        run: |
+          set -euo pipefail
+          # The forced command takes `start <ref> [fail-on-regression]`, and that second word is
+          # the ONLY way the input reaches the box: a forced command carries no environment, so
+          # exporting BENCH_STAND_FAIL_ON_REGRESSION here would reach nothing. A busy box answers
+          # exit 3 and names the run it is still working on.
+          FLAG=""
+          [ "$FAIL_ON_REGRESSION" = "true" ] && FLAG=" fail-on-regression"
+          RUN_ID=$($RSH "$BOX_USER@$BOX" "start $REF$FLAG")
+          echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT"
+          echo "started run $RUN_ID for $REF (regression fails the night: $FAIL_ON_REGRESSION)"
+
+      - name: "Follow the run"
+        if: steps.key.outputs.skipped != '1'
+        run: |
+          set -uo pipefail
+          # An ssh drop (exit 255) reconnects rather than failing the lane; every other code is
+          # the run's own, recorded for the summary step to raise.
+          for attempt in $(seq 1 60); do
+              $RSH "$BOX_USER@$BOX" "follow ${{ steps.start.outputs.run_id }}"
+              rc=$?
+              if [ $rc -ne 255 ]; then
+                  echo "run finished with exit $rc"
+                  echo "$rc" > run_exit
+                  exit 0
+              fi
+              echo "ssh dropped (attempt $attempt) - reconnecting"
+              sleep 30
+          done
+          echo "::error::could not stay connected to the box"
+          echo 255 > run_exit
+
+      - name: "Post the summary"
+        # always(), so a night that ended badly still publishes its summary here. The step's own
+        # exit code is the lane's verdict: the run's code when there is one, and a red when the
+        # summary could not be fetched at all - an unfetched summary is the one artifact this
+        # lane exists to deliver, so swallowing it would leave a green lane saying nothing.
+        if: always() && steps.key.outputs.skipped != '1'
+        run: |
+          set -uo pipefail
+          summary_rc=0
+          $RSH "$BOX_USER@$BOX" "summary" > summary.md || summary_rc=$?
+          if [ $summary_rc -ne 0 ] || [ ! -s summary.md ]; then
+              echo "::error::could not fetch the night's summary from the box (ssh exit $summary_rc)"
+              echo "The night's own verdict, if it reached one, is below." >> "$GITHUB_STEP_SUMMARY"
+          fi
+          [ -s summary.md ] && cat summary.md >> "$GITHUB_STEP_SUMMARY"
+          rc=$(cat run_exit 2>/dev/null || echo 1)
+          case "$rc" in
+              0) echo "night ok" ;;
+              2) echo "::error::the night regressed and fail_on_regression is on - see the summary" ;;
+              *) echo "::error::the night failed (exit $rc) - see the summary and https://daslang.io/bench/" ;;
+          esac
+          [ "$rc" = 0 ] && exit "$summary_rc"
+          exit "$rc"

CMakeLists.txt

@@ -178,8 +178,17 @@ MACRO(DAS_AOT_EXT input_files genList mainTarget dasAotTool dasAotToolArg)
     set(all_depends ${dasAotTool} ${PROJECT_SOURCE_DIR}/utils/aot/main.das ${DAS_AOT_DASLIB_DEPENDS} ${DAS_AOT_EXTRA_DEPENDS})
     set(all_outputs "")
     # Per-batch arg buffer; chunked to keep each invocation under Windows'
-    # 32K command-line limit when worktree paths are long.
-    set(_DAS_AOT_BATCH_SIZE 60)
+    # 32K command-line limit when worktree paths are long. A caller may set
+    # DAS_AOT_BATCH_SIZE to something smaller: one tool run compiles every file it
+    # is given in one process, so a file whose require graph installs macros can
+    # perturb the ones after it in its batch (error[20600] / error[30100] on an
+    # annotation that exists). A batch of 1 costs a process per file and makes that
+    # impossible.
+    if(DEFINED DAS_AOT_BATCH_SIZE)
+        set(_DAS_AOT_BATCH_SIZE ${DAS_AOT_BATCH_SIZE})
+    else()
+        set(_DAS_AOT_BATCH_SIZE 60)
+    endif()
     set(_batches "")
     set(_batch_idx 0)
     set(_batch_count 0)

benchmarks/decs/bench_from_decs_count.das

@@ -20,7 +20,7 @@ struct BenchCountRow {
 
 def fixture(n : int) {
     restart()
-    create_entities(n) $(eid : EntityId; i : int; var cmp : ComponentMap) {
+    create_entities(n) $(_eid : EntityId; i : int; var cmp : ComponentMap) {
         apply_decs_template(cmp, BenchCountRow(val = i))
     }
 }
@@ -32,7 +32,7 @@ def from_decs_count_m1_hand(b : B?) {
     b |> run("m1_hand_arch_size/{N}", N) {
         var erq : EcsRequest
         erq.req |> push("bench_val")
-        var total = 0
+        var total = 0l
         for_each_archetype(erq) $(arch : Archetype) {
             total += arch.size
         }

skills/internal/preflight.md

@@ -62,6 +62,7 @@ working-tree copy.
 |---|---|---|
 | `build.yml` (per-PR) | every PR commit (`pull_request`) + pushes to `master` | `build` matrix (`ci/ci_matrix.py build`: Debug + Release on linux, linux_arm, darwin15, darwin26; windows 32 Release, windows 64 Release), `bundle_smoke`, `build_linux_gcc` |
 | `build.yml` (nightly) | `schedule` cron (daily 02:00 UTC) | `build_windows_mingw` + `build_windows_clangcl` (gated OFF per-PR) **plus the full build matrix - the per-PR cells, the sanitizer cells (linux Release asan/tsan/ubsan), the fast-math cell (linux Release `-DDAS_FAST_MATH=ON`) and windows 64 Debug - whose Release cells run the full AOT sweep** ("Slow Release Tests"). Breaks surface within ~24 h, not at PR time |
+| `nightly_bench.yml` | `schedule` cron (daily 05:00 UTC) + `workflow_dispatch` (`ref`, `fail_on_regression`) | one job that drives the benchmark stand on the daslang.io box over ssh (`utils/internal/bench-stand/nightly.sh`): Release build of `ref` plus `test_aot`, then every `benchmarks/**/*.das` its `suite.json` does not exclude or skip, in the interp, jit and aot lanes, the report to https://daslang.io/bench/; red = build or any benchmark file failed. Local mirror: `bin/daslang utils/internal/bench-stand/main.das -- run --meta <meta.json> --out <record> --filter <substr>` then `... -- report --runs <dir> --out-data <data.json> --out-summary <summary.md>` (README there) |
 | `nightly_imgui.yml` | `schedule` cron (daily 03:00 UTC) + `workflow_dispatch` | dasImgui playwright suite on ubuntu + macos - section below |
 | `extended_checks.yml` (per-PR) | every PR | two darwin15-arm64 jobs, `core` and `modules` (`ci/ci_matrix.py extended`), ALL release modules ON - section below |
 | `extended_checks.yml` (nightly) | `schedule` cron (daily 04:00 UTC) + `workflow_dispatch` | one job each on linux, darwin15 and windows running every step (role `all`), including the ones too slow for a PR: tutorial dry-runs, the run form of examples, coverage, the nano cross-compile, the AST verify tree sweep, doc-verify |

tests/aot/CMakeLists.txt

@@ -40,6 +40,26 @@ FILE(GLOB AOT_TESTS_FILES RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS "test
 # Exclude _-prefixed helper modules (required transitively, not standalone test entry points)
 list(FILTER AOT_TESTS_FILES EXCLUDE REGEX "/_")
 
+# The nightly benchmark stand's AOT lane (utils/internal/bench-stand): the benchmark bodies get
+# their stubs linked into test_aot, so `test_aot -use-aot dastest/dastest.das -- --use-aot --bench`
+# measures native code instead of the interpreter. Predefined rather than globbed by the loop
+# below, because the tree is nested and the loop's convention is tests/<name>/*.das.
+FILE(GLOB_RECURSE AOT_BENCHMARKS_FILES RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS "benchmarks/*.das")
+# _-prefixed helper modules are required transitively, never standalone; benchmarks/sql/tests is
+# a dastest suite, not a benchmark.
+list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "/_")
+list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "^benchmarks/sql/tests/")
+# Suites whose require graph reaches a native module: each would need that module's own AOT half
+# linked here, so they measure in the interpreter and JIT lanes only. suite.json states the
+# reason per file, and widening this set is what lets a lane skip go away.
+list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "^benchmarks/(audio|terminal|sql)/")
+list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "benchmarks/core/math/scalar_crt\\.das$")
+list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "benchmarks/micro/(join_select|single_last|sort_distinct_take)_shapes\\.das$")
+# benchmarks/core/jit_aot rides the language suite on purpose (see AOT_JIT_BENCH_FILES below), so
+# its stubs are already in both AOT binaries. Registering it twice emits two TUs for one file and
+# the anonymous-module namespaces collide at link (multiple definition of das::_anon_<hash>).
+list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "^benchmarks/core/jit_aot/")
+
 FILE(GLOB AOT_BARE_BLOCK_FILES RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS "tests/bare_block/*.das")
 list(FILTER AOT_BARE_BLOCK_FILES EXCLUDE REGEX "failed_|cant_|invalid_")
 
@@ -394,7 +414,7 @@ set(DAS_AOT_SUITES
     jobque json jsonrpc language linq lint long_array_table loops lpipe lsp
     macro_boost macro_call match math mcp md_boost module_cache module_tests option promote quote
     reader_macro regex rtti safe_addr soa spoof strings stbimage table_packed template
-    tests type_lattice type_traits typemacro uri with_boost delegate)
+    tests type_lattice type_traits typemacro uri with_boost delegate benchmarks)
 foreach(_s IN LISTS DAS_AOT_SUITES)
     string(TOUPPER ${_s} _u)
     # suites with an irregular dir / a filter / a curated list define AOT_<UC>_FILES above;
@@ -405,9 +425,18 @@ foreach(_s IN LISTS DAS_AOT_SUITES)
     add_custom_target(test_aot_${_s})
     set(${_u}_AOT_GENERATED_SRC)
     list(LENGTH AOT_${_u}_FILES _nf)
+    # One process per benchmark: ten of the core/hash bodies install macros that make a LATER
+    # file in the same batch fail on `[benchmark]` (error[20600]). Batching is a speed knob, so
+    # the suite that trips it pays a process per file instead of losing the files.
+    if(_s STREQUAL "benchmarks")
+        set(DAS_AOT_BATCH_SIZE 1)
+    else()
+        unset(DAS_AOT_BATCH_SIZE)
+    endif()
     if(_nf GREATER 0)
         DAS_AOT("${AOT_${_u}_FILES}" ${_u}_AOT_GENERATED_SRC test_aot_${_s} daslang)
     endif()
+    unset(DAS_AOT_BATCH_SIZE)
     list(APPEND TEST_AOT_TARGETS test_aot_${_s})
     list(APPEND TEST_AOT_GENVARS ${_u}_AOT_GENERATED_SRC)
     list(APPEND TEST_AOT_ALL_DAS ${AOT_${_u}_FILES})

utils/CMakeLists.txt

@@ -223,6 +223,7 @@ SET(DAS_UTILS_TO_TEST
     internal/hygiene
     internal/preflight/tests
     internal/ast-fuzz
+    internal/bench-stand
     internal/requirefix
     internal/test-release
     daspkg/test_daspkg.das