From ffa605d5fdeb4c06213c744073a6b05a74d28217 Mon Sep 17 00:00:00 2001 From: Churkin Aleksey Date: Tue, 8 Sep 2026 20:11:31 +0300 Subject: [PATCH] the nightly benchmark stand: one box, every [benchmark] in the tree, plotted over commits utils/internal/bench-stand is the tool - `run` benchmarks a tree into one run record (dastest --bench per file per lane under a wall-clock and RSS cap, the minimum ns/op over repeats as the value), `report` turns every record into the viewer's data.json plus the markdown summary the workflow posts, and its exit code is the night's verdict: 0 ok, 1 a failed build or file, 2 a regression when asked. A regression is a move past both 10% and 3x the baseline's relative MAD over the median of the previous seven runs, so a noisy box cannot manufacture one. Benchmarks group by their directory under benchmarks/, so a new file joins its group by living in the right folder and no manifest tracks names. A night is only worth reading if a red one says what broke, so every failure carries the thing to act on: the first error line of a compile failure, the functions that failed an assertion, the kill reason with the knob that set the limit, the FATAL line of a non-zero exit, and - for a build that never got as far as a benchmark - the tail of the build log, which the summary leads with and the viewer prints inline. A file killed at a limit keeps that verdict whatever the child printed afterwards, and keeps the arms it had finished. nightly.sh is the box side: it checks out the ref, builds Release (RelWithDebInfo arms the C++ allocation tracker, whose exit-time report makes every clean process exit 1), runs the suite, renders the report, and publishes the static viewer at /bench/. A night whose build fails still publishes - the last passing build's binary renders the red night, and the run record of a night that never benchmarked is written by the tool itself, so the record schema has one writer. status.json closes even when the driver dies, so the workflow's follow always returns. bench-stand-deploy.sh provisions the box and splices caddy.snippet, deriving the routes it checks from the snippet rather than copying them. The workflow reaches the box with a key whose authorized_keys line forces `nightly.sh gate`, which answers only start, follow, status and summary, at most two path-character arguments each; asking for a regression to fail the night rides start's second word, because a forced command carries no environment. There is no self-hosted runner: this repository is public, and one would execute pull-request code on the production web box. REVIEW.das is the checklist's mechanical half - key parity across suite.json, SuiteConfig and the README, a skip that states its reason, sibling requires by bare module name, no markup writing in the viewer, every BENCH_STAND_* knob documented in both places, no route literal copied out of the snippet, and a placement line for every file. benchmarks/core/array/test01.das is skipped in suite.json with its reason - it allocates ~19 GB under the persistent heap and gets OOM-killed; a skip is listed every night, so it stays visible debt. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/nightly_bench.yml | 149 +++++ CMakeLists.txt | 13 +- benchmarks/decs/bench_from_decs_count.das | 4 +- skills/internal/preflight.md | 1 + tests/aot/CMakeLists.txt | 31 +- utils/CMakeLists.txt | 1 + utils/internal/bench-stand/README.md | 206 +++++++ utils/internal/bench-stand/REVIEW.das | 254 ++++++++ utils/internal/bench-stand/REVIEW.md | 62 ++ .../bench-stand/bench-stand-deploy.sh | 127 ++++ utils/internal/bench-stand/bench_history.das | 544 ++++++++++++++++ utils/internal/bench-stand/bench_runner.das | 432 +++++++++++++ utils/internal/bench-stand/bench_suite.das | 155 +++++ utils/internal/bench-stand/caddy.snippet | 17 + utils/internal/bench-stand/main.das | 240 ++++++++ utils/internal/bench-stand/nightly.sh | 363 +++++++++++ utils/internal/bench-stand/site/app.js | 581 ++++++++++++++++++ utils/internal/bench-stand/site/index.html | 66 ++ utils/internal/bench-stand/site/style.css | 181 ++++++ utils/internal/bench-stand/suite.json | 32 + utils/internal/bench-stand/test_bench_cli.das | 321 ++++++++++ .../bench-stand/test_bench_history.das | 406 ++++++++++++ .../bench-stand/test_bench_runner.das | 358 +++++++++++ .../internal/bench-stand/test_bench_suite.das | 137 +++++ 24 files changed, 4676 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/nightly_bench.yml create mode 100644 utils/internal/bench-stand/README.md create mode 100644 utils/internal/bench-stand/REVIEW.das create mode 100644 utils/internal/bench-stand/REVIEW.md create mode 100755 utils/internal/bench-stand/bench-stand-deploy.sh create mode 100644 utils/internal/bench-stand/bench_history.das create mode 100644 utils/internal/bench-stand/bench_runner.das create mode 100644 utils/internal/bench-stand/bench_suite.das create mode 100644 utils/internal/bench-stand/caddy.snippet create mode 100644 utils/internal/bench-stand/main.das create mode 100755 utils/internal/bench-stand/nightly.sh create mode 100644 utils/internal/bench-stand/site/app.js create mode 100644 utils/internal/bench-stand/site/index.html create mode 100644 utils/internal/bench-stand/site/style.css create mode 100644 utils/internal/bench-stand/suite.json create mode 100644 utils/internal/bench-stand/test_bench_cli.das create mode 100644 utils/internal/bench-stand/test_bench_history.das create mode 100644 utils/internal/bench-stand/test_bench_runner.das create mode 100644 utils/internal/bench-stand/test_bench_suite.das diff --git a/.github/workflows/nightly_bench.yml b/.github/workflows/nightly_bench.yml new file mode 100644 index 0000000000..4c110c849e --- /dev/null +++ b/.github/workflows/nightly_bench.yml @@ -0,0 +1,149 @@ +name: nightly benchmark stand + +# Drives the benchmark stand on the web box (dasweb-1, the daslang.io origin) and +# publishes the night at https://daslang.io/bench/. The box does the work - checkout, +# Release build, every `[benchmark]` file in benchmarks/ in the interpreter and JIT +# lanes, the report - so the numbers come from ONE machine night after night; this +# runner only kicks it off over ssh, streams the log, and posts the summary. +# +# Red means the night is not a usable data point: the build failed, or at least one +# benchmark file failed (compile error, failed assertion, crash, timeout, memory cap). +# A regression is a warning in the step summary and on the site, not a red - flip +# `fail_on_regression` on a dispatch to make it one. +# +# No self-hosted runner: this repository is public, and a self-hosted runner would +# execute pull-request code on the production web box. The key reaches only the +# stand's gate verbs - start, follow, status, summary - never a shell (a forced +# command in the box's authorized_keys). + +on: + schedule: + # 05:00 UTC - after nightly_playground's 04:30, while the box is otherwise idle. + - cron: '0 5 * * *' + workflow_dispatch: + inputs: + ref: + description: 'commit or branch to benchmark (default: master)' + required: false + default: 'master' + fail_on_regression: + description: 'turn a regression into a red run' + required: false + default: 'false' + +defaults: + run: + shell: bash + +# One night at a time: the box holds a run lock, so a second start would only be refused. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +# Empty on purpose: every step of the one job talks to the box over ssh. Nothing checks the +# repository out and nothing calls the GitHub API, so the token needs no scope at all. +permissions: {} + +jobs: + bench: + # The cron only runs on the canonical repo: it drives daslang.io's box, which a fork + # neither owns nor holds the key to, and a red nightly emails the fork owner. + if: github.event_name != 'schedule' || github.repository == 'GaijinEntertainment/daScript' + runs-on: ubuntu-latest + # Build (~20-40 min on the box) + two lanes of the suite (~1-2 h) + report. + timeout-minutes: 360 + # SECRET SCOPE, like pages.yml: DASWEB_BENCH_KEY is an environment secret on + # `github-pages` (the repository has no repo-level secrets). + environment: + name: github-pages + env: + BOX: 89.167.63.131 + BOX_USER: bench + steps: + - name: "Set up the ssh key" + id: key + env: + BENCH_KEY: ${{ secrets.DASWEB_BENCH_KEY }} + run: | + set -euo pipefail + # A fork has no access to the secret and legitimately skips, the way pages.yml's deploy + # step does; on the canonical repo an empty key means the nightly silently stopped + # measuring, so it is loud there. + if [ -z "$BENCH_KEY" ]; then + if [ "$GITHUB_REPOSITORY" = "GaijinEntertainment/daScript" ]; then + echo "::error::DASWEB_BENCH_KEY is empty on the canonical repo - the stand was not driven." + echo "It is an ENVIRONMENT secret on 'github-pages'; check the job still declares that environment." + exit 1 + fi + echo "DASWEB_BENCH_KEY not set (fork) - skipping the run" + echo "skipped=1" >> "$GITHUB_OUTPUT" + exit 0 + fi + mkdir -p ~/.ssh + printf '%s\n' "$BENCH_KEY" > ~/.ssh/bench_key + chmod 600 ~/.ssh/bench_key + # Pinned host key - the same box and key pages.yml deploys to; no trust-on-first-use. + echo "$BOX ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILP2wzOeEGD2r7apldQlTe2gwR1dVTCXCfrD+pJooMmY" >> ~/.ssh/known_hosts + echo "RSH=ssh -i $HOME/.ssh/bench_key -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ConnectTimeout=30" >> "$GITHUB_ENV" + + - name: "Start the night on the box" + id: start + if: steps.key.outputs.skipped != '1' + env: + REF: ${{ github.event.inputs.ref || 'master' }} + FAIL_ON_REGRESSION: ${{ github.event.inputs.fail_on_regression || 'false' }} + run: | + set -euo pipefail + # The forced command takes `start [fail-on-regression]`, and that second word is + # the ONLY way the input reaches the box: a forced command carries no environment, so + # exporting BENCH_STAND_FAIL_ON_REGRESSION here would reach nothing. A busy box answers + # exit 3 and names the run it is still working on. + FLAG="" + [ "$FAIL_ON_REGRESSION" = "true" ] && FLAG=" fail-on-regression" + RUN_ID=$($RSH "$BOX_USER@$BOX" "start $REF$FLAG") + echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" + echo "started run $RUN_ID for $REF (regression fails the night: $FAIL_ON_REGRESSION)" + + - name: "Follow the run" + if: steps.key.outputs.skipped != '1' + run: | + set -uo pipefail + # An ssh drop (exit 255) reconnects rather than failing the lane; every other code is + # the run's own, recorded for the summary step to raise. + for attempt in $(seq 1 60); do + $RSH "$BOX_USER@$BOX" "follow ${{ steps.start.outputs.run_id }}" + rc=$? + if [ $rc -ne 255 ]; then + echo "run finished with exit $rc" + echo "$rc" > run_exit + exit 0 + fi + echo "ssh dropped (attempt $attempt) - reconnecting" + sleep 30 + done + echo "::error::could not stay connected to the box" + echo 255 > run_exit + + - name: "Post the summary" + # always(), so a night that ended badly still publishes its summary here. The step's own + # exit code is the lane's verdict: the run's code when there is one, and a red when the + # summary could not be fetched at all - an unfetched summary is the one artifact this + # lane exists to deliver, so swallowing it would leave a green lane saying nothing. + if: always() && steps.key.outputs.skipped != '1' + run: | + set -uo pipefail + summary_rc=0 + $RSH "$BOX_USER@$BOX" "summary" > summary.md || summary_rc=$? + if [ $summary_rc -ne 0 ] || [ ! -s summary.md ]; then + echo "::error::could not fetch the night's summary from the box (ssh exit $summary_rc)" + echo "The night's own verdict, if it reached one, is below." >> "$GITHUB_STEP_SUMMARY" + fi + [ -s summary.md ] && cat summary.md >> "$GITHUB_STEP_SUMMARY" + rc=$(cat run_exit 2>/dev/null || echo 1) + case "$rc" in + 0) echo "night ok" ;; + 2) echo "::error::the night regressed and fail_on_regression is on - see the summary" ;; + *) echo "::error::the night failed (exit $rc) - see the summary and https://daslang.io/bench/" ;; + esac + [ "$rc" = 0 ] && exit "$summary_rc" + exit "$rc" diff --git a/CMakeLists.txt b/CMakeLists.txt index 44a817832d..8c88d11b6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -178,8 +178,17 @@ MACRO(DAS_AOT_EXT input_files genList mainTarget dasAotTool dasAotToolArg) set(all_depends ${dasAotTool} ${PROJECT_SOURCE_DIR}/utils/aot/main.das ${DAS_AOT_DASLIB_DEPENDS} ${DAS_AOT_EXTRA_DEPENDS}) set(all_outputs "") # Per-batch arg buffer; chunked to keep each invocation under Windows' - # 32K command-line limit when worktree paths are long. - set(_DAS_AOT_BATCH_SIZE 60) + # 32K command-line limit when worktree paths are long. A caller may set + # DAS_AOT_BATCH_SIZE to something smaller: one tool run compiles every file it + # is given in one process, so a file whose require graph installs macros can + # perturb the ones after it in its batch (error[20600] / error[30100] on an + # annotation that exists). A batch of 1 costs a process per file and makes that + # impossible. + if(DEFINED DAS_AOT_BATCH_SIZE) + set(_DAS_AOT_BATCH_SIZE ${DAS_AOT_BATCH_SIZE}) + else() + set(_DAS_AOT_BATCH_SIZE 60) + endif() set(_batches "") set(_batch_idx 0) set(_batch_count 0) diff --git a/benchmarks/decs/bench_from_decs_count.das b/benchmarks/decs/bench_from_decs_count.das index 978d96ea50..3157206b62 100644 --- a/benchmarks/decs/bench_from_decs_count.das +++ b/benchmarks/decs/bench_from_decs_count.das @@ -20,7 +20,7 @@ struct BenchCountRow { def fixture(n : int) { restart() - create_entities(n) $(eid : EntityId; i : int; var cmp : ComponentMap) { + create_entities(n) $(_eid : EntityId; i : int; var cmp : ComponentMap) { apply_decs_template(cmp, BenchCountRow(val = i)) } } @@ -32,7 +32,7 @@ def from_decs_count_m1_hand(b : B?) { b |> run("m1_hand_arch_size/{N}", N) { var erq : EcsRequest erq.req |> push("bench_val") - var total = 0 + var total = 0l for_each_archetype(erq) $(arch : Archetype) { total += arch.size } diff --git a/skills/internal/preflight.md b/skills/internal/preflight.md index 504e5fadde..45dada07a5 100644 --- a/skills/internal/preflight.md +++ b/skills/internal/preflight.md @@ -62,6 +62,7 @@ working-tree copy. |---|---|---| | `build.yml` (per-PR) | every PR commit (`pull_request`) + pushes to `master` | `build` matrix (`ci/ci_matrix.py build`: Debug + Release on linux, linux_arm, darwin15, darwin26; windows 32 Release, windows 64 Release), `bundle_smoke`, `build_linux_gcc` | | `build.yml` (nightly) | `schedule` cron (daily 02:00 UTC) | `build_windows_mingw` + `build_windows_clangcl` (gated OFF per-PR) **plus the full build matrix - the per-PR cells, the sanitizer cells (linux Release asan/tsan/ubsan), the fast-math cell (linux Release `-DDAS_FAST_MATH=ON`) and windows 64 Debug - whose Release cells run the full AOT sweep** ("Slow Release Tests"). Breaks surface within ~24 h, not at PR time | +| `nightly_bench.yml` | `schedule` cron (daily 05:00 UTC) + `workflow_dispatch` (`ref`, `fail_on_regression`) | one job that drives the benchmark stand on the daslang.io box over ssh (`utils/internal/bench-stand/nightly.sh`): Release build of `ref` plus `test_aot`, then every `benchmarks/**/*.das` its `suite.json` does not exclude or skip, in the interp, jit and aot lanes, the report to https://daslang.io/bench/; red = build or any benchmark file failed. Local mirror: `bin/daslang utils/internal/bench-stand/main.das -- run --meta --out --filter ` then `... -- report --runs --out-data --out-summary ` (README there) | | `nightly_imgui.yml` | `schedule` cron (daily 03:00 UTC) + `workflow_dispatch` | dasImgui playwright suite on ubuntu + macos - section below | | `extended_checks.yml` (per-PR) | every PR | two darwin15-arm64 jobs, `core` and `modules` (`ci/ci_matrix.py extended`), ALL release modules ON - section below | | `extended_checks.yml` (nightly) | `schedule` cron (daily 04:00 UTC) + `workflow_dispatch` | one job each on linux, darwin15 and windows running every step (role `all`), including the ones too slow for a PR: tutorial dry-runs, the run form of examples, coverage, the nano cross-compile, the AST verify tree sweep, doc-verify | diff --git a/tests/aot/CMakeLists.txt b/tests/aot/CMakeLists.txt index cbd2c5f89e..6693887026 100644 --- a/tests/aot/CMakeLists.txt +++ b/tests/aot/CMakeLists.txt @@ -40,6 +40,26 @@ FILE(GLOB AOT_TESTS_FILES RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS "test # Exclude _-prefixed helper modules (required transitively, not standalone test entry points) list(FILTER AOT_TESTS_FILES EXCLUDE REGEX "/_") +# The nightly benchmark stand's AOT lane (utils/internal/bench-stand): the benchmark bodies get +# their stubs linked into test_aot, so `test_aot -use-aot dastest/dastest.das -- --use-aot --bench` +# measures native code instead of the interpreter. Predefined rather than globbed by the loop +# below, because the tree is nested and the loop's convention is tests//*.das. +FILE(GLOB_RECURSE AOT_BENCHMARKS_FILES RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS "benchmarks/*.das") +# _-prefixed helper modules are required transitively, never standalone; benchmarks/sql/tests is +# a dastest suite, not a benchmark. +list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "/_") +list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "^benchmarks/sql/tests/") +# Suites whose require graph reaches a native module: each would need that module's own AOT half +# linked here, so they measure in the interpreter and JIT lanes only. suite.json states the +# reason per file, and widening this set is what lets a lane skip go away. +list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "^benchmarks/(audio|terminal|sql)/") +list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "benchmarks/core/math/scalar_crt\\.das$") +list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "benchmarks/micro/(join_select|single_last|sort_distinct_take)_shapes\\.das$") +# benchmarks/core/jit_aot rides the language suite on purpose (see AOT_JIT_BENCH_FILES below), so +# its stubs are already in both AOT binaries. Registering it twice emits two TUs for one file and +# the anonymous-module namespaces collide at link (multiple definition of das::_anon_). +list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "^benchmarks/core/jit_aot/") + FILE(GLOB AOT_BARE_BLOCK_FILES RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS "tests/bare_block/*.das") list(FILTER AOT_BARE_BLOCK_FILES EXCLUDE REGEX "failed_|cant_|invalid_") @@ -394,7 +414,7 @@ set(DAS_AOT_SUITES jobque json jsonrpc language linq lint long_array_table loops lpipe lsp macro_boost macro_call match math mcp md_boost module_cache module_tests option promote quote reader_macro regex rtti safe_addr soa spoof strings stbimage table_packed template - tests type_lattice type_traits typemacro uri with_boost delegate) + tests type_lattice type_traits typemacro uri with_boost delegate benchmarks) foreach(_s IN LISTS DAS_AOT_SUITES) string(TOUPPER ${_s} _u) # suites with an irregular dir / a filter / a curated list define AOT__FILES above; @@ -405,9 +425,18 @@ foreach(_s IN LISTS DAS_AOT_SUITES) add_custom_target(test_aot_${_s}) set(${_u}_AOT_GENERATED_SRC) list(LENGTH AOT_${_u}_FILES _nf) + # One process per benchmark: ten of the core/hash bodies install macros that make a LATER + # file in the same batch fail on `[benchmark]` (error[20600]). Batching is a speed knob, so + # the suite that trips it pays a process per file instead of losing the files. + if(_s STREQUAL "benchmarks") + set(DAS_AOT_BATCH_SIZE 1) + else() + unset(DAS_AOT_BATCH_SIZE) + endif() if(_nf GREATER 0) DAS_AOT("${AOT_${_u}_FILES}" ${_u}_AOT_GENERATED_SRC test_aot_${_s} daslang) endif() + unset(DAS_AOT_BATCH_SIZE) list(APPEND TEST_AOT_TARGETS test_aot_${_s}) list(APPEND TEST_AOT_GENVARS ${_u}_AOT_GENERATED_SRC) list(APPEND TEST_AOT_ALL_DAS ${AOT_${_u}_FILES}) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 6534ff120f..46c9a722ba 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -223,6 +223,7 @@ SET(DAS_UTILS_TO_TEST internal/hygiene internal/preflight/tests internal/ast-fuzz + internal/bench-stand internal/requirefix internal/test-release daspkg/test_daspkg.das diff --git a/utils/internal/bench-stand/README.md b/utils/internal/bench-stand/README.md new file mode 100644 index 0000000000..84e78f79ed --- /dev/null +++ b/utils/internal/bench-stand/README.md @@ -0,0 +1,206 @@ +# bench-stand - the nightly benchmark stand + +Every night one box checks out master, builds it, runs every `[benchmark]` file under +`benchmarks/` in the interpreter and JIT lanes, and publishes the result at +https://daslang.io/bench/ - a chart per benchmark arm over commits, a per-group index, and +a plain statement of what failed and what moved. The GitHub workflow +`.github/workflows/nightly_bench.yml` is the trigger and the red/green; the box is the +measuring instrument. Review rules: `REVIEW.md`. + +## 1. Layout + +- `main.das` - the tool: `run` (benchmark the tree into one run record) and `report` (every + record into `data.json` + `summary.md`, with the exit code the workflow keys on). +- `bench_suite.das` - `suite.json` loading and file discovery: which files, in which group, + under which limits. Pure; no processes. +- `bench_runner.das` - one file in one lane: spawn `dastest --bench`, enforce the wall-clock + and memory limits, turn its output into samples and a verdict. The parser is pure and + tested on captured output shapes. +- `bench_history.das` - records to dataset: series, the group index, regression verdicts, + the markdown summary. Pure. +- `suite.json` - the configuration (section 4). +- `site/` - the viewer: `index.html`, `app.js`, `style.css`. Static, no build step, no + dependencies; reads `data.json` and `status.json` beside it. +- `nightly.sh` - the box-side driver (section 3); `bench-stand-deploy.sh` provisions the box; + `caddy.snippet` is the public route, and the only place a route is written down. +- `REVIEW.md` + `REVIEW.das` - the review checklist and its mechanical half. The gate checks the + key parity of `suite.json` / `SuiteConfig` / section 4 below, that a `skip` states a reason, + that a sibling module is required by bare name, that the viewer writes no markup, that every + `BENCH_STAND_*` knob is documented in both places, that no route literal is copied into + `bench-stand-deploy.sh`, and that every file here has its placement line. +- `test_bench_*.das` - the three dastest suites; `run_utils_tests` (per PR) runs them. + +## 2. Data model + +Benchmark identity: `#/` per lane, where the file id is the path under +`benchmarks/` without `.das` (`core/hash/test02#builtin_table/insert/600000`, lane `interp`). +The group is the file's directory (`core/hash`, `sort`, `sql`); the viewer sections and the +index follow it, so a new benchmark joins its group by living in the right folder. + +`runs/.json` - one per night, `RunRecord` in `bench_history.das`. The run id is +`-`. It carries the commit, the machine (host, CPU, cores, memory, kernel, +compiler, load at start), the build (status, seconds, log tail), the lane states, and one +`FileResult` per file per lane: status, exit code, seconds, message, the last 60 log lines, +and the samples that completed. A sample is one arm aggregated over the night's repeats: +`ns` = the minimum ns/op (the noise-robust statistic), `ns_median`, `spread` = +(max - min) / min, and per-op allocation medians. + +A record's own `status` is `ok`, `build_failed` or `bench_failed`, and `load_runs` re-derives it +from the record's parts on every read, so a hand-edited status cannot mislead the report. + +File statuses: `ok`, `skipped` (listed in `suite.json` with a reason), `compile_error`, +`failed` (a `[benchmark]` function failed an assertion or panicked - its arms are dropped, the +other functions' arms kept), `timeout`, `memory` (killed at the RSS cap), `exit_nonzero` (the +process died without a finished dastest report, so the suite was cut short), `spawn_failed`. + +A file whose dastest report is finished and whose arms are all there stays `ok` even when the +process then exits non-zero: the measurement stands and only the shutdown went wrong, so the +message records it (`measured, then exited with code 1 at shutdown`). Keying the verdict on the +exit code instead would fail every file on a RelWithDebInfo host, where the C++ allocation +tracker's exit-time report takes every process to 1. + +The JIT lane is probed once per night with a trivial program, and the probe believes the marker +that program prints, not the process exit code - for the same reason. + +## 2.3 The AOT lane + +AOT is not a flag on `daslang`: the benchmark bodies are compiled to C++ at build time and linked +into a binary. That binary is `test_aot`, which the benchmark files join through the +`benchmarks` row of `DAS_AOT_SUITES` in `tests/aot/CMakeLists.txt` (repo root); `--aot-bin` names +it and the night builds `--target test_aot` before measuring. The lane's argv carries **both** +`-use-aot` on the host and `--use-aot` on dastest. The second one is what arms `fail_on_no_aot`: +without it every function silently interprets and the lane reports interpreter numbers under an +AOT label, and with it a function missing its stub is `error[50101]`, so a run that finishes is +proof the stubs were live. + +Do not read the tier from dastest's own label: `is_in_aot()` is true only while the AOT emitter +runs, so an AOT run still prints `[INTERP]`. The lane name is the record of what ran. + +A benchmark whose require graph reaches a native module needs that module's own AOT half linked +into the same binary. Those suites are excluded from the AOT set in `tests/aot/CMakeLists.txt` +and carry their reason under `lane_excludes` in `suite.json`; widening the first is what lets the +second go away. + +`site/data.json` - `Dataset`: run summaries, groups, series (columnar: `runs` the run index into +`Dataset.runs`, `ns` the minimum, `spread` the night's spread), the changes for the latest run +(`regression`, `improvement`, `first_seen`, `missing`), the group index (`runs` and `value`). `site/summary.md` is the +same night in markdown; the workflow posts it as its step summary. `site/status.json` is +written by `nightly.sh` at start and end (`running` / `finished`, run id, exit), so a night +whose build failed before any binary existed is still visible. + +### 2.1 Statistics + +A change verdict compares tonight's value with the median of the series' previous +`baseline_runs` points (at least 3). It is a regression when the relative move exceeds both +`regression_threshold` and `noise_multiplier` times the baseline's relative median absolute +deviation; an improvement when it falls by the same gate. A series with no earlier point is +`new`; one with a baseline but no point tonight while its file ran ok is `missing` - an arm +was renamed or removed. The group index is the geometric mean over the group's series of +value / reference, times 100, where each series' reference is the median of its first five +points; a run contributes when at least half the group's series measured that night. + +### 2.2 Exit codes + +`run`: 0 when every file ran ok, 1 otherwise. `report`: 0 when the latest night is ok, 1 when it +failed (build or any file), 2 when it is ok but regressed and `--fail-on-regression` was +given. `nightly.sh run` exits with the run's code, or the report's when the run was clean. + +`run --failed ""` benchmarks nothing: it writes the record of a night that could not run +the suite and exits 1. That is how the driver records a failed build or its own failure without a +second writer of the run-record schema; a build that failed carries no file, since its log tail +already rides the record's `build`. + +## 3. The box + +`dasweb-1` (the daslang.io origin) runs the stand as the unprivileged `bench` user under +`/srv/bench-stand`: `src/` (the clone each night checks out), `runs/`, `site/` (what Caddy +serves at `/bench/`), `logs/`, `last_good/` (the last passing build's binary, `daslib/` and +this tool - renders the report when tonight's build fails, so a broken master publishes a red +night instead of a stale page), `state/` (run lock, current run id). + +The workflow reaches the box over ssh as `bench` with a key whose `authorized_keys` line +forces `nightly.sh gate`: only `start [fail-on-regression]`, `follow [run_id]`, `status` +and `summary` exist on that connection, at most two arguments each and only path characters in +them, no pty, no forwarding. The gate reads the caller's words from `SSH_ORIGINAL_COMMAND`, which +is also why `start`'s second word is the only way the workflow can ask for a regression to fail +the night - a forced command carries no environment. `start` +resolves the ref, launches the night detached and prints the run id; `follow` streams the log +and exits with the run's code once `status.json` says finished, so a dropped ssh session +reconnects instead of failing the lane. There is deliberately no self-hosted runner: the +repository is public, and one would execute pull-request code on the production web box. + +The night builds Release (`-DDAS_SQLITE_DISABLED=OFF -DDAS_PUGIXML_DISABLED=OFF +-DDAS_LLVM_DISABLED=OFF`, GLFW and HV off), the targets the benchmarks need, and `test_aot` for +the AOT lane - that last one is ~1080 translation units and the bulk of the night's build. Release +matters: RelWithDebInfo arms the C++ allocation tracker, whose exit-time leak report turns +every clean process into exit 1. The JIT lane is probed once per night (`daslang -jit` on a +trivial program); when the probe fails the lane is recorded as unavailable with the reason and +the interpreter lane still runs. + +One-time setup, as root on the box, with the public half of a fresh ed25519 key: + +```sh +sudo install -m 755 utils/internal/bench-stand/bench-stand-deploy.sh /usr/local/sbin/ +sudo bench-stand-deploy.sh provision ./bench_key.pub +sudo bench-stand-deploy.sh caddy +``` + +Then store the private half as the `DASWEB_BENCH_KEY` environment secret on `github-pages` +and dispatch `nightly_bench.yml` once. `bench-stand-deploy.sh status` shows the layout, the +last `status.json` and whether Caddy carries every route the snippet declares (it derives that +list from `caddy.snippet`, so a route added there is checked without editing the script; a +vhost holding some routes and not others is reported and never repaired automatically). The +deploy script installs the packages the build needs (git, cmake, ninja, g++, ccache, rsync) - +ccache is what keeps the nightly rebuild at minutes rather than the cold 20-40. + +Environment knobs `nightly.sh` reads, all optional: `BENCH_STAND_HOME` (layout root), +`BENCH_STAND_REPO_URL` (commit-link base), `BENCH_STAND_SITE_URL` (what the summary links), +`BENCH_STAND_JOBS` (build parallelism; the default counts the box's CPUs through `nproc`, `sysctl hw.ncpu`, or 4), `BENCH_STAND_CMAKE_ARGS` (extra configure arguments), +`BENCH_STAND_LANES` / `BENCH_STAND_REPEAT` / `BENCH_STAND_FAIL_ON_REGRESSION` (forwarded to +the tool), `BENCH_STAND_AOT=skip` (leave the AOT binary +unbuilt, so the aot lane reports itself unavailable and the other two still measure), and two for +local dry runs only: `BENCH_STAND_BUILD=skip` (reuse `src/bin/daslang`) and `BENCH_STAND_FILTER` +(a path substring). + +## 4. Configuration - `suite.json` + +`root` (the benchmark tree, `benchmarks`), `exclude` (globs over the path under root; helper +modules `_*.das` and `**/tests/**` by default), `lanes` (`interp`, `jit`, `aot`), +`lane_excludes` (per lane, a glob to the reason those files do not run in it - a file the AOT +binary cannot carry still measures in the other two, and the reason shows on the site every +night), `repeat` (dastest `--count`; the +minimum over repeats is the recorded value), `timeout_seconds` and `memory_limit_mb` per file +(the runner kills the child and records `timeout` / `memory`), `regression_threshold`, +`noise_multiplier`, `baseline_runs` (section 2.1), and `files` - per-file overrides of the +limits or a `skip` with its reason. A skipped file is listed on the site and in the summary +every night, so a skip is visible debt, not a deletion; an override naming a file that no +longer exists is an error, so a renamed benchmark cannot silently lose its skip. + +## 5. Running locally + +```sh +# one night of a slice of the suite, both lanes, into a scratch record +bin/daslang utils/internal/bench-stand/main.das -- run --meta meta.json --out /tmp/stand/runs/n1.json --filter core/math/ --repeat 2 +# the report over every record, then open site/ over any static server +bin/daslang utils/internal/bench-stand/main.das -- report --runs /tmp/stand/runs --out-data /tmp/stand/site/data.json --out-summary /tmp/stand/site/summary.md +cp utils/internal/bench-stand/site/* /tmp/stand/site/ && ln -sfn ../runs /tmp/stand/site/runs +bin/daslang dastest/dastest.das -- --test utils/internal/bench-stand +``` + +`meta.json` is what `nightly.sh` writes: `{"run_id", "started", "commit": {"sha", "date", +"subject", "author"}, "machine": {...}, "build": {"status", "seconds", "log_tail"}}`; any +subset parses. The whole box pipeline runs locally too: point `BENCH_STAND_HOME` at a scratch +layout whose `src/` is a worktree with a built `bin/daslang`, and run +`BENCH_STAND_BUILD=skip BENCH_STAND_FILTER=core/math/ nightly.sh run HEAD`. + +## 6. Reading a red night + +A night whose build failed says so first: the summary leads with the build's own section and the +last lines of its log, and stops there - nothing was measured, so there is nothing else to read. +Otherwise the step summary (and `summary.md`) leads with the status and the commit, then a failures +table - file, lane, status, message - where the message is the thing to act on: the first +`error[...]` line of a compile error, the failed function names, the kill reason with the knob +that set the limit, or the `FATAL` line of a non-zero exit. The run record has the last 60 log +lines of every failed file. A `build_failed` night's record carries the build log tail. A +regression row names the arm, both values, the change and the noise that the gate was +measured against; the chart behind it is one click from the site's latest-night panel. diff --git a/utils/internal/bench-stand/REVIEW.das b/utils/internal/bench-stand/REVIEW.das new file mode 100644 index 0000000000..58c9d04745 --- /dev/null +++ b/utils/internal/bench-stand/REVIEW.das @@ -0,0 +1,254 @@ +options gen2 +options indenting = 4 + +require strings +require daslib/strings_boost +require daslib/fio +require daslib/json_boost +require math +require dastest/review_gate + +// The mechanical half of utils/internal/bench-stand/REVIEW.md (contract: REVIEW_COMMON.md at +// the repo root). Run from the repo root: bin/daslang utils/internal/bench-stand/REVIEW.das - +// exit 0 clean, 1 with findings. + +let TOOL = "utils/internal/bench-stand" + +def private tool_path(rel : string) : string { + return path_join(TOOL, rel) +} + +// Every committed file of the tool. A dot-prefixed path is build or cache output the tree +// ignores (`.jitted_scripts/`), never something a rule reaches. +def private files_here() : array { + var out : array + dir_rec(TOOL) $(name, is_dir) { + return if (is_dir) + let rel = to_generic_path(name) + return if (starts_with(rel, ".") || find(rel, "/.") >= 0) + out |> push(rel) + } + out |> sort + return <- out +} + +// Every `.das` here is one module family: a sibling is required by its bare module name, never +// by a path, so the files stay movable as a set. +def private check_sibling_requires() { + let names = files_here() + var modules : table + for (rel in names) { + if (extension(rel) == ".das" && !starts_with(base_name(rel), "test_") && base_name(rel) != "REVIEW.das") { + modules |> insert(stem(rel)) + } + } + for (rel in names) { + continue if (extension(rel) != ".das") + let path = tool_path(rel) + for (req in das_requires(path)) { + let leaf = stem(base_name(req)) + continue if (req == leaf || !key_exists(modules, leaf)) + gate_finding(path, find_line(fread(path), req), + "requires the sibling module '{leaf}' as '{req}' - spell a sibling by its bare module name") + } + } +} + +// suite.json, SuiteConfig and README section 4 are three spellings of one key set; a key that +// loses any of the three is a knob nobody can set, type or find. +def private check_suite_keys() { + let suite_path = tool_path("suite.json") + let config_path = tool_path("bench_suite.das") + let readme_path = tool_path("README.md") + var error = "" + var root = read_json(fread(suite_path), error) + if (root == null || !empty(error)) { + gate_finding(suite_path, "does not parse as JSON: {error}") + return + } + let config_text = fread(config_path) + let struct_at = find(config_text, "struct SuiteConfig \{") + if (struct_at < 0) { + gate_finding(config_path, "no `struct SuiteConfig \{` - the suite config's field list is what suite.json is checked against") + return + } + let struct_end = find(config_text, "\n\}", struct_at) + let struct_body = slice(config_text, struct_at, struct_end < 0 ? length(config_text) : struct_end) + let readme_section = readme_section_of(readme_path, "## 4.") + if (empty(readme_section)) { + gate_finding(readme_path, "no `## 4.` section - the suite keys are documented there") + } + var json_keys : table + if (!(root.value is _object)) { + gate_finding(suite_path, "is not a JSON object") + return + } + unsafe { + assume obj = root.value as _object + for (key in keys(obj)) { + json_keys |> insert(key) + if (find(struct_body, "\n {key} ") < 0) { + gate_finding(suite_path, "key '{key}' is not a SuiteConfig field in bench_suite.das") + } + if (!empty(readme_section) && find(readme_section, key) < 0) { + gate_finding(readme_path, "section 4 does not mention the suite key '{key}'") + } + } + } + for (line in split(struct_body, "\n")) { + let trimmed = strip(line) + continue if (empty(trimmed) || starts_with(trimmed, "//") || starts_with(trimmed, "struct")) + let colon = find(trimmed, " :") + continue if (colon <= 0) + let field = slice(trimmed, 0, colon) + continue if (find(field, " ") >= 0) + if (!key_exists(json_keys, field)) { + gate_finding(suite_path, "SuiteConfig field '{field}' has no key here - a knob with no line in suite.json is one nobody finds") + } + } +} + +// A skipped benchmark is debt the site shows every night; a skip with no reason hides what it +// is standing in for. +def private check_skip_reasons() { + let suite_path = tool_path("suite.json") + var error = "" + var root = read_json(fread(suite_path), error) + return if (root == null) + var files = root?["files"] + return if (files == null || !(files.value is _object)) + unsafe { + assume entries = files.value as _object + for (name in keys(entries)) { + var over = files?[name] + continue if (over == null) + let skip = over?["skip"] + // a key the entry does not carry comes back as a non-null _null node, never as a + // pointer that tests false - the distinction is what keeps a plain override quiet + continue if (skip == null || skip.value is _null) + if (empty(strip(over?["skip"] ?? ""))) { + gate_finding(suite_path, "the skip of '{name}' states no reason - name the failure it hides") + } + } + } +} + +// The viewer renders series names, commit subjects and failure messages that come from a run +// record; every one of them reaches the DOM as text, never as markup. +def private check_viewer_no_markup() { + for (rel in files_here()) { + continue if (dir_name(rel) != "site" || extension(rel) != ".js") + let path = tool_path(rel) + let text = strip_line_comments(fread(path)) + for (needle in ["innerHTML", "outerHTML", "insertAdjacentHTML", "document.write"]) { + let at = find(text, needle) + continue if (at < 0) + gate_finding(path, find_line(fread(path), needle), + "writes markup with {needle} - a label from a run record reaches the DOM through textContent") + } + } +} + +// nightly.sh's knobs are invisible: one list in its own header, one in README section 3. A knob +// missing from either is one an operator cannot discover, and a documented knob nothing reads is +// one they would set in vain. +def private check_env_knobs() { + let script_path = tool_path("nightly.sh") + let readme_path = tool_path("README.md") + let text = fread(script_path) + let header = slice(text, 0, max(find(text, "\nset -"), 0)) + let body = slice(text, max(find(text, "\nset -"), 0)) + let readme_section = readme_section_of(readme_path, "## 3.") + if (empty(readme_section)) { + gate_finding(readme_path, "no `## 3.` section - nightly.sh's knobs are documented there") + } + var read_names : table + for (name in keys(prefixed_tokens(body, "BENCH_STAND_"))) { + read_names |> insert(name) + if (find(header, name) < 0) { + gate_finding(script_path, "reads {name} without naming it in the script's own header") + } + if (!empty(readme_section) && find(readme_section, name) < 0) { + gate_finding(readme_path, "section 3 does not document {name}, which nightly.sh reads") + } + } + for (name in keys(prefixed_tokens(header, "BENCH_STAND_"))) { + if (!key_exists(read_names, name)) { + gate_finding(script_path, "the header documents {name}, which the script never reads") + } + } +} + +// caddy.snippet is the one place the stand's public routes are written down; the deploy script +// derives its checks from it, so a route literal here is a second copy that can go stale. +def private check_no_route_literals() { + let path = tool_path("bench-stand-deploy.sh") + var in_extractor = false + var lineno = 0 + for (raw in split(fread(path), "\n")) { + lineno++ + let line = strip(raw) + // snippet_routes is the extractor: the directive names ARE its subject, so its body is + // the one place in the script licensed to spell them. + if (starts_with(line, "snippet_routes() \{")) { + in_extractor = true + continue + } + if (in_extractor) { + in_extractor = line != "\}" + continue + } + continue if (empty(line) || starts_with(line, "#")) + let code = find(line, " # ") >= 0 ? slice(line, 0, find(line, " # ")) : line + for (needle in ["handle_path", "redir ", "file_server", "reverse_proxy"]) { + continue if (find(code, needle) < 0) + gate_finding(path, lineno, + "spells the route directive '{strip(needle)}' - read the routes from caddy.snippet instead (snippet_routes)") + } + } +} + +// The placement block is the folder's inventory. A file with no line is a file no rule reaches. +def private check_placement_inventory() { + let review_path = tool_path("REVIEW.md") + let text = fread(review_path) + let block_at = find(text, "**Placement - one file, one line") + if (block_at < 0) { + gate_finding(review_path, "no placement block - the folder's file inventory lives there") + return + } + let inventory = slice(text, block_at) + for (rel in files_here()) { + let leaf = base_name(rel) + continue if (leaf == "REVIEW.md" || leaf == "README.md") + let listed = dir_name(rel) == "site" ? find(inventory, "`site/`") >= 0 : find(inventory, "`{leaf}`") >= 0 + if (!listed) { + gate_finding(review_path, "the placement block has no line for {rel}") + } + } +} + +// A `## N.` section of a markdown file, up to the next same-level heading. +def private readme_section_of(path : string; heading : string) : string { + let text = fread(path) + let at = find(text, heading) + return "" if (at < 0) + let next = find(text, "\n## ", at + 1) + return next < 0 ? slice(text, at) : slice(text, at, next) +} + +[export] +def main() : int { + if (!stat(TOOL).is_dir) { + print("run from the repo root: {TOOL} not found\n") + return 1 + } + check_sibling_requires() + check_suite_keys() + check_skip_reasons() + check_viewer_no_markup() + check_env_knobs() + check_no_route_literals() + check_placement_inventory() + return gate_verdict("bench-stand") +} diff --git a/utils/internal/bench-stand/REVIEW.md b/utils/internal/bench-stand/REVIEW.md new file mode 100644 index 0000000000..5755dbdeae --- /dev/null +++ b/utils/internal/bench-stand/REVIEW.md @@ -0,0 +1,62 @@ +# bench-stand Code Review Checklist + +**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: +`README.md`. A diff here that changes what the nightly workflow invokes - a `main.das` verb or +flag, an exit code, or the `nightly.sh gate` vocabulary - applies `.github/workflows/REVIEW.md` +(repo root) too. + +**Never put a `[test]` file outside this directory - a test of this tool lives beside the file it +tests.** + +**Never add a test that touches the filesystem outside a `temp_directory`-rooted path, or that +leaves behind what it creates.** + +**A failure message a diff adds or changes states the underlying reason, and the summary line +carrying it names the file and the lane it ran in (`interp` or `jit`).** + +**Never add a run-record field without saying in `README.md` section 2 what reads it** - the +summary, the viewer, or a person opening the record. A record is an archive as well as the +viewer's input, so a field only a human reads is fine; a field nobody named is one nobody +notices going wrong. + +**A diff that changes a run-record field keeps the new reader parsing a record written by the old +code, a missing field keeping its declared default.** Records already on the box are never +rewritten. + +**Never let a child's own output overwrite a `timeout` or `memory` status in `run_bench_file` - +a killed child that printed a passing report is still killed. Loosen a value in `suite.json` +instead.** + +**Weakening the limit arms of `test_run_bench_file_limits` (`test_bench_runner.das`) is a defect: +dropping an arm, or relaxing what it asserts about the kill.** + +**A series a diff adds or recolors in `site/app.js` takes its color from its lane, never from its +position in the series list** - a color that moves when a series is added makes two nights +uncomparable. + +**A chart a diff adds to `site/app.js` shows a legend whenever it draws more than one lane.** + +**A chart a diff adds to `site/app.js` also ships its table view - the same points as a table of +numbers, behind the card's `table` toggle.** + +**Never read a benchmark's identity from anywhere but its path under `benchmarks/`** - the group +is the directory, the id is the path without `.das`. + +**Weakening `REVIEW.das` (beside this file) is a defect: dropping a check, narrowing what a check +walks, or rewriting a finding text so it no longer names what failed.** + +**Placement - one file, one line: a diff keeps each file inside its line, and a new file adds its +line here, with its tests, in the same change.** + +- `main.das` - verbs, argv, exit-code mapping. No parsing of dastest output, no statistics. +- `bench_suite.das` - `suite.json` and file discovery. No processes. +- `bench_runner.das` - one file in one lane: spawn, limits, output to samples and verdict. +- `bench_history.das` - records to dataset, verdicts, summary markdown. No processes, no argv. +- `suite.json` - configuration. +- `site/` - the viewer. Zero dependencies, zero build step. +- `nightly.sh` - the box-side driver. +- `bench-stand-deploy.sh` - root-side provisioning. +- `caddy.snippet` - the public route, and the only place a route is written down. +- `REVIEW.das` - the mechanical half of this checklist. +- `test_bench_suite.das`, `test_bench_runner.das`, `test_bench_history.das` - the module suites. +- `test_bench_cli.das` - the two programs, spawned: `main.das`'s verbs and `nightly.sh`'s ssh gate. diff --git a/utils/internal/bench-stand/bench-stand-deploy.sh b/utils/internal/bench-stand/bench-stand-deploy.sh new file mode 100755 index 0000000000..f4c6f22877 --- /dev/null +++ b/utils/internal/bench-stand/bench-stand-deploy.sh @@ -0,0 +1,127 @@ +#!/bin/sh +# bench-stand box-side provisioning for dasweb-1. The ONE privileged surface of the stand: +# installed root-owned at /usr/local/sbin/bench-stand-deploy.sh and run with sudo. Verbs: +# +# provision one-time: packages, the `bench` user, the layout, the clone, +# the forced-command ssh key the GitHub workflow uses +# caddy splice caddy.snippet into the daslang.io vhost, validate, reload +# status user, layout, last status.json, Caddy route presence +# +# Everything the stand does afterwards runs unprivileged as `bench` through nightly.sh, whose +# gate verbs (start / follow / status / summary) are all the key can reach - never a shell. The +# workflow's key (DASWEB_BENCH_KEY, an environment secret on `github-pages`) is generated on +# the operator's machine: `ssh-keygen -t ed25519 -f bench_key -N '' -C bench-stand`; the +# public half is what `provision` installs, the private half goes into the secret. +set -eu + +HOME_DIR=/srv/bench-stand +SVCUSER=bench +REPO=https://github.com/GaijinEntertainment/daScript.git +CADDYFILE=/etc/caddy/Caddyfile +TOOL=utils/internal/bench-stand + +verb="${1:?usage: bench-stand-deploy.sh provision | caddy | status}" + +provision() { + keyfile="${1:?provision needs the public key file the workflow will authenticate with}" + [ -f "$keyfile" ] || { echo "no such key file: $keyfile"; exit 1; } + # Only what the night needs: the build itself, ccache to keep the nightly rebuild + # incremental, rsync for the last_good copy, flock (util-linux) for the run lock. No GLFW or + # X11 development packages - `nightly.sh` configures with -DDAS_GLFW_DISABLED=ON. + apt-get install -y --no-install-recommends git cmake ninja-build g++ ccache rsync curl util-linux >/dev/null + id -u "$SVCUSER" >/dev/null 2>&1 || useradd --system --create-home --home-dir "$HOME_DIR" --shell /bin/bash "$SVCUSER" + install -d -o "$SVCUSER" -g "$SVCUSER" "$HOME_DIR" "$HOME_DIR/runs" "$HOME_DIR/site" "$HOME_DIR/logs" "$HOME_DIR/state" "$HOME_DIR/last_good" + if [ ! -d "$HOME_DIR/src/.git" ]; then + su -s /bin/sh "$SVCUSER" -c "git clone --quiet '$REPO' '$HOME_DIR/src'" + fi + # Forced command: the key can only start/follow/status the stand, never open a shell. + install -d -m 700 -o "$SVCUSER" -g "$SVCUSER" "$HOME_DIR/.ssh" + key=$(head -n 1 "$keyfile") + line="command=\"$HOME_DIR/src/$TOOL/nightly.sh gate\",no-port-forwarding,no-agent-forwarding,no-X11-forwarding,no-pty $key" + auth="$HOME_DIR/.ssh/authorized_keys" + touch "$auth" + grep -qF "$key" "$auth" || echo "$line" >> "$auth" + chown "$SVCUSER:$SVCUSER" "$auth" + chmod 600 "$auth" + echo "provisioned: user=$SVCUSER home=$HOME_DIR clone=$HOME_DIR/src key installed with forced command" + echo "next: sudo bench-stand-deploy.sh caddy; then a workflow_dispatch of nightly_bench.yml" +} + +vhost_has() { + # true when the daslang.io block of the shared Caddyfile carries $1 as a literal - the other + # vhosts may spell the same directive, so a whole-file grep is not the question + awk '/^daslang\.io \{/ { b = 1 } b { print } b && /^\}/ { b = 0 }' "$CADDYFILE" | grep -qF "$1" +} + +# Every route line the snippet declares, each the literal the vhost must carry. Derived from the +# snippet rather than written twice, so a route added there is checked without editing this script. +snippet_routes() { + sed -n 's/ *{$//; s/^\(redir\|handle_path\|handle\|header\|respond\|reverse_proxy\|root\) /\1 /p' "$1" +} + +# Prints "all", "none" or "partial": how much of the snippet the vhost already carries. A partial +# splice is never repaired automatically - re-splicing would duplicate the routes already there. +routes_state() { + local snippet=$1 route have=0 miss=0 + while IFS= read -r route; do + [ -n "$route" ] || continue + if vhost_has "$route"; then have=$((have + 1)); else miss=$((miss + 1)); fi + done < 0) { + if (line ~ /^#/ || line ~ /^[[:space:]]*$/) continue + print "\t" line + } + close(snip); done = 1; next + } + { print } + ' "$CADDYFILE.bak-$ts" > "$CADDYFILE" + if [ "$(routes_state "$snippet")" != all ]; then + echo "caddy: splice did not land every route (daslang.io vhost not matched?) - restoring"; cp "$CADDYFILE.bak-$ts" "$CADDYFILE"; exit 1 + fi + if ! caddy validate --config "$CADDYFILE" --adapter caddyfile >/dev/null 2>&1; then + echo "caddy validate FAILED - restoring $CADDYFILE.bak-$ts"; cp "$CADDYFILE.bak-$ts" "$CADDYFILE"; exit 1 + fi + systemctl reload caddy + echo "caddy: /bench/ spliced + reloaded (backup $CADDYFILE.bak-$ts)" +} + +status() { + id "$SVCUSER" 2>/dev/null || echo "user $SVCUSER: missing" + for d in src runs site logs state last_good; do + [ -d "$HOME_DIR/$d" ] && echo "$HOME_DIR/$d: ok" || echo "$HOME_DIR/$d: missing" + done + [ -f "$HOME_DIR/site/status.json" ] && cat "$HOME_DIR/site/status.json" || echo "status.json: never run" + echo "caddy: routes $(routes_state "$HOME_DIR/src/$TOOL/caddy.snippet" 2>/dev/null || echo "unknown - no snippet")" + ls "$HOME_DIR/runs" 2>/dev/null | wc -l | sed 's/^/runs: /' +} + +case "$verb" in + provision) provision "${2:-}" ;; + caddy) caddy_apply ;; + status) status ;; + *) echo "unknown verb: $verb"; exit 2 ;; +esac diff --git a/utils/internal/bench-stand/bench_history.das b/utils/internal/bench-stand/bench_history.das new file mode 100644 index 0000000000..6b5354f1ac --- /dev/null +++ b/utils/internal/bench-stand/bench_history.das @@ -0,0 +1,544 @@ +options gen2 +options indenting = 4 + +module bench_history public + +require bench_suite +require bench_runner +require daslib/fio +require daslib/json_boost +require daslib/strings_boost +require strings +require math + +//! What a night amounts to. `build_failed` never measured anything; `bench_failed` measured +//! what it could and at least one file failed. json_boost writes an enum as its name, so a +//! record and the viewer both read these spellings. +enum RunStatus { + ok + build_failed + bench_failed +} + +//! A verdict on one benchmark arm at the latest run. `first_seen` had no earlier point to +//! compare with; `missing` had a baseline and no point tonight. +enum ChangeKind { + regression + improvement + first_seen + missing +} + +struct CommitInfo { + sha : string + date : string + subject : string + author : string +} + +struct MachineInfo { + host : string + cpu : string + kernel : string + compiler : string + cores : int + mem_gb : int + load_start : string +} + +//! `status` defaults to `ok` so a meta that says nothing about a build - a local run driven by +//! hand - is not read as a night whose build failed. The driver always writes it explicitly. +struct BuildInfo { + status : string = "ok" + seconds : double + log_tail : string +} + +//! What the box-side driver hands `run`: identity of the night and of the tree it built. +struct RunMeta { + run_id : string + started : string + commit : CommitInfo + machine : MachineInfo + build : BuildInfo = BuildInfo() +} + +//! One night's record (`runs/.json`). `status` is `ok`, `build_failed` or +//! `bench_failed`; `lanes` maps a lane to `ok` or the reason it did not run. +struct RunRecord { + schema : int = 1 + run_id : string + started : string + finished : string + seconds : double + commit : CommitInfo + machine : MachineInfo + build : BuildInfo = BuildInfo() + lanes : table + status : RunStatus + files : array +} + +struct Failure { + path : string + lane : string + status : string + message : string +} + +//! One row of the viewer's run history. `failures` covers every non-ok, non-skipped file. +struct RunSummary { + id : string + sha : string + date : string + subject : string + started : string + status : RunStatus + seconds : double + build_status : string + build_seconds : double + build_log_tail : string + host : string + lanes : table + failures : array + skipped : array + files_ok : int + regressions : int + improvements : int +} + +//! One benchmark arm in one lane over every run that measured it. The three arrays are one +//! point per run, in run order: the run's index into `Dataset.runs`, its ns/op minimum, and its +//! spread over that night's repeats. +struct Series { + id : string + group : string + file : string + lane : string + runs : array + ns : array + spread : array +} + +//! `noise` is the baseline's relative median absolute deviation - the gate `change` had to pass. +struct Change { + id : string + group : string + lane : string + kind : ChangeKind + baseline : double + value : double + change : double + noise : double +} + +//! Geometric-mean index of one group in one lane, 100 = each series at its own reference +//! (the median of its first points). One point per run, in run order. +struct IndexSeries { + group : string + lane : string + runs : array + value : array +} + +//! `site/data.json`: everything the viewer renders, and what the summary is rendered from. +struct Dataset { + schema : int = 1 + generated : string + repo_url : string + latest : int + regression_threshold : double + noise_multiplier : double + baseline_runs : int + runs : array + groups : array + series : array + changes : array + index : array +} + +let INDEX_REFERENCE_POINTS = 5 +let MIN_BASELINE_POINTS = 3 + +//! The median of `vals`, which it does not reorder; 0 for an empty input. +def median(vals : array) : double { + return 0.0lf if (empty(vals)) + var sorted := vals + sort(sorted) + let n = length(sorted) + return (n % 2 == 1) ? sorted[n / 2] : (sorted[n / 2 - 1] + sorted[n / 2]) / 2.0lf +} + +//! Median absolute deviation divided by the median; 0 unless the median is positive. +def relative_mad(vals : array) : double { + return 0.0lf if (empty(vals)) + let med = median(vals) + return 0.0lf if (med <= 0.0lf) + var devs <- [for (x in vals); abs(x - med)] + return median(devs) / med +} + +def geomean(vals : array) : double { + return 0.0lf if (empty(vals)) + var acc = 0.0lf + for (x in vals) { + acc += log(x) + } + return exp(acc / double(length(vals))) +} + +//! The status a record earns from its parts. A night that measured nothing is not `ok`, whatever +//! the reason - every lane unavailable, a filter matching no file, a suite that vanished: an +//! empty night carries no data point and must not read as a good one. +def derive_status(rec : RunRecord) : RunStatus { + return RunStatus.build_failed if (rec.build.status != "ok") + var measured = 0 + for (f in rec.files) { + return RunStatus.bench_failed if (is_failure(f.status)) + measured++ if (f.status == FileStatus.ok) + } + return RunStatus.bench_failed if (measured == 0) + return RunStatus.ok +} + +let DRIVER_FILE_PATH = "utils/internal/bench-stand/nightly.sh" + +//! Fill `rec` in for a night that never benchmarked. A build that failed needs no file - its log +//! tail already rides `build` - while a build that passed and a driver that could not run the +//! suite gets one synthetic failed file carrying `reason`, so the summary and the viewer name it +//! instead of showing an empty night. `rec` keeps the identity the driver's meta gave it. +def mark_failed_run(var rec : RunRecord&; finished, reason : string) { + rec.finished = finished + rec.seconds = 0.0lf + rec.files |> clear() + if (rec.build.status == "ok") { + rec.files |> emplace(FileResult(path = DRIVER_FILE_PATH, id = "bench-stand", group = "stand", + lane = "driver", status = FileStatus.exit_nonzero, exit_code = 1, + message = empty(reason) ? "the driver could not run the suite" : reason)) + } + rec.status = derive_status(rec) +} + +//! Every `*.json` under `dir`, oldest commit first (commit date, then start time). A file that +//! does not parse is named in `errors` and left out, so one corrupt night cannot hide the rest. +def load_runs(runs_dir : string; var errors : array) : array { + var names : array + dir(runs_dir) $(name) { + return if (extension(name) != ".json") + names |> push(name) + } + sort(names) + var runs : array + for (name in names) { + let path = path_join(runs_dir, name) + var rec = RunRecord() + if (!sscan_json(fread(path), rec) || empty(rec.run_id)) { + errors |> push("run record does not parse or has no run_id: {path}") + continue + } + rec.status = derive_status(rec) + runs |> emplace(rec) + } + sort(runs) $(a, b) { + return a.commit.date != b.commit.date ? a.commit.date < b.commit.date : a.started < b.started + } + return <- runs +} + +//! Index of the run that started last - the night the summary and the changes describe; -1 when +//! there are no runs, which every caller branches on. +def latest_run_index(runs : array) : int { + var best = -1 + for (i in range(length(runs))) { + if (best < 0 || runs[i].started >= runs[best].started) { + best = i + } + } + return best +} + +def private summarize_run(rec : RunRecord) : RunSummary { + var s <- RunSummary(id = rec.run_id, sha = rec.commit.sha, date = rec.commit.date, subject = rec.commit.subject, + started = rec.started, status = rec.status, seconds = rec.seconds, build_status = rec.build.status, + build_seconds = rec.build.seconds, build_log_tail = rec.build.log_tail, host = rec.machine.host) + s.lanes := rec.lanes + for (f in rec.files) { + if (f.status == FileStatus.skipped) { + s.skipped |> push(Failure(path = f.path, lane = f.lane, status = "{f.status}", message = f.message)) + } elif (is_failure(f.status)) { + s.failures |> push(Failure(path = f.path, lane = f.lane, status = "{f.status}", message = f.message)) + } else { + s.files_ok++ + } + } + return <- s +} + +def private series_key(f : FileResult; sample_id : string) : string { + return "{f.lane}\t{f.id}#{sample_id}" +} + +//! `groups` collects only the groups that have at least one series - a group whose every file +//! failed every night has no chart to filter, and the failure panel is where it shows. +def private build_series(runs : array; var groups : table) : array { + var index_of : table + var out : array + for (ri, rec in range(length(runs)), runs) { + for (f in rec.files) { + continue if (f.status == FileStatus.skipped) + if (!empty(f.samples)) { + groups |> insert(f.group) + } + for (smp in f.samples) { + let key = series_key(f, smp.id) + var si = index_of?[key] ?? -1 + if (si < 0) { + si = length(out) + index_of[key] = si + out |> emplace(Series(id = "{f.id}#{smp.id}", group = f.group, file = f.id, lane = f.lane)) + } + let n = length(out[si].runs) + continue if (n > 0 && out[si].runs[n - 1] == ri) + out[si].runs |> push(ri) + out[si].ns |> push(smp.ns) + out[si].spread |> push(smp.spread) + } + } + } + sort(out) $(a, b) { + return a.lane != b.lane ? a.lane < b.lane : a.id < b.id + } + return <- out +} + +def private lane_ok_at(rec : RunRecord; lane, file_id : string) : bool { + for (f in rec.files) { + return f.status == FileStatus.ok if (f.lane == lane && f.id == file_id) + } + return false +} + +//! Verdicts for the latest run: each series measured there is compared with the median of its +//! last `baseline_runs` earlier points; a move beyond `regression_threshold` and beyond +//! `noise_multiplier` times the baseline's relative MAD is a regression or an improvement. A +//! series with no earlier point is `new`; one with a baseline but no point tonight, whose file +//! ran ok in that lane, is `missing`. +def detect_changes(runs : array; series : array; latest : int; cfg : SuiteConfig) : array { + var out : array + return <- out if (latest < 0) + for (ser in series) { + var tonight = -1.0lf + var prior : array + for (i in range(length(ser.runs))) { + if (ser.runs[i] == latest) { + tonight = ser.ns[i] + } elif (ser.runs[i] < latest) { + prior |> push(ser.ns[i]) + } + } + if (length(prior) > cfg.baseline_runs) { + prior |> erase(0, length(prior) - cfg.baseline_runs) + } + if (tonight < 0.0lf) { + if (!empty(prior) && lane_ok_at(runs[latest], ser.lane, ser.file)) { + out |> push(Change(id = ser.id, group = ser.group, lane = ser.lane, kind = ChangeKind.missing, baseline = median(prior))) + } + continue + } + if (empty(prior)) { + out |> push(Change(id = ser.id, group = ser.group, lane = ser.lane, kind = ChangeKind.first_seen, value = tonight)) + continue + } + continue if (length(prior) < MIN_BASELINE_POINTS) + let baseline = median(prior) + continue if (baseline <= 0.0lf) + let noise = relative_mad(prior) + let change = tonight / baseline - 1.0lf + let gate = max(cfg.regression_threshold, cfg.noise_multiplier * noise) + continue if (change <= gate && change >= -gate) + let kind = change > gate ? ChangeKind.regression : ChangeKind.improvement + out |> push(Change(id = ser.id, group = ser.group, lane = ser.lane, kind = kind, + baseline = baseline, value = tonight, change = change, noise = noise)) + } + return <- out +} + +def private reference_of(ser : Series) : double { + let firsts <- [for (i in range(min(INDEX_REFERENCE_POINTS, length(ser.ns)))); ser.ns[i]] + return median(firsts) +} + +//! Per group and lane, the geometric mean over its series of value / reference at every run +//! where at least half of the group's series were measured, times 100. +def build_index(series : array; run_count : int) : array { + var index_of : table + var keys_in_order : array + var members : array> + for (si, ser in range(length(series)), series) { + continue if (length(ser.ns) < MIN_BASELINE_POINTS) + let key = "{ser.group}\t{ser.lane}" + var gi = index_of?[key] ?? -1 + if (gi < 0) { + gi = length(members) + index_of[key] = gi + keys_in_order |> push(key) + var fresh_group : array + members |> emplace(fresh_group) + } + members[gi] |> push(si) + } + var out : array + out |> reserve(length(keys_in_order)) + for (key, group_members in keys_in_order, members) { + let tab = find(key, "\t") + var idx <- IndexSeries(group = slice(key, 0, tab), lane = slice(key, tab + 1)) + let refs <- [for (si in group_members); reference_of(series[si])] + for (ri in range(run_count)) { + var ratios : array + for (mi, si in range(length(group_members)), group_members) { + continue if (refs[mi] <= 0.0lf) + for (pi in range(length(series[si].runs))) { + if (series[si].runs[pi] == ri) { + ratios |> push(series[si].ns[pi] / refs[mi]) + break + } + } + } + continue if (length(ratios) * 2 < length(group_members)) + idx.runs |> push(ri) + idx.value |> push(geomean(ratios) * 100.0lf) + } + out |> emplace(idx) + } + sort(out) $(a, b) { + return a.group != b.group ? a.group < b.group : a.lane < b.lane + } + return <- out +} + +def build_dataset(runs : array; cfg : SuiteConfig; generated, repo_url : string) : Dataset { + var ds <- Dataset(generated = generated, repo_url = repo_url, regression_threshold = cfg.regression_threshold, + noise_multiplier = cfg.noise_multiplier, baseline_runs = cfg.baseline_runs) + ds.latest = latest_run_index(runs) + var groups : table + ds.series <- build_series(runs, groups) + ds.groups <- [for (g in keys(groups)); g] + sort(ds.groups) + ds.changes <- detect_changes(runs, ds.series, ds.latest, cfg) + ds.index <- build_index(ds.series, length(runs)) + ds.runs |> reserve(length(runs)) + for (rec in runs) { + ds.runs |> emplace(summarize_run(rec)) + } + if (ds.latest >= 0) { + for (ch in ds.changes) { + if (ch.kind == ChangeKind.regression) { + ds.runs[ds.latest].regressions++ + } elif (ch.kind == ChangeKind.improvement) { + ds.runs[ds.latest].improvements++ + } + } + } + return <- ds +} + +def private count_kind(changes : array; kind : ChangeKind) : int { + var n = 0 + for (ch in changes) { + n++ if (ch.kind == kind) + } + return n +} + +def private write_change_rows(var w : StringBuilderWriter; changes : array; kind : ChangeKind) { + w |> write("| benchmark | lane | baseline ns/op | now ns/op | change | noise |\n|---|---|---:|---:|---:|---:|\n") + for (ch in changes) { + continue if (ch.kind != kind) + w |> write("| `{ch.id}` | {ch.lane} | {ch.baseline:.1f} | {ch.value:.1f} | {ch.change * 100.0lf:+.1f}% | {ch.noise * 100.0lf:.1f}% |\n") + } + w |> write("\n") +} + +//! The markdown the workflow posts as its step summary and the site serves as `summary.md`. +def render_summary(ds : Dataset; site_url : string) : string { + return build_string() $(var w) { + if (ds.latest < 0) { + w |> write("# Benchmark stand\n\nNo run records yet.\n") + return + } + assume run = ds.runs[ds.latest] + w |> write("# Benchmark stand - {run.status} - {run.started}\n\n") + // a hand-driven local run carries no commit and no machine; naming neither beats naming + // both as empty backticks + if (!empty(run.sha) || !empty(run.subject)) { + let short_sha = length(run.sha) > 8 ? slice(run.sha, 0, 8) : run.sha + w |> write("Commit `{short_sha}` {run.subject} ({run.date})") + w |> write(empty(run.host) ? ".\n\n" : " on `{run.host}`.\n\n") + } elif (!empty(run.host)) { + w |> write("On `{run.host}`.\n\n") + } + if (run.build_status != "ok") { + w |> write("## The build failed after {run.build_seconds:.0f} s - nothing was measured\n\n") + w |> write(empty(run.build_log_tail) + ? "The driver recorded no build log; `logs/build-{run.id}.log` on the box has it.\n\n" + : "Last lines of the build log (`logs/build-{run.id}.log` on the box has all of it):\n\n```\n{run.build_log_tail}\n```\n\n") + if (!empty(site_url)) { + w |> write("Charts: {site_url}\n") + } + return + } + w |> write("Build {run.build_seconds:.0f} s, suite {run.seconds:.0f} s; {run.files_ok} file runs ok, {length(run.failures)} failed, {length(run.skipped)} skipped.\n\n") + w |> write("Lanes:") + for (lane, state in keys(run.lanes), values(run.lanes)) { + w |> write(" {lane} = {state};") + } + w |> write("\n\n") + if (!empty(run.failures)) { + w |> write("## Failures ({length(run.failures)})\n\n| file | lane | status | message |\n|---|---|---|---|\n") + for (f in run.failures) { + w |> write("| `{f.path}` | {f.lane} | {f.status} | {replace(f.message, "|", "\\|")} |\n") + } + w |> write("\nEach failure's last log lines are in the run record `runs/{run.id}.json`.\n\n") + } + let regs = count_kind(ds.changes, ChangeKind.regression) + let imps = count_kind(ds.changes, ChangeKind.improvement) + let news = count_kind(ds.changes, ChangeKind.first_seen) + let miss = count_kind(ds.changes, ChangeKind.missing) + w |> write("## Changes: {regs} regressions, {imps} improvements, {news} first seen, {miss} missing\n\n") + w |> write("A change counts when it passes both {ds.regression_threshold * 100.0lf:.0f}% and {ds.noise_multiplier:.1f}x the baseline noise; baseline = median of the previous {ds.baseline_runs} runs.\n\n") + if (regs > 0) { + w |> write("### Regressions\n\n") + write_change_rows(w, ds.changes, ChangeKind.regression) + } + if (imps > 0) { + w |> write("### Improvements\n\n") + write_change_rows(w, ds.changes, ChangeKind.improvement) + } + if (miss > 0) { + w |> write("### Missing tonight (measured before, file ran ok)\n\n") + for (ch in ds.changes) { + continue if (ch.kind != ChangeKind.missing) + w |> write("- `{ch.id}` ({ch.lane})\n") + } + w |> write("\n") + } + if (!empty(run.skipped)) { + w |> write("## Skipped ({length(run.skipped)})\n\n") + for (f in run.skipped) { + w |> write("- `{f.path}` ({f.lane}): {f.message}\n") + } + w |> write("\n") + } + if (!empty(site_url)) { + w |> write("Charts: {site_url}\n") + } + } +} + +//! 0 when the latest run is ok; 1 when it failed (build or any file); 2 when it is ok but +//! regressed and the caller asked regressions to fail. +def summary_exit_code(ds : Dataset; fail_on_regression : bool) : int { + return 1 if (ds.latest < 0 || ds.runs[ds.latest].status != RunStatus.ok) + return 2 if (fail_on_regression && ds.runs[ds.latest].regressions > 0) + return 0 +} diff --git a/utils/internal/bench-stand/bench_runner.das b/utils/internal/bench-stand/bench_runner.das new file mode 100644 index 0000000000..e52430ceb2 --- /dev/null +++ b/utils/internal/bench-stand/bench_runner.das @@ -0,0 +1,432 @@ +options gen2 +options indenting = 4 + +module bench_runner public + +require bench_suite +require daslib/fio +require daslib/json_boost +require daslib/strings_boost +require daslib/strings_convert +require strings + +//! How one benchmark file's process ended. +enum FileStatus { + ok + skipped + spawn_failed + compile_error + failed + timeout + memory + exit_nonzero +} + +//! One `run("name")` arm of one `[benchmark]` function, aggregated over the repeats: +//! `ns` is the minimum ns/op (the noise-robust statistic), `ns_median` the median, +//! `spread` `(max - min) / min`. The four allocation fields are per-op medians. +struct Sample { + id : string + ns : double + ns_median : double + spread : double + runs : int + bytes : int64 + allocs : int64 + string_bytes : int64 + string_allocs : int64 +} + +//! One benchmark file in one lane. `samples` carries only the arms whose `[benchmark]` function +//! passed - a failed function's arms are dropped, so a broken assertion cannot publish a number. +struct FileResult { + path : string + id : string + group : string + lane : string + status : FileStatus + exit_code : int + seconds : double + tier : string + message : string + log_tail : string + samples : array +} + +struct RunLimits { + timeout_seconds : int = 900 + memory_limit_mb : int = 4096 + repeat : int = 3 +} + +//! `dastest --bench-format json` emits one of these per arm per repeat; the report file +//! (`--json-file`) carries the per-function verdicts. +struct BenchStatsLine { + name : string + sub_name : string + n : int + time_ns : int64 + allocs : uint64 + heap_bytes : uint64 + string_allocs : uint64 + string_heap_bytes : uint64 + func_type : string +} + +struct ReportTest { + name : string + passed : bool +} + +struct DastestReport { + success : bool + tests : array +} + +let TAIL_LINES = 60 +let TAIL_LINE_CHARS = 400 + +def is_failure(st : FileStatus) : bool { + return st != FileStatus.ok && st != FileStatus.skipped +} + +def private median(var vals : array) : double { + return 0.0lf if (empty(vals)) + sort(vals) + let n = length(vals) + return (n % 2 == 1) ? vals[n / 2] : (vals[n / 2 - 1] + vals[n / 2]) / 2.0lf +} + +def private median_i64(var vals : array) : int64 { + return 0l if (empty(vals)) + sort(vals) + return vals[length(vals) / 2] +} + +def private keep_tail(var tail : array; line : string) { + if (length(tail) >= TAIL_LINES) { + tail |> erase(0) + } + tail |> push(length(line) > TAIL_LINE_CHARS ? slice(line, 0, TAIL_LINE_CHARS) : line) +} + +def private first_line(text : string) : string { + let nl = find(text, "\n") + return nl < 0 ? text : slice(text, 0, nl) +} + +def private error_or_last_line(text : string) : string { + var last = "" + for (raw in split(text, "\n")) { + let line = strip(raw) + continue if (empty(line)) + return line if (starts_with(line, "FATAL") || starts_with(line, "error") || find(line, "[E]") >= 0) + last = line + } + return last +} + +struct private OutputScan { + by_arm : table> + arm_order : array + tail : array + compile_error : string + failed_to_compile : string + fatal : string + failed_fns : table + tier : string +} + +def private note_diagnostic(var scan : OutputScan; line : string) { + if (empty(scan.compile_error) && find(line, "error[") >= 0 && find(line, "]: ") >= 0) { + scan.compile_error = strip(line) + } + if (empty(scan.failed_to_compile) && find(line, "Failed to compile") >= 0) { + scan.failed_to_compile = strip(line) + } + if (empty(scan.fatal) && starts_with(line, "FATAL")) { + scan.fatal = strip(line) + } + let fail_at = find(line, "--- FAIL '") + return if (fail_at < 0) + let name_start = fail_at + length("--- FAIL '") + let name_end = find(line, "'", name_start) + if (name_end > name_start) { + scan.failed_fns |> insert(slice(line, name_start, name_end)) + } +} + +def private scan_line(var scan : OutputScan; line : string) { + if (starts_with(line, "\{")) { + var st : BenchStatsLine + if (sscan_json(line, st) && st.n > 0) { + let key = "{st.name}/{st.sub_name}" + if (!key_exists(scan.by_arm, key)) { + scan.arm_order |> push(key) + } + scan.by_arm[key] |> push(st) + if (empty(scan.tier)) { + scan.tier = st.func_type + } + return + } + } + keep_tail(scan.tail, line) + note_diagnostic(scan, line) +} + +//! True when the report parsed; its failed functions join the scan's. +def private note_report(var scan : OutputScan; report_text : string) : bool { + return false if (empty(report_text)) + var report : DastestReport + return false if (!sscan_json(report_text, report)) + for (test in report.tests) { + if (!test.passed) { + scan.failed_fns |> insert(test.name) + } + } + return true +} + +def private aggregate_arm(key : string; stats : array) : Sample { + var ns <- [for (st in stats); double(st.time_ns) / double(st.n)] + var bytes <- [for (st in stats); int64(st.heap_bytes) / int64(st.n)] + var allocs <- [for (st in stats); int64(st.allocs) / int64(st.n)] + var sbytes <- [for (st in stats); int64(st.string_heap_bytes) / int64(st.n)] + var sallocs <- [for (st in stats); int64(st.string_allocs) / int64(st.n)] + sort(ns) + let lo = ns[0] + let hi = ns[length(ns) - 1] + return Sample(id = key, ns = lo, ns_median = median(ns), + spread = lo > 0.0lf ? (hi - lo) / lo : 0.0lf, runs = length(ns), + bytes = median_i64(bytes), allocs = median_i64(allocs), + string_bytes = median_i64(sbytes), string_allocs = median_i64(sallocs)) +} + +def private decide_status(scan : OutputScan; report_ok : bool; exit_code : int; var res : FileResult&) { + if (!empty(scan.failed_to_compile)) { + res.status = FileStatus.compile_error + res.message = !empty(scan.compile_error) ? scan.compile_error : scan.failed_to_compile + } elif (!empty(scan.failed_fns)) { + res.status = FileStatus.failed + var names <- [for (n in keys(scan.failed_fns)); n] + sort(names) + res.message = "benchmark function(s) failed: {join(names, ", ")}" + } elif (exit_code != 0 && report_ok && !empty(res.samples)) { + // dastest wrote a finished report and the arms are here, so the measurement stands and + // only the shutdown went wrong - a leak dump, a smart-pointer assert, a signal after the + // last benchmark. Worth saying, never worth discarding a night for: a RelWithDebInfo + // host's allocation tracker takes every process to 1 and would fail the whole suite. + res.status = FileStatus.ok + if (empty(scan.fatal)) { + res.message = "measured, then exited with code {exit_code} at shutdown" + } else { + res.message = "measured, then exited with code {exit_code} at shutdown: {scan.fatal}" + } + } elif (exit_code != 0) { + res.status = FileStatus.exit_nonzero + if (!empty(scan.fatal)) { + res.message = scan.fatal + } elif (empty(res.samples)) { + res.message = "process exited with code {exit_code} before measuring anything, printing no diagnostic (a signal number when it was killed; a missing or unrunnable daslang binary lands here)" + } else { + res.message = "process exited with code {exit_code} without a finished dastest report - the suite did not run to its end" + } + } elif (!report_ok) { + res.status = FileStatus.exit_nonzero + res.message = "dastest wrote no report file - the process ended before the suite finished" + } elif (empty(res.samples)) { + res.status = FileStatus.failed + res.message = "the file ran but produced no benchmark samples (no [benchmark] function, or every arm reported n = 0)" + } else { + res.status = FileStatus.ok + } +} + +//! Turn a finished process's output into the result's samples, status and message. `lines` is +//! the merged stdout+stderr, `report_text` the `--json-file` contents (empty when the process +//! never wrote it). `res.status` on entry is the limit verdict (`timeout` / `memory`) or `ok`, +//! and a limit verdict is kept whatever the output says. +def parse_bench_output(lines : array; report_text : string; exit_code : int; var res : FileResult&) { + var scan <- OutputScan() + for (line in lines) { + scan_line(scan, line) + } + res.log_tail = join(scan.tail, "\n") + res.exit_code = exit_code + res.tier = scan.tier + let report_ok = note_report(scan, report_text) + res.samples |> reserve(length(scan.arm_order)) + for (key in scan.arm_order) { + let fn_name = slice(key, 0, find(key, "/")) + continue if (key_exists(scan.failed_fns, fn_name)) + res.samples |> emplace(aggregate_arm(key, scan.by_arm[key])) + } + return if (res.status == FileStatus.timeout || res.status == FileStatus.memory) + decide_status(scan, report_ok, exit_code, res) +} + +def private rss_mb_from_proc(pid : int) : int { + let text = fread("/proc/{pid}/status") + return -1 if (empty(text)) + let at = find(text, "VmRSS:") + return -1 if (at < 0) + let line = first_line(slice(text, at + length("VmRSS:"))) + let kb = try_to_int(strip(replace(line, "kB", ""))) ?? -1 + return -1 if (kb < 0) + return (kb + 1023) / 1024 +} + +def private rss_mb_from_ps(pid : int) : int { + var out = "" + let code = run_and_capture(["/bin/ps", "-o", "rss=", "-p", "{pid}"], out, 5.0) + return -1 if (code != 0) + let kb = try_to_int(strip(out)) ?? -1 + return -1 if (kb < 0) + return (kb + 1023) / 1024 +} + +//! Resident set size of a live process in MB, rounded up so a process holding memory never +//! reads as 0 and a cap fires a hair early rather than late: `/proc` where it exists, `ps` +//! otherwise (macOS), -1 when neither answers - a caller reading -1 as "under the cap" is what +//! makes a missing source disable the memory limit rather than kill every child. +def process_rss_mb(pid : int) : int { + let from_proc = rss_mb_from_proc(pid) + return from_proc if (from_proc >= 0) + return rss_mb_from_ps(pid) +} + +//! The three lanes differ only in argv. `aot` needs BOTH `-use-aot` on the host and `--use-aot` +//! on dastest: without the second one every function silently interprets and the lane measures +//! the interpreter under an AOT label. With it, a function missing its stub is `error[50101]`, +//! so a run that finishes is proof the stubs were live. +def private bench_argv(bin, repo_root : string; f : BenchFile; lane : string; report_path : string; repeat : int) : array { + var argv <- [bin] + if (lane == "jit") { + argv |> push("-jit") + } elif (lane == "aot") { + argv |> push("-use-aot") + } + argv |> push(to_generic_path(path_join(repo_root, "dastest/dastest.das"))) + argv |> push("--") + if (lane == "aot") { + argv |> push("--use-aot") + } + argv |> push_from(["--bench", "--bench-format", "json", "--json-file", report_path, + "--count", "{repeat}", "--test", to_generic_path(path_join(path_join(repo_root, "benchmarks"), f.path))]) + return <- argv +} + +def private new_result(f : BenchFile; lane : string) : FileResult { + return <- FileResult(path = f.path, id = f.id, group = f.group, lane = lane) +} + +//! Run one benchmark file in one lane under the limits, from the repo root. The child is +//! killed when it outlives `timeout_seconds` or its RSS passes `memory_limit_mb`; the result +//! then says so and keeps the arms that completed before the kill. +def run_bench_file(bin, repo_root : string; f : BenchFile; lane : string; limits : RunLimits) : FileResult { + var res <- new_result(f, lane) + let skipped = skip_reason(f, lane) + if (!empty(skipped)) { + res.status = FileStatus.skipped + res.message = skipped + return <- res + } + var err = "" + let report_path = create_temp_file("bench_stand_", ".json", err) + if (empty(report_path)) { + res.status = FileStatus.spawn_failed + res.message = "cannot create the dastest report file: {err}" + return <- res + } + remove(report_path) + let argv <- bench_argv(bin, repo_root, f, lane, report_path, limits.repeat) + var lines : array + var exit_code = process_running + let t0 = ref_time_ticks() + let noenv : array + with_process(argv, repo_root, noenv) $(var p) { + unsafe { + if (process_pid(p) <= 0) { + res.status = FileStatus.spawn_failed + res.message = "cannot spawn {argv[0]}" + return + } + var checks = 0 + while (true) { + process_drain(p) $(line) { + lines |> push(line) + } + exit_code = process_poll(p) + break if (exit_code != process_running) + let elapsed = double(get_time_usec(t0)) / 1000000.0lf + if (elapsed > double(limits.timeout_seconds)) { + res.status = FileStatus.timeout + res.message = "killed after {limits.timeout_seconds} s (timeout_seconds)" + process_kill(p) + exit_code = process_wait(p, 10.0) + break + } + checks++ + if (checks % 10 == 0) { + let rss = process_rss_mb(process_pid(p)) + if (rss > limits.memory_limit_mb) { + res.status = FileStatus.memory + res.message = "killed at {rss} MB resident (memory_limit_mb = {limits.memory_limit_mb})" + process_kill(p) + exit_code = process_wait(p, 10.0) + break + } + } + sleep(100u) + } + process_drain(p) $(line) { + lines |> push(line) + } + } + } + res.seconds = double(get_time_usec(t0)) / 1000000.0lf + return <- res if (res.status == FileStatus.spawn_failed) + let report_text = fread(report_path) + remove(report_path) + parse_bench_output(lines, report_text, exit_code, res) + return <- res +} + +//! Empty when the AOT binary exists and reports the AOT tier; otherwise the reason. An absent +//! binary is the ordinary case on a tree nobody built the AOT target in. +def probe_aot(aot_bin : string) : string { + return "no AOT binary at {aot_bin} - build the test_aot target first" if (!fexist(aot_bin)) + var err = "" + let probe = create_temp_file("bench_stand_aot_", ".das", err) + return "cannot create the probe file: {err}" if (empty(probe)) + fwrite(probe, "options gen2\n[export]\ndef main() \{\n print(\"aot-probe-ok\\n\")\n\}\n") + var out = "" + let code = run_and_capture([aot_bin, "-use-aot", probe], out, 120.0) + remove(probe) + // The marker, as with the JIT probe: it prints only if the binary got as far as running a + // program. A trivial program has no stubs of its own, so this proves the binary runs, never + // that a benchmark's stubs are present - `--use-aot` and error[50101] prove that, per file. + return "" if (find(out, "aot-probe-ok") >= 0) + return "{aot_bin} -use-aot ran no program (exit {code}): {error_or_last_line(out)}" +} + +//! Empty when `daslang -jit` runs a trivial program on this box; otherwise the reason the +//! JIT lane is unavailable (dasLLVM not built, a missing LLVM library). +def probe_jit(daslang_bin : string) : string { + var err = "" + let probe = create_temp_file("bench_stand_jit_", ".das", err) + return "cannot create the probe file: {err}" if (empty(probe)) + fwrite(probe, "options gen2\n[export]\ndef main() \{\n print(\"jit-probe-ok\\n\")\n\}\n") + var out = "" + let code = run_and_capture([daslang_bin, "-jit", probe], out, 120.0) + remove(probe) + // The marker, not the exit code, is the evidence: it prints only after the JIT compiled and + // ran the function. A RelWithDebInfo host exits 1 from its allocation tracker's report on + // every process, and keying on the code there would delete half the stand's measurements + // on every developer build. + return "" if (find(out, "jit-probe-ok") >= 0) + let said = error_or_last_line(out) + return "daslang -jit produced no jit-compiled run (exit {code}) and printed nothing - is {daslang_bin} runnable?" if (empty(said)) + return "daslang -jit produced no jit-compiled run (exit {code}): {said}" +} diff --git a/utils/internal/bench-stand/bench_suite.das b/utils/internal/bench-stand/bench_suite.das new file mode 100644 index 0000000000..a277fbbbf6 --- /dev/null +++ b/utils/internal/bench-stand/bench_suite.das @@ -0,0 +1,155 @@ +options gen2 +options indenting = 4 + +module bench_suite public + +require daslib/fio +require daslib/json_boost + +//! Per-file knobs from `suite.json` `files`; a zero value means "inherit the suite default". +//! `skip` takes the file out of every lane, `skip_lanes` out of the ones it names - a file the +//! AOT emitter cannot handle still measures in the interpreter. +struct FileOverride { + @optional skip : string + @optional skip_lanes : table + @optional timeout_seconds : int + @optional memory_limit_mb : int +} + +//! Field initializers are the defaults a missing key keeps. +struct SuiteConfig { + root : string = "benchmarks" + exclude : array + lanes : array + repeat : int = 3 + timeout_seconds : int = 900 + memory_limit_mb : int = 4096 + regression_threshold : double = 0.10lf + noise_multiplier : double = 3.0lf + baseline_runs : int = 7 + files : table + lane_excludes : table> +} + +//! One benchmark file the stand runs. `path` is relative to the suite root with `/` separators, +//! `id` is `path` without the extension, `group` is its directory. A non-empty `skip` is the reason +//! the file is listed but not run. +struct BenchFile { + path : string + id : string + group : string + timeout_seconds : int + memory_limit_mb : int + skip : string + skip_lanes : table +} + +//! Why this file does not run in this lane, empty when it does. A whole-file skip answers for +//! every lane; a lane skip answers only for its own. +def skip_reason(f : BenchFile; lane : string) : string { + return f.skip if (!empty(f.skip)) + return f.skip_lanes?[lane] ?? "" +} + +//! Load `suite.json`. Fails closed: any missing, unparseable or out-of-range field sets `error` +//! and returns the defaults, so a typo cannot silently shrink the suite. `exclude` takes a file +//! out of the tree; `lane_excludes` takes it out of one lane, keyed by glob and carrying the +//! reason the site shows every night. +def load_suite_config(path : string; var error : string&) : SuiteConfig { + error = "" + var cfg = SuiteConfig() + if (!stat(path).is_reg) { + error = "suite config not found: {path}" + return <- cfg + } + if (!sscan_json(fread(path), cfg)) { + error = "suite config does not parse as JSON: {path}" + return <- cfg + } + if (empty(cfg.lanes)) { + error = "suite config lists no lanes: {path}" + } elif (cfg.repeat <= 0 || cfg.timeout_seconds <= 0 || cfg.memory_limit_mb <= 0) { + error = "suite config needs positive repeat, timeout_seconds and memory_limit_mb: {path}" + } elif (cfg.baseline_runs < 3) { + error = "suite config needs baseline_runs >= 3: {path}" + } + for (lane in keys(cfg.lane_excludes)) { + if (find_index(cfg.lanes, lane) < 0) { + error = "suite config excludes files from lane \"{lane}\", which it does not run: {path}" + } + } + for (lane in cfg.lanes) { + if (lane != "interp" && lane != "jit" && lane != "aot") { + error = "suite config names an unknown lane \"{lane}\" (interp, jit or aot): {path}" + } + } + return <- cfg +} + +def private is_excluded(rel : string; excludes : array) : bool { + for (pattern in excludes) { + return true if (match_glob(pattern, rel)) + } + return false +} + +//! Every `.das` file under `/` that no `exclude` glob matches, sorted by +//! path, with the per-file overrides applied. A `files` override naming a path that does not +//! exist is reported through `error` so a renamed benchmark cannot silently lose its skip. +def discover_files(repo_root : string; cfg : SuiteConfig; var error : string&) : array { + error = "" + let root = path_join(repo_root, cfg.root) + var rels : array + dir_rec(root) $(name, is_dir) { + let rel = to_generic_path(name) + return if (is_dir || extension(rel) != ".das" || is_excluded(rel, cfg.exclude)) + rels |> push(rel) + } + sort(rels) + var out : array + out |> reserve(length(rels)) + for (rel in rels) { + var f = BenchFile(path = rel, id = stem_path(rel), group = group_of(rel), + timeout_seconds = cfg.timeout_seconds, memory_limit_mb = cfg.memory_limit_mb) + for (lane in keys(cfg.lane_excludes)) { + cfg.lane_excludes |> get(lane) $(globs) { + for (pattern in keys(globs)) { + continue if (!match_glob(pattern, rel)) + f.skip_lanes[lane] = globs?[pattern] ?? "" + } + } + } + cfg.files |> get(rel) $(ov) { + f.skip = ov.skip + for (lane in keys(ov.skip_lanes)) { + f.skip_lanes[lane] = ov.skip_lanes?[lane] ?? "" + } + if (ov.timeout_seconds > 0) { + f.timeout_seconds = ov.timeout_seconds + } + if (ov.memory_limit_mb > 0) { + f.memory_limit_mb = ov.memory_limit_mb + } + } + out |> emplace(f) + } + for (k in keys(cfg.files)) { + if (!stat(path_join(root, k)).is_reg) { + error = "suite config overrides \"{k}\", which is not a file under {root}" + } + } + return <- out +} + +//! `core/hash/test02.das` -> `core/hash`; a file at the root has group `root`. +def group_of(rel : string) : string { + let d = to_generic_path(dir_name(rel)) + return (empty(d) || d == ".") ? "root" : d +} + +//! `core/hash/test02.das` -> `core/hash/test02`. +def stem_path(rel : string) : string { + let d = to_generic_path(dir_name(rel)) + let s = stem(rel) + return (empty(d) || d == ".") ? s : "{d}/{s}" +} diff --git a/utils/internal/bench-stand/caddy.snippet b/utils/internal/bench-stand/caddy.snippet new file mode 100644 index 0000000000..3f500be9d9 --- /dev/null +++ b/utils/internal/bench-stand/caddy.snippet @@ -0,0 +1,17 @@ +# bench-stand routes for the daslang.io vhost in /etc/caddy/Caddyfile. +# This file is the authoritative copy of the stand's public boundary — the +# deployed Caddyfile is edited to match it, never the other way round. +# +# Paste inside the `daslang.io { ... }` block, ahead of `root`/`file_server`. +# `bench-stand-deploy.sh caddy` does the splice, validates, and reloads. + +# The nightly benchmark stand: a static tree the `bench` user rewrites every +# night (viewer, data.json, status.json, summary.md, runs/). No service behind +# it. no-cache: the viewer fetches data.json on every visit and a day-old +# cached copy would show yesterday's night as tonight's. +redir /bench /bench/ 308 +handle_path /bench/* { + root * /srv/bench-stand/site + header Cache-Control "no-cache" + file_server +} diff --git a/utils/internal/bench-stand/main.das b/utils/internal/bench-stand/main.das new file mode 100644 index 0000000000..20cea7aeb3 --- /dev/null +++ b/utils/internal/bench-stand/main.das @@ -0,0 +1,240 @@ +options gen2 +options indenting = 4 + +require bench_suite +require bench_runner +require bench_history +require daslib/clargs +require daslib/fio +require daslib/json_boost +require daslib/strings_boost +require strings + +//! The daslang benchmark stand - the tool half of the nightly. Two verbs, both run from the +//! repo root with the tree's own binary: +//! daslang utils/internal/bench-stand/main.das -- run --meta meta.json --out runs/.json +//! daslang utils/internal/bench-stand/main.das -- report --runs runs/ --out-data site/data.json --out-summary site/summary.md +//! The box-side driver is `nightly.sh`; the exit codes are stated where they are produced. + +[CommandLineArgs] +struct Config { + @clarg_positional + @clarg_doc = "run | report" + verb : string + + @clarg_doc = "Repo root (default: .)" + root : string + + @clarg_doc = "suite.json to use (default: the one beside this tool)" + suite : string + + @clarg_doc = "run: daslang binary the interp and jit lanes use (default: bin/daslang under --root)" + bin : string + + @clarg_doc = "run: binary the aot lane uses - daslang with the benchmarks' AOT stubs linked in (default: bin/test_aot under --root)" + aot_bin : string + + @clarg_doc = "run: JSON with run_id, started, commit, machine, build - written by nightly.sh" + meta : string + + @clarg_doc = "run: where to write the run record" + out : string + + @clarg_doc = "run: only files whose path contains this substring" + filter : string + + @clarg_doc = "run: comma-separated lanes to run (default: suite.json lanes)" + lanes : string + + @clarg_doc = "run: benchmark nothing - write the record of a night that could not run the suite, with this reason (non-empty), and exit 1" + failed : string + + @clarg_doc = "run: repeats per file (default: suite.json repeat)" + repeat : int + + @clarg_doc = "report: directory of run records" + runs : string + + @clarg_doc = "report: where to write data.json" + out_data : string + + @clarg_doc = "report: where to write summary.md" + out_summary : string + + @clarg_doc = "report: ISO timestamp stamped into data.json (default: now, UTC)" + generated : string + + @clarg_doc = "report: repository URL for commit links" + repo_url : string + + @clarg_doc = "report: site URL the summary links to" + site_url : string + + @clarg_doc = "report: exit 2 when the latest run regressed" + fail_on_regression : bool + + @clarg_short = "?" + @clarg_name = "show-help" + @clarg_doc = "Show this help and exit" + help : bool +} + +def private iso_now() : string { + return "{iso8601_now()}" +} + +def private write_text_file(path, text : string) : bool { + if (!fwrite(path, text)) { + to_log(LOG_ERROR, "cannot write {path}\n") + return false + } + return true +} + +def private suite_path_of(cfg : Config) : string { + return empty(cfg.suite) ? path_join(get_das_root(), "utils/internal/bench-stand/suite.json") : cfg.suite +} + +def private read_meta(path : string; var rec : RunRecord&) : bool { + var meta = RunMeta() + return false if (!sscan_json(fread(path), meta) || empty(meta.run_id)) + rec.run_id = meta.run_id + rec.started = meta.started + rec.commit = meta.commit + rec.machine = meta.machine + rec.build = meta.build + return true +} + +//! The binary a lane measures with: the AOT lane needs its own, the other two share daslang. +def private lane_binary(lane, bin, aot_bin : string) : string { + return lane == "aot" ? aot_bin : bin +} + +def private probe_lanes(var rec : RunRecord&; lanes : array; bin, aot_bin : string) { + for (lane in lanes) { + if (lane == "jit") { + let why = probe_jit(bin) + rec.lanes[lane] = empty(why) ? "ok" : why + } elif (lane == "aot") { + let why = probe_aot(aot_bin) + rec.lanes[lane] = empty(why) ? "ok" : why + } else { + rec.lanes[lane] = "ok" + } + to_log(LOG_INFO, "lane {lane}: {rec.lanes[lane]}\n") + } +} + +def private matches_filter(f : BenchFile; filter : string) : bool { + return empty(filter) || find(f.path, filter) >= 0 +} + +def private verb_run(cfg : Config; suite : SuiteConfig) : int { + if (empty(cfg.meta) || empty(cfg.out)) { + to_log(LOG_ERROR, "bench-stand run: --meta and --out are required\n") + return 1 + } + let root = empty(cfg.root) ? "." : cfg.root + let bin = empty(cfg.bin) ? path_join(root, "bin/daslang") : cfg.bin + let aot_bin = empty(cfg.aot_bin) ? path_join(root, "bin/test_aot") : cfg.aot_bin + var rec = RunRecord() + if (!read_meta(cfg.meta, rec)) { + to_log(LOG_ERROR, "bench-stand run: --meta {cfg.meta} does not parse or has no run_id\n") + return 1 + } + if (!empty(cfg.failed)) { + mark_failed_run(rec, iso_now(), cfg.failed) + return 1 if (!write_text_file(cfg.out, sprint_json(rec, true))) + to_log(LOG_ERROR, "bench-stand run: {rec.status} - {cfg.failed}; record {cfg.out}\n") + return 1 + } + var error = "" + let all_files <- discover_files(root, suite, error) + if (!empty(error)) { + to_log(LOG_ERROR, "bench-stand run: {error}\n") + return 1 + } + let lanes <- empty(cfg.lanes) ? clone_to_move(suite.lanes) : split(cfg.lanes, ",") + probe_lanes(rec, lanes, bin, aot_bin) + let repeat = cfg.repeat > 0 ? cfg.repeat : suite.repeat + let t0 = ref_time_ticks() + let failed = benchmark_every_file(rec, all_files, cfg.filter, lanes, bin, aot_bin, root, repeat) + rec.seconds = double(get_time_usec(t0)) / 1000000.0lf + rec.finished = iso_now() + rec.status = derive_status(rec) + return 1 if (!write_text_file(cfg.out, sprint_json(rec, true))) + let ok = rec.status == RunStatus.ok + to_log(ok ? LOG_INFO : LOG_ERROR, "bench-stand run: {rec.status} - {length(rec.files)} file runs, {failed} failed, {rec.seconds:.0f} s; record {cfg.out}\n") + return ok ? 0 : 1 +} + +//! Runs every file in every runnable lane into `rec.files`, one progress line each; returns how +//! many of those runs failed. A lane the probe marked unavailable contributes no file. +def private benchmark_every_file(var rec : RunRecord&; files : array; filter : string; + lanes : array; bin, aot_bin, root : string; repeat : int) : int { + var limits = RunLimits(repeat = repeat) + var matching = 0 + for (f in files) { + matching++ if (matches_filter(f, filter)) + } + let total = matching * length(lanes) + var done = 0 + var failed = 0 + rec.files |> reserve(total) + for (f in files) { + continue if (!matches_filter(f, filter)) + for (lane in lanes) { + done++ + continue if (rec.lanes[lane] != "ok") + limits.timeout_seconds = f.timeout_seconds + limits.memory_limit_mb = f.memory_limit_mb + var res <- run_bench_file(lane_binary(lane, bin, aot_bin), root, f, lane, limits) + failed++ if (is_failure(res.status)) + to_log(is_failure(res.status) ? LOG_ERROR : LOG_INFO, + "[{done}/{total}] {f.path} {lane}: {res.status} {res.seconds:.1f}s {length(res.samples)} arms{empty(res.message) ? "" : " - " + res.message}\n") + rec.files |> emplace(res) + } + } + return failed +} + +def private verb_report(cfg : Config; suite : SuiteConfig) : int { + if (empty(cfg.runs) || empty(cfg.out_data) || empty(cfg.out_summary)) { + to_log(LOG_ERROR, "bench-stand report: --runs, --out-data and --out-summary are required\n") + return 1 + } + var errors : array + let runs <- load_runs(cfg.runs, errors) + for (e in errors) { + to_log(LOG_WARNING, "bench-stand report: {e}\n") + } + let ds <- build_dataset(runs, suite, empty(cfg.generated) ? iso_now() : cfg.generated, cfg.repo_url) + return 1 if (!write_text_file(cfg.out_data, sprint_json(ds, false))) + let md = render_summary(ds, cfg.site_url) + return 1 if (!write_text_file(cfg.out_summary, md)) + to_log(LOG_INFO, "bench-stand report: {length(runs)} runs, {length(ds.series)} series, {length(ds.changes)} changes -> {cfg.out_data}, {cfg.out_summary}\n") + return summary_exit_code(ds, cfg.fail_on_regression) +} + +[export] +def main() : int { + var r <- parse_args(type) + if (r |> is_err) { + to_log(LOG_ERROR, "error: {r |> unwrap_err}\n") + print_help(get_command_info(type), "bench-stand") + return 1 + } + let cfg <- r |> move_unwrap + if (cfg.help || (cfg.verb != "run" && cfg.verb != "report")) { + print_help(get_command_info(type), "bench-stand") + return cfg.help ? 0 : 1 + } + var error = "" + let suite <- load_suite_config(suite_path_of(cfg), error) + if (!empty(error)) { + to_log(LOG_ERROR, "bench-stand: {error}\n") + return 1 + } + return cfg.verb == "run" ? verb_run(cfg, suite) : verb_report(cfg, suite) +} diff --git a/utils/internal/bench-stand/nightly.sh b/utils/internal/bench-stand/nightly.sh new file mode 100755 index 0000000000..6fd2366c2b --- /dev/null +++ b/utils/internal/bench-stand/nightly.sh @@ -0,0 +1,363 @@ +#!/bin/bash +# bench-stand nightly driver - the box side of the daslang benchmark stand. Runs as the +# `bench` user on the web box; the GitHub workflow (.github/workflows/nightly_bench.yml) +# reaches it over ssh with a forced command (`gate`). Verbs: +# +# start [fail-on-regression] +# fetch, resolve , launch `run` detached; prints the run id +# run the whole night, synchronous: checkout, build, benchmark, report, publish +# follow [run_id] stream the run's log until it finishes; exits with the run's exit code +# status print status.json +# gate ssh forced command: dispatches start/follow/status from SSH_ORIGINAL_COMMAND +# +# Layout under BENCH_STAND_HOME (default /srv/bench-stand): +# src/ clone of the repository - the tree each night checks out, builds and benchmarks +# runs/ one JSON record per night (utils/internal/bench-stand/README.md - data model) +# site/ what Caddy serves at /bench/: viewer, data.json, status.json, summary.md, runs -> ../runs +# logs/ nightly-.log +# last_good/ bin/daslang + daslib + the tool, from the last night whose build passed; renders +# the report when tonight's build fails, so a broken master still publishes a red night +# state/ lock and current run id +# +# Environment knobs (all optional; documented in the README): +# BENCH_STAND_HOME layout root (default /srv/bench-stand) +# BENCH_STAND_REPO_URL commit-link base (default https://github.com/GaijinEntertainment/daScript) +# BENCH_STAND_SITE_URL what the summary links to (default https://daslang.io/bench/) +# BENCH_STAND_JOBS build parallelism (default nproc) +# BENCH_STAND_CMAKE_ARGS extra configure arguments +# BENCH_STAND_BUILD `skip` reuses the existing build - local dry runs only +# BENCH_STAND_AOT `skip` leaves the AOT binary unbuilt; the aot lane then reports itself +# unavailable and the other lanes still measure +# BENCH_STAND_FILTER forwarded as `run --filter` - local dry runs only +# BENCH_STAND_LANES forwarded as `run --lanes` (comma-separated) +# BENCH_STAND_REPEAT forwarded as `run --repeat` +# BENCH_STAND_FAIL_ON_REGRESSION non-empty: `report --fail-on-regression` (exit 2 on a regression); +# `start`'s second word is the same request from a caller with no environment +# +# One variable this script reads that nothing here sets: SSH_ORIGINAL_COMMAND, which the `gate` +# verb parses. An inbound ssh connection whose authorized_keys line forces `nightly.sh gate` +# arrives with the caller's words there and nowhere else. +set -euo pipefail + +HOME_DIR=${BENCH_STAND_HOME:-/srv/bench-stand} +SRC=$HOME_DIR/src +RUNS=$HOME_DIR/runs +SITE=$HOME_DIR/site +LOGS=$HOME_DIR/logs +LAST_GOOD=$HOME_DIR/last_good +STATE=$HOME_DIR/state +TOOL=utils/internal/bench-stand +REPO_URL=${BENCH_STAND_REPO_URL:-https://github.com/GaijinEntertainment/daScript} +SITE_URL=${BENCH_STAND_SITE_URL:-https://daslang.io/bench/} +# Portable, because every verb of this script runs wherever a developer sits: nproc is +# GNU-only, and on a box without it the bare $(nproc) failed the whole script with 127 +# before it ever reached a verb. +cpu_count() { + if command -v nproc >/dev/null 2>&1; then nproc + elif command -v sysctl >/dev/null 2>&1 && sysctl -n hw.ncpu >/dev/null 2>&1; then sysctl -n hw.ncpu + else echo 4 + fi +} +JOBS=${BENCH_STAND_JOBS:-$(cpu_count)} +# The same module set the sql/xml/audio/terminal benchmarks require; dasLLVM for the jit lane. +# Release: RelWithDebInfo arms the C++ allocation tracker whose exit-time leak report turns +# every clean run into exit 1. +CMAKE_ARGS=(-G Ninja -DCMAKE_BUILD_TYPE=Release -DDAS_SQLITE_DISABLED=OFF -DDAS_PUGIXML_DISABLED=OFF + -DDAS_LLVM_DISABLED=OFF -DDAS_GLFW_DISABLED=ON -DDAS_HV_DISABLED=ON) +BUILD_TARGETS=(daslang dasModuleSQLITE dasModulePUGIXML dasModuleAudio dasModuleMinfft dasModuleTerminal dasModuleUnitTest dasModuleLLVM) +# The AOT lane measures native code, so its binary is built, not flagged: test_aot carries the +# benchmark bodies' stubs (the `benchmarks` row of DAS_AOT_SUITES). It is EXCLUDE_FROM_ALL and +# ~1080 TUs, so it is its own target and its own opt-out - a stand running only interp and jit +# should not pay for it. +AOT_TARGET=test_aot + +now_iso() { date -u +%Y-%m-%dT%H:%M:%SZ; } +log() { echo "[$(now_iso)] $*"; } + +# JSON string escaper for meta.json. Multi-byte UTF-8 passes through untouched - it is legal +# inside a JSON string, and stripping it would mangle a commit subject or an author name written +# in any non-ASCII script. +json_str() { + printf '%s' "$1" | tr -d '\r' \ + | LC_ALL=C awk 'BEGIN{ORS=""} {gsub(/\\/,"\\\\"); gsub(/"/,"\\\""); gsub(/\t/,"\\t"); if (NR>1) printf "\\n"; printf "%s", $0}' \ + | tr '\000-\010\013\014\016-\037' ' ' +} + +write_status() { + local state=$1 run_id=$2 started=$3 sha=$4 exit_code=${5:-null} tmp=$SITE/status.json.tmp + printf '{"state":"%s","run_id":"%s","started":"%s","finished":"%s","sha":"%s","exit":%s}\n' \ + "$state" "$run_id" "$started" "$(now_iso)" "$sha" "$exit_code" > "$tmp" + mv "$tmp" "$SITE/status.json" +} + +machine_json() { + local cpu cores mem kernel compiler load + cpu=$(lscpu 2>/dev/null | sed -n 's/^Model name:[[:space:]]*//p' | head -1) + [ -n "$cpu" ] || cpu=$(sysctl -n machdep.cpu.brand_string 2>/dev/null || true) + cores=$(cpu_count) + mem=$(free -g 2>/dev/null | awk '/^Mem:/{print $2}') + [ -n "$mem" ] || mem=$(( $(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824 )) + kernel=$(uname -r) + compiler=$( (c++ --version 2>/dev/null || cc --version 2>/dev/null) | head -1) + load=$(cut -d' ' -f1-3 /proc/loadavg 2>/dev/null || echo "") + printf '{"host":"%s","cpu":"%s","kernel":"%s","compiler":"%s","cores":%s,"mem_gb":%s,"load_start":"%s"}' \ + "$(json_str "$(hostname)")" "$(json_str "$cpu")" "$(json_str "$kernel")" "$(json_str "$compiler")" "${cores:-0}" "${mem:-0}" "$(json_str "$load")" +} + +commit_json() { + local sha date subject author + sha=$(git -C "$SRC" rev-parse HEAD) + date=$(TZ=UTC git -C "$SRC" log -1 --format=%cd --date=iso-strict-local | sed 's/+00:00$/Z/') + subject=$(git -C "$SRC" log -1 --format=%s) + author=$(git -C "$SRC" log -1 --format=%an) + printf '{"sha":"%s","date":"%s","subject":"%s","author":"%s"}' "$sha" "$date" "$(json_str "$subject")" "$(json_str "$author")" +} + +resolve_sha() { + git -C "$SRC" fetch --quiet origin "+refs/heads/*:refs/remotes/origin/*" 2>/dev/null || git -C "$SRC" fetch --quiet origin + git -C "$SRC" rev-parse --verify --quiet "$1^{commit}" \ + || git -C "$SRC" rev-parse --verify --quiet "origin/$1^{commit}" \ + || { echo "cannot resolve '$1' to a commit" >&2; return 1; } +} + +publish_viewer() { + local from=$1 + cp "$from/site/index.html" "$from/site/app.js" "$from/site/style.css" "$SITE/" + ln -sfn ../runs "$SITE/runs" +} + +report_with() { + local root=$1 bin=$2 rc=0 + local extra=() + [ -n "${BENCH_STAND_FAIL_ON_REGRESSION:-}" ] && extra+=(--fail-on-regression) + "$bin" "$root/$TOOL/main.das" -- report --suite "$root/$TOOL/suite.json" --runs "$RUNS" \ + --out-data "$SITE/data.json.tmp" --out-summary "$SITE/summary.md.tmp" \ + --repo-url "$REPO_URL" --site-url "$SITE_URL" "${extra[@]}" || rc=$? + if [ -s "$SITE/data.json.tmp" ] && [ -s "$SITE/summary.md.tmp" ]; then + mv "$SITE/data.json.tmp" "$SITE/data.json" + mv "$SITE/summary.md.tmp" "$SITE/summary.md" + REPORT_RENDERED=1 + fi + rm -f "$SITE/data.json.tmp" "$SITE/summary.md.tmp" + return $rc +} + +run_night() { + local run_id=$1 sha=$2 started rc=0 build_rc=0 build_start build_end + started=$(now_iso) + mkdir -p "$RUNS" "$SITE" "$LOGS" "$STATE" "$LAST_GOOD" + exec 9>"$STATE/lock" + if ! flock -n 9; then + log "another run holds $STATE/lock - refusing to start" + return 3 + fi + echo "$run_id" > "$STATE/current" + write_status running "$run_id" "$started" "$sha" + # a run that dies on an unexpected error still closes status.json, so `follow` returns + trap 'write_status finished "$run_id" "$started" "$sha" 1; log "run $run_id: aborted"' ERR + log "run $run_id: checkout $sha" + git -C "$SRC" checkout --quiet --detach "$sha" + git -C "$SRC" submodule update --quiet --init --recursive || log "submodule update failed - continuing with what is checked out" + # build/, lib/ (the LLVM download + shared modules) and the module cache survive so the rebuild stays incremental + git -C "$SRC" clean -fdxq --exclude=build --exclude=bin --exclude=lib --exclude=.jitted_scripts + + local meta=$STATE/meta.json build_log=$LOGS/build-$run_id.log + build_start=$(date +%s) + if [ "${BENCH_STAND_BUILD:-}" = "skip" ]; then + log "BENCH_STAND_BUILD=skip - reusing $SRC/bin/daslang" + [ -x "$SRC/bin/daslang" ] || build_rc=1 + : > "$build_log" + else + log "build: cmake ${CMAKE_ARGS[*]} ${BENCH_STAND_CMAKE_ARGS:-} -> ${BUILD_TARGETS[*]} (-j $JOBS)" + local launcher=() + command -v ccache >/dev/null && launcher=(-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache) + local targets=("${BUILD_TARGETS[@]}") + if [ "${BENCH_STAND_AOT:-}" = skip ]; then + log "BENCH_STAND_AOT=skip - the aot lane will report itself unavailable" + else + targets+=("$AOT_TARGET") + fi + # shellcheck disable=SC2086 - BENCH_STAND_CMAKE_ARGS must word-split into separate arguments + if ! ( cd "$SRC" && cmake --no-warn-unused-cli -B build "${CMAKE_ARGS[@]}" "${launcher[@]}" ${BENCH_STAND_CMAKE_ARGS:-} \ + && cmake --build build --parallel "$JOBS" --target "${targets[@]}" ) > "$build_log" 2>&1; then + build_rc=1 + fi + fi + build_end=$(date +%s) + local build_status=ok + [ $build_rc -eq 0 ] || build_status=failed + log "build: $build_status in $((build_end - build_start)) s (log $build_log)" + printf '{"run_id":"%s","started":"%s","commit":%s,"machine":%s,"build":{"status":"%s","seconds":%s,"log_tail":"%s"}}\n' \ + "$run_id" "$started" "$(commit_json)" "$(machine_json)" "$build_status" "$((build_end - build_start))" \ + "$(json_str "$(tail -n 40 "$build_log" | cut -c1-400)")" > "$meta" + + local record=$RUNS/$run_id.json + if [ $build_rc -eq 0 ]; then + local run_args=() + [ -n "${BENCH_STAND_FILTER:-}" ] && run_args+=(--filter "$BENCH_STAND_FILTER") + [ -n "${BENCH_STAND_LANES:-}" ] && run_args+=(--lanes "$BENCH_STAND_LANES") + [ -n "${BENCH_STAND_REPEAT:-}" ] && run_args+=(--repeat "$BENCH_STAND_REPEAT") + log "benchmarks: start" + local tool_rc=0 + ( cd "$SRC" && bin/daslang "$TOOL/main.das" -- run --root . --meta "$meta" --out "$record" "${run_args[@]}" ) || tool_rc=$? + log "benchmarks: exit $tool_rc" + rc=$tool_rc + if [ ! -s "$record" ]; then + log "the tool wrote no run record - writing a failed one" + rc=1 + record_failed "$record" "$meta" "the bench-stand tool exited $tool_rc without writing a record; see logs/nightly-$run_id.log on the box" + fi + else + rc=1 + record_failed "$record" "$meta" "the build failed after $((build_end - build_start)) s - logs/build-$run_id.log on the box has the whole log" + fi + + local report_root=$SRC report_bin=$SRC/bin/daslang report_rc=0 + if [ $build_rc -ne 0 ] || [ ! -x "$report_bin" ]; then + if [ -x "$LAST_GOOD/bin/daslang" ]; then + report_root=$LAST_GOOD; report_bin=$LAST_GOOD/bin/daslang + log "report: rendering with last_good's binary" + else + log "report: no binary at all (first night, build failed) - status.json is the only publication" + report_bin="" + fi + fi + if [ -n "$report_bin" ]; then + REPORT_RENDERED=0 + report_with "$report_root" "$report_bin" || report_rc=$? + if [ "$REPORT_RENDERED" = 0 ] && [ "$report_root" != "$LAST_GOOD" ] && [ -x "$LAST_GOOD/bin/daslang" ]; then + # tonight's tree built but its copy of this tool cannot render: the night is still + # published, by the last good tool, so the site shows the red night rather than yesterday's + log "report: tonight's tool failed to render (exit $report_rc) - retrying with last_good" + report_root=$LAST_GOOD; report_bin=$LAST_GOOD/bin/daslang; report_rc=0 + report_with "$report_root" "$report_bin" || report_rc=$? + fi + publish_viewer "$report_root/$TOOL" + log "report: exit $report_rc (rendered=$REPORT_RENDERED)" + fi + # last_good advances only when tonight's own tree rendered the report: a tree whose build + # passed but whose copy of this tool is broken must not become the fallback + if [ $build_rc -eq 0 ] && [ "${REPORT_RENDERED:-0}" = 1 ] && [ "$report_root" = "$SRC" ]; then + mkdir -p "$LAST_GOOD/bin" + cp "$SRC/bin/daslang" "$LAST_GOOD/bin/daslang" + rsync -a --delete "$SRC/daslib/" "$LAST_GOOD/daslib/" + mkdir -p "$LAST_GOOD/$TOOL" + rsync -a --delete "$SRC/$TOOL/" "$LAST_GOOD/$TOOL/" + fi + local final=$rc + [ $final -eq 0 ] && final=$report_rc + trap - ERR + write_status finished "$run_id" "$started" "$sha" "$final" + log "run $run_id: finished, exit $final" + return $final +} + +record_failed() { + # The record of a night that never benchmarked, written by the tool so the run-record schema + # has exactly one writer. `reason` must be non-empty - an empty one would read to the tool as + # "no --failed given" and start a real run. Neither binary available (a first night whose + # build failed) leaves status.json as the night's only publication. + local record=$1 meta=$2 reason=$3 root out + for root in "$SRC" "$LAST_GOOD"; do + [ -x "$root/bin/daslang" ] || continue + out=$("$root/bin/daslang" "$root/$TOOL/main.das" -- run --suite "$root/$TOOL/suite.json" \ + --meta "$meta" --out "$record" --failed "$reason" 2>&1) || true + if [ -s "$record" ]; then + log "failed record written by $root/bin/daslang" + return 0 + fi + # the one path whose failure is otherwise invisible: it runs when everything else already + # went wrong, so its own output is the only thing left to read + log "record_failed: $root/bin/daslang wrote nothing - $(printf '%s' "$out" | tail -n 3 | tr '\n' ' ')" + done + log "no binary could write the failed record - status.json is the night's only publication" + return 1 +} + +verb_start() { + # start [fail-on-regression]. The second word is how the caller asks for a regression + # to fail the night: an ssh forced command carries no environment, so BENCH_STAND_* cannot + # reach here from the workflow - the verb's own argument is the only channel. + local ref=${1:-master} regression=${2:-} sha run_id + mkdir -p "$LOGS" "$STATE" "$SITE" + case "$regression" in + ""|fail-on-regression) ;; + *) echo "start: second argument is 'fail-on-regression' or nothing, got '$regression'" >&2; return 2 ;; + esac + sha=$(resolve_sha "$ref") + run_id="$(date -u +%Y%m%dT%H%M%SZ)-${sha:0:8}" + if [ -e "$STATE/lock" ] && ! flock -n "$STATE/lock" true; then + echo "busy: $(cat "$STATE/current" 2>/dev/null) is still running" >&2 + return 3 + fi + echo "$run_id" > "$STATE/current" + local fail_flag=${BENCH_STAND_FAIL_ON_REGRESSION:-} + [ "$regression" = fail-on-regression ] && fail_flag=1 + BENCH_STAND_FAIL_ON_REGRESSION=$fail_flag \ + setsid nohup bash "$0" run-id "$run_id" "$sha" > "$LOGS/nightly-$run_id.log" 2>&1 < /dev/null & + echo "$run_id" +} + +verb_follow() { + local run_id=${1:-$(cat "$STATE/current" 2>/dev/null || true)} + [ -n "$run_id" ] || { echo "no current run" >&2; return 1; } + local logf=$LOGS/nightly-$run_id.log + local waited=0 + while [ ! -f "$logf" ] && [ $waited -lt 60 ]; do sleep 1; waited=$((waited + 1)); done + if [ ! -f "$logf" ]; then + # a run driven by `run` (no detached log) still answers through status.json + if grep -q "\"run_id\":\"$run_id\"" "$SITE/status.json" 2>/dev/null && grep -q '"state":"finished"' "$SITE/status.json"; then + cat "$SITE/status.json" + return "$(sed -n 's/.*"exit":\([0-9]*\).*/\1/p' "$SITE/status.json")" + fi + echo "no log for $run_id" >&2; return 1 + fi + local pos=0 + while :; do + local size + size=$(stat -c %s "$logf") + if [ "$size" -gt "$pos" ]; then + tail -c +"$((pos + 1))" "$logf" | head -c "$((size - pos))" + pos=$size + fi + if grep -q "\"run_id\":\"$run_id\"" "$SITE/status.json" 2>/dev/null && grep -q '"state":"finished"' "$SITE/status.json"; then + local exit_code + exit_code=$(sed -n 's/.*"exit":\([0-9]*\).*/\1/p' "$SITE/status.json") + echo "--- run $run_id finished with exit ${exit_code:-?} ---" + return "${exit_code:-1}" + fi + sleep 20 + done +} + +verb_gate() { + # SSH forced command: only these verbs, at most two arguments, only path characters in each. + local cmd=${SSH_ORIGINAL_COMMAND:-status} + # shellcheck disable=SC2086 - the forced command's one string is split into words on purpose + set -- $cmd + if [ $# -gt 3 ]; then + echo "refused: too many arguments" >&2; exit 2 + fi + local verb=${1:-status} arg=${2:-} arg2=${3:-} + case "$arg$arg2" in + *[!A-Za-z0-9._/-]*) echo "refused: argument '$arg$arg2' has characters outside A-Za-z0-9._/-" >&2; exit 2 ;; + esac + case "$verb" in + start) verb_start "$arg" "$arg2" ;; + follow) verb_follow "$arg" ;; + status) cat "$SITE/status.json" 2>/dev/null || echo '{"state":"never-run"}' ;; + summary) cat "$SITE/summary.md" 2>/dev/null || echo "no summary yet" ;; + *) echo "refused: unknown verb '$verb'" >&2; exit 2 ;; + esac +} + +case "${1:-}" in + start) verb_start "${2:-master}" "${3:-}" ;; + run) sha=$(resolve_sha "${2:-master}"); run_night "$(date -u +%Y%m%dT%H%M%SZ)-${sha:0:8}" "$sha" ;; + run-id) run_night "$2" "$3" ;; + follow) verb_follow "${2:-}" ;; + status) cat "$SITE/status.json" 2>/dev/null || echo '{"state":"never-run"}' ;; + summary) cat "$SITE/summary.md" 2>/dev/null || echo "no summary yet" ;; + gate) verb_gate ;; + *) echo "usage: nightly.sh start [fail-on-regression] | run | follow [run_id] | status | summary | gate" >&2; exit 2 ;; +esac diff --git a/utils/internal/bench-stand/site/app.js b/utils/internal/bench-stand/site/app.js new file mode 100644 index 0000000000..000f6519a8 --- /dev/null +++ b/utils/internal/bench-stand/site/app.js @@ -0,0 +1,581 @@ +// daslang benchmark stand viewer. Reads data.json (built by bench-stand `report`) and +// status.json (written by nightly.sh) from the same directory and renders everything +// client-side: the latest night, the group index, one chart per benchmark arm, run history. +"use strict"; + +const LANES = ["interp", "jit"]; +const state = { + data: null, + status: null, + runWindow: 365, + lanes: new Set(LANES), + group: "", + search: "", + onlyChanged: false, +}; + +const $ = (sel, root) => (root || document).querySelector(sel); +const el = (tag, cls, text) => { + const e = document.createElement(tag); + if (cls) e.className = cls; + if (text !== undefined) e.textContent = text; + return e; +}; + +function fmtNs(v) { + if (v >= 1e6) return (v / 1e6).toFixed(2) + " ms"; + if (v >= 1e3) return (v / 1e3).toFixed(2) + " us"; + if (v >= 100) return v.toFixed(0) + " ns"; + return v.toFixed(v >= 10 ? 1 : 2) + " ns"; +} +const fmtPct = (x) => (x >= 0 ? "+" : "") + (x * 100).toFixed(1) + "%"; +const fmtSec = (s) => (s >= 3600 ? (s / 3600).toFixed(1) + " h" : s >= 60 ? Math.round(s / 60) + " min" : Math.round(s) + " s"); +const shortSha = (sha) => (sha || "").slice(0, 8); +const dateOf = (iso) => (iso || "").slice(0, 10); +function commitUrl(sha) { + const base = state.data && state.data.repo_url; + return base && sha ? base.replace(/\/$/, "") + "/commit/" + sha : null; +} + +// ---- data loading ---------------------------------------------------------------------------- + +async function load() { + const [dataRes, statusRes] = await Promise.allSettled([ + fetch("data.json", { cache: "no-store" }).then((r) => (r.ok ? r.json() : Promise.reject(new Error("data.json " + r.status)))), + fetch("status.json", { cache: "no-store" }).then((r) => (r.ok ? r.json() : null)), + ]); + state.status = statusRes.status === "fulfilled" ? statusRes.value : null; + if (dataRes.status === "fulfilled") { + state.data = dataRes.value; + } else { + const main = $("main"); + main.replaceChildren(el("p", "notice notice--error", "No data.json yet: the stand has not published a report (" + dataRes.reason.message + ").")); + } + renderStatus(); + if (state.data) renderAll(); +} + +// ---- visible runs (the range filter) --------------------------------------------------------- + +function visibleRunIndices() { + const runs = state.data.runs; + const n = runs.length; + const from = state.runWindow > 0 ? Math.max(0, n - state.runWindow) : 0; + const out = []; + for (let i = from; i < n; i++) out.push(i); + return out; +} + +// ---- status header --------------------------------------------------------------------------- + +function renderStatus() { + const box = $("#status"); + box.replaceChildren(); + const st = state.status; + if (st && st.state === "running") { + const pill = el("span", "pill pill--running", "running " + (st.run_id || "")); + box.append(pill, el("span", "muted", "started " + (st.started || ""))); + } + if (!state.data || state.data.latest < 0) { + if (!st) box.append(el("span", "pill", "no runs")); + return; + } + const run = state.data.runs[state.data.latest]; + box.append(el("span", "pill pill--" + run.status, run.status.replace("_", " "))); + const link = el("a", null, shortSha(run.sha) + " " + run.subject); + link.href = commitUrl(run.sha) || "#"; + link.title = run.date; + box.append(link); + box.append(el("span", "muted", dateOf(run.started) + " on " + run.host + ", build " + fmtSec(run.build_seconds) + ", suite " + fmtSec(run.seconds))); + const ageDays = (Date.now() - Date.parse(run.started)) / 86400000; + if (ageDays > 2 && !(st && st.state === "running")) { + box.append(el("span", "pill pill--stale", "last run " + Math.floor(ageDays) + " days ago")); + } + $("#generated").textContent = "report generated " + (state.data.generated || ""); +} + +// ---- filters --------------------------------------------------------------------------------- + +function wireFilters() { + document.querySelectorAll(".chip[data-range]").forEach((b) => { + b.addEventListener("click", () => { + document.querySelectorAll(".chip[data-range]").forEach((x) => x.classList.remove("is-on")); + b.classList.add("is-on"); + state.runWindow = Number(b.dataset.range); + renderAll(); + }); + }); + document.querySelectorAll("input[data-lane]").forEach((c) => { + c.addEventListener("change", () => { + if (c.checked) state.lanes.add(c.dataset.lane); else state.lanes.delete(c.dataset.lane); + renderAll(); + }); + }); + $("#group").addEventListener("change", (e) => { state.group = e.target.value; renderSeries(); }); + let timer = 0; + $("#search").addEventListener("input", (e) => { + clearTimeout(timer); + timer = setTimeout(() => { state.search = e.target.value.trim().toLowerCase(); renderSeries(); }, 120); + }); + $("#only-changed").addEventListener("change", (e) => { state.onlyChanged = e.target.checked; renderSeries(); }); +} + +function fillGroups() { + const sel = $("#group"); + const keep = sel.value; + sel.replaceChildren(el("option", null, "all groups")); + sel.firstChild.value = ""; + for (const g of state.data.groups) { + const o = el("option", null, g); + o.value = g; + sel.append(o); + } + sel.value = state.data.groups.includes(keep) ? keep : ""; +} + +// ---- latest night ---------------------------------------------------------------------------- + +function renderNight() { + const box = $("#night"); + box.replaceChildren(); + const d = state.data; + if (d.latest < 0) return; + const run = d.runs[d.latest]; + + const fails = el("div", "card"); + if (run.build_status && run.build_status !== "ok") { + fails.append(header("The build failed", "", "nothing was measured")); + const why = el("div", "msg"); + why.append(el("div", null, "after " + fmtSec(run.build_seconds) + "; last lines of the build log:")); + const pre = el("pre", "logtail"); + pre.textContent = run.build_log_tail || "the driver recorded no build log"; + why.append(pre); + fails.append(why); + const rec = el("a", "msg", "run record"); + rec.href = "runs/" + encodeURIComponent(run.id) + ".json"; + fails.append(rec); + box.append(fails); + return; + } + fails.append(header("Failures", run.failures.length)); + fails.append(list(run.failures, (f) => { + const li = el("li"); + li.append(el("span", "tag tag--failure", f.status.replace("_", " ")), el("span", "tag tag--" + f.lane, f.lane), el("code", null, f.path)); + const m = el("div", "msg"); + m.append(el("code", null, f.message)); + li.append(m); + return li; + }, "every file ran")); + const lanes = el("div", "msg"); + for (const lane of Object.keys(run.lanes)) { + if (run.lanes[lane] !== "ok") lanes.append(el("div", null, lane + " lane did not run: " + run.lanes[lane])); + } + if (run.skipped.length) { + const s = el("div", "msg"); + s.append(el("div", null, run.skipped.length + " skipped:")); + for (const f of run.skipped) { + const line = el("div"); + line.append(el("code", null, f.path), " (" + f.lane + "): " + f.message); + s.append(line); + } + lanes.append(s); + } + fails.append(lanes); + const link = el("a", "msg", "run record"); + link.href = "runs/" + encodeURIComponent(run.id) + ".json"; + fails.append(link); + + const regs = d.changes.filter((c) => c.kind === "regression"); + const imps = d.changes.filter((c) => c.kind === "improvement"); + const newOrMissing = d.changes.filter((c) => c.kind === "first_seen" || c.kind === "missing"); + const changeItem = (c) => { + const li = el("li"); + const a = el("a", null, c.id); + a.href = "#" + anchorId(c.id); + li.append(el("span", "tag tag--" + c.lane, c.lane), a); + if (c.kind === "regression" || c.kind === "improvement") { + li.append(el("span", "delta " + (c.change > 0 ? "delta--up" : "delta--down"), fmtPct(c.change))); + li.append(el("span", "muted", fmtNs(c.baseline) + " -> " + fmtNs(c.value) + ", noise " + (c.noise * 100).toFixed(1) + "%")); + } else { + li.append(el("span", "tag", c.kind)); + } + return li; + }; + const regCard = el("div", "card"); + regCard.append(header("Regressions", regs.length, "slower than " + fmtPct(d.regression_threshold) + " and " + d.noise_multiplier + "x noise over the median of " + d.baseline_runs + " runs")); + regCard.append(list(regs, changeItem, "nothing slower")); + const impCard = el("div", "card"); + impCard.append(header("Improvements", imps.length)); + impCard.append(list(imps, changeItem, "nothing faster")); + if (newOrMissing.length) { + impCard.append(header("First seen or missing", newOrMissing.length)); + impCard.append(list(newOrMissing, changeItem, "")); + } + box.append(fails, regCard, impCard); +} + +function header(title, count, sub) { + const h = el("h3"); + h.append(title); + if (count !== "") h.append(el("span", "count", String(count))); + if (sub) h.append(el("span", "muted", sub)); + return h; +} +function list(items, render, emptyText) { + const ul = el("ul"); + if (!items.length) { + if (emptyText) ul.append(el("li", "empty", emptyText)); + return ul; + } + for (const it of items) ul.append(render(it)); + return ul; +} + +// ---- charts ---------------------------------------------------------------------------------- + +const W = 420, H = 150, PAD = { l: 44, r: 54, t: 10, b: 22 }; + +function niceTicks(lo, hi, count) { + if (!(hi > lo)) { hi = lo + 1; } + const span = hi - lo; + const step0 = span / Math.max(1, count); + const mag = Math.pow(10, Math.floor(Math.log10(step0))); + const step = [1, 2, 2.5, 5, 10].map((m) => m * mag).find((s) => s >= step0) || 10 * mag; + const start = Math.floor(lo / step) * step; + const out = []; + for (let v = start; v <= hi + step * 0.5; v += step) out.push(+v.toPrecision(12)); + return out; +} + +// lines: [{lane, points: [{r, v, s}]}], runIdx: visible run indices (x positions) +function drawChart(lines, runIdx, opts) { + const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg"); + svg.setAttribute("viewBox", `0 0 ${W} ${H}`); + svg.setAttribute("class", "chart"); + svg.setAttribute("role", "img"); + svg.setAttribute("aria-label", opts.label); + const xOf = new Map(); + const n = runIdx.length; + runIdx.forEach((ri, i) => xOf.set(ri, PAD.l + (n === 1 ? (W - PAD.l - PAD.r) / 2 : (i * (W - PAD.l - PAD.r)) / (n - 1)))); + let lo = Infinity, hi = -Infinity; + for (const ln of lines) for (const p of ln.points) { + if (!xOf.has(p.r)) continue; + const top = p.v * (1 + (p.s || 0)); + if (p.v < lo) lo = p.v; + if (top > hi) hi = top; + } + if (!isFinite(lo)) { lo = 0; hi = 1; } + if (opts.zeroBased) lo = Math.min(lo, 0); + const pad = (hi - lo) * 0.12 || hi * 0.1 || 1; + lo = Math.max(opts.zeroBased ? 0 : -Infinity, lo - pad); + hi = hi + pad; + const ticks = niceTicks(lo, hi, 4); + lo = Math.min(lo, ticks[0]); + hi = Math.max(hi, ticks[ticks.length - 1]); + const yOf = (v) => PAD.t + (H - PAD.t - PAD.b) * (1 - (v - lo) / (hi - lo)); + const svgEl = (tag, attrs) => { + const e = document.createElementNS("http://www.w3.org/2000/svg", tag); + for (const k in attrs) e.setAttribute(k, attrs[k]); + return e; + }; + for (const tv of ticks) { + if (tv < lo || tv > hi) continue; + const y = yOf(tv); + svg.append(svgEl("line", { class: "grid-line", x1: PAD.l, x2: W - PAD.r, y1: y, y2: y })); + const t = svgEl("text", { x: PAD.l - 6, y: y + 3.5, "text-anchor": "end" }); + t.textContent = opts.fmt(tv); + svg.append(t); + } + svg.append(svgEl("line", { class: "axis-line", x1: PAD.l, x2: W - PAD.r, y1: H - PAD.b, y2: H - PAD.b })); + const runs = state.data.runs; + const labelEvery = Math.max(1, Math.ceil(n / 5)); + runIdx.forEach((ri, i) => { + if (i % labelEvery !== 0 && i !== n - 1) return; + const t = svgEl("text", { x: xOf.get(ri), y: H - 6, "text-anchor": i === n - 1 ? "end" : i === 0 ? "start" : "middle" }); + t.textContent = dateOf(runs[ri].date).slice(5); + svg.append(t); + }); + if (opts.failedRuns) { + for (const ri of opts.failedRuns) { + if (!xOf.has(ri)) continue; + svg.append(svgEl("rect", { class: "fail-mark", x: xOf.get(ri) - 2, y: H - PAD.b - 5, width: 4, height: 5 })); + } + } + for (const ln of lines) { + const pts = ln.points.filter((p) => xOf.has(p.r)); + if (!pts.length) continue; + if (opts.band) { + let d = ""; + for (const p of pts) d += (d ? "L" : "M") + xOf.get(p.r).toFixed(1) + "," + yOf(p.v * (1 + p.s)).toFixed(1); + for (let i = pts.length - 1; i >= 0; i--) d += "L" + xOf.get(pts[i].r).toFixed(1) + "," + yOf(pts[i].v).toFixed(1); + svg.append(svgEl("path", { class: "band band--" + ln.lane, d: d + "Z" })); + } + let d = ""; + let prev = -2; + for (const p of pts) { + const gap = runIdx.indexOf(p.r) - prev > 1; + d += (gap ? "M" : "L") + xOf.get(p.r).toFixed(1) + "," + yOf(p.v).toFixed(1); + prev = runIdx.indexOf(p.r); + } + svg.append(svgEl("path", { class: "series series--" + ln.lane, d })); + if (pts.length === 1) svg.append(svgEl("circle", { class: "marker marker--" + ln.lane, cx: xOf.get(pts[0].r), cy: yOf(pts[0].v), r: 4 })); + const last = pts[pts.length - 1]; + const lbl = svgEl("text", { class: "end-label", x: W - PAD.r + 6, y: yOf(last.v) + 3.5 }); + lbl.textContent = opts.fmt(last.v); + svg.append(lbl); + } + attachHover(svg, svgEl, lines, runIdx, opts, xOf, yOf); + return svg; +} + +// The hover layer: a crosshair that snaps to the nearest run, one tooltip listing every lane at +// that run, and a click that opens the commit. The hit rectangle spans the plot and overhangs it, +// so a reader aims at a date rather than at a 2px line. +function attachHover(svg, svgEl, lines, runIdx, opts, xOf, yOf) { + const runs = state.data.runs; + const cross = svgEl("line", { class: "crosshair", x1: 0, x2: 0, y1: PAD.t, y2: H - PAD.b, visibility: "hidden" }); + svg.append(cross); + const markers = lines.map((ln) => { + const m = svgEl("circle", { class: "marker marker--" + ln.lane, r: 4, visibility: "hidden" }); + svg.append(m); + return m; + }); + const hit = svgEl("rect", { class: "hit", x: PAD.l - 10, y: 0, width: W - PAD.l - PAD.r + 20, height: H }); + svg.append(hit); + const tooltip = $("#tooltip"); + const nearestRun = (evt) => { + const rect = svg.getBoundingClientRect(); + const px = ((evt.clientX - rect.left) / rect.width) * W; + let best = -1, bestD = Infinity; + for (const ri of runIdx) { + const dx = Math.abs(xOf.get(ri) - px); + if (dx < bestD) { bestD = dx; best = ri; } + } + return best; + }; + const show = (evt) => { + const best = nearestRun(evt); + if (best < 0) return; + const x = xOf.get(best); + cross.setAttribute("x1", x); cross.setAttribute("x2", x); cross.setAttribute("visibility", "visible"); + tooltip.replaceChildren(); + const run = runs[best]; + tooltip.append(el("div", "tt-head", dateOf(run.date) + " " + shortSha(run.sha) + " " + run.subject)); + lines.forEach((ln, i) => { + const p = ln.points.find((q) => q.r === best); + if (!p) { markers[i].setAttribute("visibility", "hidden"); return; } + markers[i].setAttribute("cx", x); markers[i].setAttribute("cy", yOf(p.v)); markers[i].setAttribute("visibility", "visible"); + const row = el("div", "tt-row"); + row.append(el("span", "tt-key tt-key--" + ln.lane), el("span", "tt-val", opts.fmt(p.v)), el("span", null, ln.lane)); + if (p.s !== undefined && opts.band) row.append(el("span", "tt-sub", "spread " + (p.s * 100).toFixed(1) + "%")); + tooltip.append(row); + }); + if (opts.failedRuns && opts.failedRuns.has(best)) tooltip.append(el("div", "tt-sub", "this file failed that night")); + tooltip.hidden = false; + const tw = tooltip.offsetWidth, th = tooltip.offsetHeight; + let left = evt.clientX + 14, top = evt.clientY + 14; + if (left + tw > window.innerWidth - 8) left = evt.clientX - tw - 14; + if (top + th > window.innerHeight - 8) top = evt.clientY - th - 14; + tooltip.style.left = left + "px"; tooltip.style.top = top + "px"; + }; + const hide = () => { + cross.setAttribute("visibility", "hidden"); + markers.forEach((m) => m.setAttribute("visibility", "hidden")); + tooltip.hidden = true; + }; + hit.addEventListener("pointermove", show); + hit.addEventListener("pointerleave", hide); + hit.addEventListener("click", (evt) => { + const best = nearestRun(evt); + const url = best >= 0 ? commitUrl(runs[best].sha) : null; + if (url) window.open(url, "_blank", "noopener"); + }); +} + +function legend(lanes) { + const lg = el("div", "legend"); + for (const lane of lanes) lg.append(el("span", "legend--" + lane, lane)); + return lg; +} + +function tableView(lines, runIdx, fmt) { + const t = el("table", "mini"); + const thead = el("tr"); + thead.append(el("th", null, "run")); + for (const ln of lines) thead.append(el("th", "num", ln.lane)); + t.append(thead); + const shown = runIdx.slice(-12).reverse(); + for (const ri of shown) { + const tr = el("tr"); + const run = state.data.runs[ri]; + tr.append(el("td", null, dateOf(run.date) + " " + shortSha(run.sha))); + for (const ln of lines) { + const p = ln.points.find((q) => q.r === ri); + tr.append(el("td", "num", p ? fmt(p.v) : "-")); + } + t.append(tr); + } + return t; +} + +function chartCard(title, fileLabel, lines, runIdx, opts) { + const card = el("div", "card chart-card"); + if (opts.anchor) card.id = opts.anchor; + const head = el("div", "chart-card__head"); + const ttl = el("div", "chart-card__title"); + if (fileLabel) ttl.append(el("span", "file", fileLabel + " ")); + ttl.append(title); + head.append(ttl); + const tools = el("div", "chart-card__tools"); + if (opts.badges) for (const b of opts.badges) tools.append(b); + const toggle = el("button", null, "table"); + tools.append(toggle); + head.append(tools); + card.append(head); + const svg = drawChart(lines, runIdx, opts); + card.append(svg); + if (lines.length > 1) card.append(legend(lines.map((l) => l.lane))); + let table = null; + toggle.addEventListener("click", () => { + if (table) { table.remove(); table = null; toggle.textContent = "table"; return; } + table = tableView(lines, runIdx, opts.fmt); + card.append(table); + toggle.textContent = "chart only"; + }); + return card; +} + +const anchorId = (id) => "b-" + id.replace(/[^A-Za-z0-9_-]/g, "_"); + +function renderIndex() { + const box = $("#index"); + box.replaceChildren(); + const runIdx = visibleRunIndices(); + const byGroup = new Map(); + for (const ix of state.data.index) { + if (!state.lanes.has(ix.lane)) continue; + if (!byGroup.has(ix.group)) byGroup.set(ix.group, []); + byGroup.get(ix.group).push({ lane: ix.lane, points: ix.runs.map((r, i) => ({ r, v: ix.value[i], s: 0 })) }); + } + if (!byGroup.size) { box.append(el("p", "notice", "The index needs at least three runs per series.")); return; } + for (const [group, lines] of [...byGroup.entries()].sort((a, b) => a[0].localeCompare(b[0]))) { + lines.sort((a, b) => LANES.indexOf(a.lane) - LANES.indexOf(b.lane)); + box.append(chartCard(group, "", lines, runIdx, { label: "index of " + group, fmt: (v) => v.toFixed(0), band: false, zeroBased: false })); + } +} + +function failedRunsByFile() { + const out = new Map(); + state.data.runs.forEach((run, ri) => { + for (const f of run.failures) { + const key = f.lane + "\t" + f.path.replace(/\.das$/, ""); + if (!out.has(key)) out.set(key, new Set()); + out.get(key).add(ri); + } + }); + return out; +} + +function renderSeries() { + const box = $("#series"); + box.replaceChildren(); + const d = state.data; + const runIdx = visibleRunIndices(); + const visible = new Set(runIdx); + const changeOf = new Map(); + for (const c of d.changes) changeOf.set(c.lane + "\t" + c.id, c); + const failed = failedRunsByFile(); + const arms = new Map(); + for (const s of d.series) { + if (!state.lanes.has(s.lane)) continue; + if (state.group && s.group !== state.group) continue; + if (state.search && !s.id.toLowerCase().includes(state.search)) continue; + const ch = changeOf.get(s.lane + "\t" + s.id); + if (state.onlyChanged && !(ch && (ch.kind === "regression" || ch.kind === "improvement"))) continue; + if (!s.runs.some((r) => visible.has(r))) continue; + if (!arms.has(s.id)) arms.set(s.id, { group: s.group, file: s.file, lines: [], changes: [] }); + const arm = arms.get(s.id); + arm.lines.push({ lane: s.lane, points: s.runs.map((r, i) => ({ r, v: s.ns[i], s: s.spread[i] })) }); + if (ch) arm.changes.push(ch); + } + $("#series-count").textContent = arms.size + " of " + new Set(d.series.map((s) => s.id)).size + " arms"; + if (!arms.size) { box.append(el("p", "notice", "Nothing matches the filters.")); return; } + const byGroup = new Map(); + for (const [id, arm] of arms) { + if (!byGroup.has(arm.group)) byGroup.set(arm.group, []); + byGroup.get(arm.group).push([id, arm]); + } + for (const [group, entries] of [...byGroup.entries()].sort((a, b) => a[0].localeCompare(b[0]))) { + const section = el("div", "group"); + const h = el("h3"); + h.append(group, el("span", "muted", entries.length + " arms")); + section.append(h); + const grid = el("div", "grid grid--series"); + entries.sort((a, b) => a[0].localeCompare(b[0])); + for (const [id, arm] of entries) { + arm.lines.sort((a, b) => LANES.indexOf(a.lane) - LANES.indexOf(b.lane)); + const badges = arm.changes.filter((c) => c.kind === "regression" || c.kind === "improvement") + .map((c) => el("span", "tag tag--" + c.kind, c.lane + " " + fmtPct(c.change))); + const failedRuns = new Set(); + for (const ln of arm.lines) for (const ri of failed.get(ln.lane + "\t" + arm.file) || []) failedRuns.add(ri); + const title = id.slice(arm.file.length + 1); + grid.append(chartCard(title, arm.file, arm.lines, runIdx, { + label: id, fmt: fmtNs, band: true, zeroBased: true, anchor: anchorId(id), badges, failedRuns, + })); + } + section.append(grid); + box.append(section); + } +} + +function renderRuns() { + const table = $("#runs"); + table.replaceChildren(); + const head = el("tr"); + for (const [txt, cls] of [["date", ""], ["commit", ""], ["status", ""], ["lanes", ""], ["build", "num"], ["suite", "num"], ["ok", "num"], ["failed", "num"], ["regressions", "num"], ["improvements", "num"], ["record", ""]]) { + head.append(el("th", cls, txt)); + } + table.append(head); + const runIdx = visibleRunIndices().slice().reverse(); + for (const ri of runIdx) { + const run = state.data.runs[ri]; + const tr = el("tr"); + tr.append(el("td", null, dateOf(run.started))); + const c = el("td"); + const a = el("a", null, shortSha(run.sha)); + a.href = commitUrl(run.sha) || "#"; + a.title = run.subject; + c.append(a, " ", el("span", "muted", run.subject.length > 60 ? run.subject.slice(0, 60) + "..." : run.subject)); + tr.append(c); + const s = el("td"); + s.append(el("span", "pill pill--" + run.status, run.status.replace("_", " "))); + tr.append(s); + const lanes = el("td"); + for (const lane of Object.keys(run.lanes)) { + const t = el("span", "tag tag--" + lane, lane); + if (run.lanes[lane] !== "ok") { t.textContent = lane + " off"; t.title = run.lanes[lane]; t.className = "tag"; } + lanes.append(t, " "); + } + tr.append(lanes); + tr.append(el("td", "num", fmtSec(run.build_seconds)), el("td", "num", fmtSec(run.seconds))); + tr.append(el("td", "num", String(run.files_ok)), el("td", "num", String(run.failures.length))); + tr.append(el("td", "num", ri === state.data.latest ? String(run.regressions) : ""), el("td", "num", ri === state.data.latest ? String(run.improvements) : "")); + const rec = el("td"); + const link = el("a", null, "json"); + link.href = "runs/" + encodeURIComponent(run.id) + ".json"; + rec.append(link); + tr.append(rec); + table.append(tr); + } +} + +function renderAll() { + fillGroups(); + renderNight(); + renderIndex(); + renderSeries(); + renderRuns(); +} + +wireFilters(); +load(); diff --git a/utils/internal/bench-stand/site/index.html b/utils/internal/bench-stand/site/index.html new file mode 100644 index 0000000000..119d7275ec --- /dev/null +++ b/utils/internal/bench-stand/site/index.html @@ -0,0 +1,66 @@ + + + + + +daslang benchmark stand + + + + +
+
+ daslang +

benchmark stand

+
+
+
+ +
+
+
+ + + + +
+
+ + +
+
+ + +
+
+ +
+
+ +
+ +
+

Group index geometric mean of each series against its own first runs, 100 = reference

+
+
+ +
+

Benchmarks

+
+
+ +
+

Run history

+
+
+
+ +
+ Every point is the minimum ns/op over the night's repeats of a dastest --bench run; the tooltip shows the spread. Source: utils/internal/bench-stand in the daslang repository. + +
+ + + + + diff --git a/utils/internal/bench-stand/site/style.css b/utils/internal/bench-stand/site/style.css new file mode 100644 index 0000000000..37424f3dab --- /dev/null +++ b/utils/internal/bench-stand/site/style.css @@ -0,0 +1,181 @@ +:root { + color-scheme: light; + --page: #f9f9f7; + --surface: #fcfcfb; + --ink: #0b0b0b; + --ink-2: #52514e; + --muted: #898781; + --grid: #e1e0d9; + --axis: #c3c2b7; + --border: rgba(11, 11, 11, 0.10); + --interp: #2a78d6; + --jit: #eb6834; + --good: #0ca30c; + --good-text: #006300; + --warning: #fab219; + --serious: #ec835a; + --critical: #d03b3b; + --chip-on: #e8eef9; +} +@media (prefers-color-scheme: dark) { + :root { + color-scheme: dark; + --page: #0d0d0d; + --surface: #1a1a19; + --ink: #ffffff; + --ink-2: #c3c2b7; + --muted: #898781; + --grid: #2c2c2a; + --axis: #383835; + --border: rgba(255, 255, 255, 0.10); + --interp: #3987e5; + --jit: #d95926; + --good-text: #0ca30c; + --chip-on: #22314a; + } +} + +* { box-sizing: border-box; } +html, body { margin: 0; } +body { + background: var(--page); + color: var(--ink); + font: 14px/1.45 system-ui, -apple-system, "Segoe UI", sans-serif; +} +a { color: inherit; } +code { font: 12px/1.4 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; } +h1, h2, h3 { font-weight: 600; margin: 0; } +h2 { font-size: 16px; margin: 0 0 12px; } +.muted { color: var(--muted); font-weight: 400; font-size: 13px; } + +.top { + display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: 12px; + padding: 18px 24px 8px; +} +.top__title { display: flex; align-items: baseline; gap: 10px; } +.top__brand { text-decoration: none; color: var(--ink-2); font-weight: 600; } +.top__title h1 { font-size: 20px; } +.top__status { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; color: var(--ink-2); } + +main { padding: 0 24px 32px; max-width: 1500px; } +.section { margin-top: 28px; } + +.filters { + display: flex; flex-wrap: wrap; gap: 18px; align-items: center; + padding: 10px 0 14px; border-bottom: 1px solid var(--grid); +} +.filters__group { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; } +.chip { + background: var(--surface); color: var(--ink-2); border: 1px solid var(--border); + border-radius: 999px; padding: 4px 10px; cursor: pointer; font: inherit; font-size: 13px; +} +.chip.is-on { background: var(--chip-on); color: var(--ink); border-color: transparent; font-weight: 600; } +.lane { display: inline-flex; align-items: center; gap: 6px; cursor: pointer; padding: 4px 8px; } +.lane input { accent-color: var(--ink-2); } +.lane--interp input { accent-color: var(--interp); } +.lane--jit input { accent-color: var(--jit); } +.select select, .search { + font: inherit; font-size: 13px; color: var(--ink); background: var(--surface); + border: 1px solid var(--border); border-radius: 6px; padding: 4px 8px; +} +.search { width: 260px; } + +.pill { + display: inline-flex; align-items: center; gap: 6px; padding: 2px 10px; border-radius: 999px; + font-size: 13px; font-weight: 600; background: var(--surface); border: 1px solid var(--border); +} +.pill::before { content: ""; width: 8px; height: 8px; border-radius: 50%; background: var(--muted); } +.pill--ok::before { background: var(--good); } +.pill--bench_failed::before, .pill--build_failed::before { background: var(--critical); } +.pill--running::before { background: var(--warning); } +.pill--stale::before { background: var(--serious); } + +.night { margin-top: 18px; display: grid; gap: 14px; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); } +.card { + background: var(--surface); border: 1px solid var(--border); border-radius: 10px; padding: 12px 14px; +} +.card h3 { font-size: 14px; margin-bottom: 8px; display: flex; gap: 8px; align-items: baseline; } +.card h3 .count { color: var(--muted); font-weight: 400; } +.card ul { margin: 0; padding: 0; list-style: none; } +.card li { padding: 4px 0; border-top: 1px solid var(--grid); display: flex; gap: 8px; flex-wrap: wrap; align-items: baseline; } +.card li:first-child { border-top: 0; } +.card .empty { color: var(--muted); } +.tag { font-size: 12px; padding: 0 6px; border-radius: 4px; border: 1px solid var(--border); color: var(--ink-2); } +.tag--regression { color: var(--critical); border-color: var(--critical); } +.tag--improvement { color: var(--good-text); border-color: var(--good-text); } +.tag--failure { color: var(--critical); border-color: var(--critical); } +.tag--interp { color: var(--interp); border-color: var(--interp); } +.tag--jit { color: var(--jit); border-color: var(--jit); } +.delta { font-variant-numeric: tabular-nums; font-weight: 600; } +.delta--up { color: var(--critical); } +.delta--down { color: var(--good-text); } +.msg { color: var(--ink-2); font-size: 13px; } +.msg code { white-space: pre-wrap; word-break: break-word; } +pre.logtail { + margin: 6px 0 0; padding: 8px 10px; max-height: 220px; overflow: auto; + background: var(--page); border: 1px solid var(--grid); border-radius: 6px; + font: 12px/1.45 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; color: var(--ink-2); + white-space: pre-wrap; word-break: break-word; +} + +.grid { display: grid; gap: 14px; } +.grid--index { grid-template-columns: repeat(auto-fill, minmax(320px, 1fr)); } +.group { margin-top: 18px; } +.group > h3 { font-size: 15px; margin: 0 0 8px; display: flex; gap: 10px; align-items: baseline; } +.group > h3 .muted { font-size: 13px; } +.grid--series { grid-template-columns: repeat(auto-fill, minmax(360px, 1fr)); } + +.chart-card { padding: 10px 12px 6px; } +.chart-card__head { display: flex; justify-content: space-between; gap: 8px; align-items: baseline; } +.chart-card__title { font-size: 13px; font-weight: 600; word-break: break-all; } +.chart-card__title .file { color: var(--muted); font-weight: 400; } +.chart-card__tools { display: flex; gap: 6px; align-items: center; flex-shrink: 0; } +.chart-card__tools button { + background: none; border: 1px solid var(--border); color: var(--ink-2); border-radius: 6px; + font: inherit; font-size: 12px; padding: 1px 7px; cursor: pointer; +} +.legend { display: flex; gap: 12px; font-size: 12px; color: var(--ink-2); margin: 4px 0 0; } +.legend span::before { content: ""; display: inline-block; width: 14px; height: 2px; vertical-align: middle; margin-right: 5px; border-radius: 1px; background: var(--muted); } +.legend .legend--interp::before { background: var(--interp); } +.legend .legend--jit::before { background: var(--jit); } +svg.chart { display: block; width: 100%; height: auto; overflow: visible; touch-action: pan-y; } +svg.chart text { fill: var(--muted); font-size: 11px; font-variant-numeric: tabular-nums; } +svg.chart .grid-line { stroke: var(--grid); stroke-width: 1; } +svg.chart .axis-line { stroke: var(--axis); stroke-width: 1; } +svg.chart .series { fill: none; stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; } +svg.chart .series--interp { stroke: var(--interp); } +svg.chart .series--jit { stroke: var(--jit); } +svg.chart .band { opacity: 0.10; } +svg.chart .band--interp { fill: var(--interp); } +svg.chart .band--jit { fill: var(--jit); } +svg.chart .end-label { fill: var(--ink-2); font-weight: 600; } +svg.chart .crosshair { stroke: var(--axis); stroke-width: 1; } +svg.chart .marker { stroke: var(--surface); stroke-width: 2; } +svg.chart .marker--interp { fill: var(--interp); } +svg.chart .marker--jit { fill: var(--jit); } +svg.chart .fail-mark { fill: var(--critical); } +svg.chart .hit { fill: transparent; cursor: crosshair; } + +.table-wrap { overflow-x: auto; } +table { border-collapse: collapse; width: 100%; font-size: 13px; } +th, td { text-align: left; padding: 6px 8px; border-top: 1px solid var(--grid); vertical-align: top; } +th { color: var(--muted); font-weight: 500; border-top: 0; } +td.num, th.num { text-align: right; font-variant-numeric: tabular-nums; } +table.mini { font-size: 12px; margin-top: 6px; } + +.tooltip { + position: fixed; z-index: 10; pointer-events: none; max-width: 360px; + background: var(--surface); color: var(--ink); border: 1px solid var(--border); + border-radius: 8px; padding: 8px 10px; box-shadow: 0 6px 24px rgba(0, 0, 0, 0.12); font-size: 12px; +} +.tooltip .tt-head { color: var(--ink-2); margin-bottom: 4px; } +.tooltip .tt-row { display: flex; gap: 8px; align-items: baseline; } +.tooltip .tt-key { display: inline-block; width: 14px; height: 2px; border-radius: 1px; background: var(--muted); } +.tooltip .tt-key--interp { background: var(--interp); } +.tooltip .tt-key--jit { background: var(--jit); } +.tooltip .tt-val { font-weight: 600; font-variant-numeric: tabular-nums; } +.tooltip .tt-sub { color: var(--muted); } + +.foot { display: flex; flex-wrap: wrap; justify-content: space-between; gap: 12px; padding: 16px 24px 28px; color: var(--muted); font-size: 12px; border-top: 1px solid var(--grid); } +.notice { padding: 16px; color: var(--ink-2); } +.notice--error { color: var(--critical); } diff --git a/utils/internal/bench-stand/suite.json b/utils/internal/bench-stand/suite.json new file mode 100644 index 0000000000..75afb1bccc --- /dev/null +++ b/utils/internal/bench-stand/suite.json @@ -0,0 +1,32 @@ +{ + "root": "benchmarks", + "exclude": [ + "**/tests/**", + "**/_*.das" + ], + "lanes": [ + "interp", + "jit", + "aot" + ], + "repeat": 3, + "timeout_seconds": 900, + "memory_limit_mb": 4096, + "regression_threshold": 0.1, + "noise_multiplier": 3.0, + "baseline_runs": 7, + "files": { + "core/array/test01.das": { + "skip": "allocates ~19 GB under persistent_heap and gets OOM-killed; unskip once the benchmark is fixed" + } + }, + "lane_excludes": { + "aot": { + "audio/**": "needs dasAudio's own AOT half, which the benchmark AOT set does not carry", + "terminal/**": "needs dasTerminal's own AOT half, which the benchmark AOT set does not carry", + "sql/**": "needs dasSQLITE and dasPUGIXML's own AOT halves, which the benchmark AOT set does not carry", + "micro/*_shapes.das": "the XML-source shapes need dasPUGIXML's own AOT half", + "core/math/scalar_crt.das": "requires the UnitTest module, which the benchmark AOT set does not carry" + } + } +} \ No newline at end of file diff --git a/utils/internal/bench-stand/test_bench_cli.das b/utils/internal/bench-stand/test_bench_cli.das new file mode 100644 index 0000000000..5414511b35 --- /dev/null +++ b/utils/internal/bench-stand/test_bench_cli.das @@ -0,0 +1,321 @@ +options gen2 + +require dastest/testing_boost public +require bench_history +require bench_runner +require daslib/fio +require daslib/json_boost +require strings +require daslib/strings_boost + +//! The two programs the box drives: `main.das`'s verbs and `nightly.sh`'s ssh gate. Both are +//! spawned rather than called, because their contract with the driver is argv in, exit code and +//! written file out. Every arm skips where it has no shell. + +let private TOOL_DIR = "utils/internal/bench-stand" + +def private has_shell() : bool { + return fexist("/bin/sh") +} + +def private with_temp_dir(prefix : string; blk : block<(dir : string) : void>) { + var err = "" + let dir = create_temp_directory(prefix, err) + verify(!empty(dir), "temp dir created") + invoke(blk, dir) + rmdir_rec(dir) +} + +//! A `meta.json` of the shape `nightly.sh` writes, with the build's own verdict. +def private write_meta(dir : string; build_status : string) : string { + let path = path_join(dir, "meta.json") + fwrite(path, "\{\"run_id\":\"20260908T030000Z-abcdef12\",\"started\":\"2026-09-08T03:00:00Z\"," + + "\"commit\":\{\"sha\":\"abcdef1234567890\",\"date\":\"2026-09-08T01:00:00Z\",\"subject\":\"a commit\",\"author\":\"dev\"\}," + + "\"machine\":\{\"host\":\"box\",\"cores\":4\}," + + "\"build\":\{\"status\":\"{build_status}\",\"seconds\":12,\"log_tail\":\"ninja: build stopped\"\}\}") + return path +} + +//! A stand-in daslang: prints one benchmark sample, writes the report the runner asked for, and +//! answers the JIT probe, so `main.das run` completes without a real build. +def private write_fake_daslang(dir : string) : string { + let path = path_join(dir, "fake_daslang.sh") + fwrite(path, "#!/bin/sh\n" + + "REPORT=\"\"\n" + + "for a in \"$@\"; do case \"$PREV\" in --json-file) REPORT=$a;; esac; PREV=$a; done\n" + + "case \"$*\" in *jit_probe*|*-jit*.das) echo jit-probe-ok; exit 0;; esac\n" + + "echo '\{\"name\":\"b\",\"sub_name\":\"arm\",\"n\":10,\"time_ns\":1000,\"allocs\":0,\"heap_bytes\":0,\"string_allocs\":0,\"string_heap_bytes\":0,\"func_type\":\"INTERP\"\}'\n" + + "[ -n \"$REPORT\" ] && echo '\{\"file\":\"x\",\"total\":0,\"passed\":0,\"failed\":0,\"errors\":0,\"skipped\":0,\"success\":true,\"time_usec\":1,\"tests\":[\{\"name\":\"b\",\"passed\":true,\"skipped\":false,\"time\":1,\"messages\":[],\"location\":\"\"\}]\}' > \"$REPORT\"\n" + + "exit 0\n") + var out = "" + run_and_capture(["/bin/chmod", "755", path], out) + return path +} + +def private write_suite(dir : string; lanes : string) : string { + let path = path_join(dir, "suite.json") + fwrite(path, "\{ \"root\": \"benchmarks\", \"exclude\": [\"**/_*.das\"], \"lanes\": [{lanes}], \"repeat\": 1, \"baseline_runs\": 3 \}") + return path +} + +def private seed_benchmarks(dir : string) { + mkdir_rec(path_join(dir, "benchmarks/core")) + fwrite(path_join(dir, "benchmarks/core/one.das"), "options gen2\n") + fwrite(path_join(dir, "benchmarks/core/two.das"), "options gen2\n") +} + +//! Runs the tool under the host binary and returns its exit code; `output` gets stdout+stderr. +def private run_tool(args : array; var output : string&) : int { + var argv <- ["bin/daslang", path_join(TOOL_DIR, "main.das"), "--"] + argv |> push_from(args) + return run_and_capture(argv, output, 240.0) +} + +[test] +def test_verb_run(t : T?) { + if (!has_shell()) { + t |> skip("needs /bin/sh") + return + } + with_temp_dir("bench_cli_") $(dir) { + seed_benchmarks(dir) + let fake = write_fake_daslang(dir) + let suite = write_suite(dir, "\"interp\"") + t |> run("a clean run records one file per lane and exits 0") @(tt : T?) { + let meta = write_meta(dir, "ok") + let record = path_join(dir, "run.json") + var out = "" + let code = run_tool(["run", "--root", dir, "--bin", fake, "--suite", suite, + "--meta", meta, "--out", record], out) + tt |> equal(code, 0, "exit 0 when every file ran: {out}") + var rec = RunRecord() + tt |> success(sscan_json(fread(record), rec), "the record parses") + tt |> equal(rec.run_id, "20260908T030000Z-abcdef12") + tt |> equal(rec.status, RunStatus.ok) + tt |> equal(length(rec.files), 2, "one file per lane, two benchmark files") + tt |> equal(rec.lanes?["interp"] ?? "", "ok") + tt |> success(rec.seconds >= 0.0lf, "the suite's duration is recorded") + tt |> success(!empty(rec.finished), "the finish stamp is written") + } + t |> run("a meta with no build section is a night that ran, not a night whose build failed") @(tt : T?) { + let bare = path_join(dir, "bare_meta.json") + fwrite(bare, "\{\"run_id\":\"local-1\",\"started\":\"2026-09-09T10:00:00Z\"\}") + let record = path_join(dir, "bare.json") + var out = "" + let code = run_tool(["run", "--root", dir, "--bin", fake, "--suite", suite, + "--meta", bare, "--out", record], out) + tt |> equal(code, 0, "a hand-driven local run exits 0: {out}") + var rec = RunRecord() + tt |> success(sscan_json(fread(record), rec), "the record parses") + tt |> equal(rec.status, RunStatus.ok) + tt |> equal(rec.build.status, "ok", "an absent build section reads as a build that was fine") + } + t |> run("--filter narrows the run to the files whose path contains it") @(tt : T?) { + let meta = write_meta(dir, "ok") + let record = path_join(dir, "filtered.json") + var out = "" + run_tool(["run", "--root", dir, "--bin", fake, "--suite", suite, "--meta", meta, + "--out", record, "--filter", "one"], out) + var rec = RunRecord() + tt |> success(sscan_json(fread(record), rec), "the record parses") + tt |> equal(length(rec.files), 1) + tt |> equal(rec.files[0].path, "core/one.das") + } + t |> run("--lanes overrides the suite, and an unavailable lane runs no file") @(tt : T?) { + let meta = write_meta(dir, "ok") + let record = path_join(dir, "lanes.json") + var out = "" + run_tool(["run", "--root", dir, "--bin", "/nonexistent/daslang", "--suite", suite, + "--meta", meta, "--out", record, "--lanes", "jit"], out) + var rec = RunRecord() + tt |> success(sscan_json(fread(record), rec), "the record parses") + tt |> success(!key_exists(rec.lanes, "interp"), "the suite's lane list was overridden") + tt |> success(find(rec.lanes?["jit"] ?? "", "no jit-compiled run") >= 0, + "the jit lane names why it is unavailable: {rec.lanes?["jit"] ?? ""}") + tt |> success(find(rec.lanes?["jit"] ?? "", "/nonexistent/daslang") >= 0, + "and names the binary it could not run, since that binary printed nothing") + tt |> success(empty(rec.files), "an unavailable lane contributes no file") + } + t |> run("--failed writes the record of a night that never benchmarked and exits 1") @(tt : T?) { + let record = path_join(dir, "failed.json") + var out = "" + let code = run_tool(["run", "--root", dir, "--bin", fake, "--suite", suite, + "--meta", write_meta(dir, "failed"), "--out", record, "--failed", "the build failed"], out) + tt |> equal(code, 1, "a night that could not run is a failure") + var rec = RunRecord() + tt |> success(sscan_json(fread(record), rec), "the record parses") + tt |> equal(rec.status, RunStatus.build_failed) + tt |> success(empty(rec.files), "a failed build carries no file") + } + } +} + +[test] +def test_verb_run_refusals(t : T?) { + if (!has_shell()) { + t |> skip("needs /bin/sh") + return + } + with_temp_dir("bench_cli_") $(dir) { + seed_benchmarks(dir) + let fake = write_fake_daslang(dir) + let suite = write_suite(dir, "\"interp\"") + t |> run("a missing --meta, an unparseable one, and missing arguments each exit 1") @(tt : T?) { + var out = "" + tt |> equal(run_tool(["run", "--root", dir, "--out", path_join(dir, "x.json")], out), 1, "no --meta") + tt |> success(find(out, "--meta and --out are required") >= 0, "the message names what is missing") + let bad = path_join(dir, "bad_meta.json") + fwrite(bad, "not json") + tt |> equal(run_tool(["run", "--root", dir, "--bin", fake, "--suite", suite, + "--meta", bad, "--out", path_join(dir, "y.json")], out), 1, "unparseable meta") + tt |> success(find(out, "does not parse") >= 0, "and says so: {out}") + tt |> equal(run_tool(["nonsense"], out), 1, "an unknown verb prints help and exits 1") + } + t |> run("a suite config that fails to load stops the run before it spawns anything") @(tt : T?) { + let broken = path_join(dir, "broken_suite.json") + fwrite(broken, "\{ \"lanes\": [] \}") + var out = "" + tt |> equal(run_tool(["run", "--root", dir, "--bin", fake, "--suite", broken, + "--meta", write_meta(dir, "ok"), "--out", path_join(dir, "z.json")], out), 1) + tt |> success(find(out, "no lanes") >= 0, "the config's own error reaches the operator") + tt |> success(!fexist(path_join(dir, "z.json")), "and no record is written") + } + } +} + +[test] +def test_verb_report(t : T?) { + if (!has_shell()) { + t |> skip("needs /bin/sh") + return + } + with_temp_dir("bench_cli_") $(dir) { + let suite = write_suite(dir, "\"interp\"") + let runs_dir = path_join(dir, "runs") + mkdir_rec(runs_dir) + t |> run("report writes both artifacts and exits on the latest night's verdict") @(tt : T?) { + for (n in range(4)) { + var rec <- RunRecord(run_id = "r{n}", started = "2026-09-0{n + 1}T03:00:00Z", + commit = CommitInfo(sha = "sha{n}", date = "2026-09-0{n + 1}T01:00:00Z", subject = "c{n}"), + machine = MachineInfo(host = "box"), build = BuildInfo(status = "ok", seconds = 1.0lf), + status = RunStatus.ok) + rec.lanes["interp"] = "ok" + var f <- FileResult(path = "core/one.das", id = "core/one", group = "core", + lane = "interp", status = FileStatus.ok) + f.samples |> emplace(Sample(id = "b/arm", ns = 100.0lf, ns_median = 100.0lf, runs = 1)) + rec.files |> emplace(f) + fwrite(path_join(runs_dir, "r{n}.json"), sprint_json(rec, false)) + } + let data = path_join(dir, "data.json") + let summary = path_join(dir, "summary.md") + var out = "" + let code = run_tool(["report", "--suite", suite, "--runs", runs_dir, + "--out-data", data, "--out-summary", summary, "--generated", "2026-09-09T00:00:00Z", + "--repo-url", "https://example.invalid/repo", "--site-url", "https://example.invalid/bench/"], out) + tt |> equal(code, 0, "a clean latest night exits 0: {out}") + var ds = Dataset() + tt |> success(sscan_json(fread(data), ds), "data.json parses") + tt |> equal(length(ds.runs), 4) + tt |> equal(ds.latest, 3) + tt |> equal(ds.generated, "2026-09-09T00:00:00Z") + tt |> equal(ds.repo_url, "https://example.invalid/repo") + let md = fread(summary) + tt |> success(find(md, "# Benchmark stand - ok") >= 0, "summary.md leads with the verdict") + tt |> success(find(md, "https://example.invalid/bench/") >= 0, "and links the site") + } + t |> run("a corrupt record is warned about and left out, not fatal") @(tt : T?) { + fwrite(path_join(runs_dir, "zz_corrupt.json"), "\{ not json") + var out = "" + let code = run_tool(["report", "--suite", suite, "--runs", runs_dir, + "--out-data", path_join(dir, "d2.json"), "--out-summary", path_join(dir, "s2.md")], out) + tt |> equal(code, 0, "the other nights still publish") + tt |> success(find(out, "zz_corrupt.json") >= 0, "the corrupt file is named: {out}") + remove(path_join(runs_dir, "zz_corrupt.json")) + } + t |> run("an empty runs directory exits 1 and says there is nothing yet") @(tt : T?) { + let empty_dir = path_join(dir, "no_runs") + mkdir_rec(empty_dir) + var out = "" + let code = run_tool(["report", "--suite", suite, "--runs", empty_dir, + "--out-data", path_join(dir, "d3.json"), "--out-summary", path_join(dir, "s3.md")], out) + tt |> equal(code, 1) + tt |> success(find(fread(path_join(dir, "s3.md")), "No run records") >= 0, "the summary says so") + } + t |> run("missing arguments exit 1 before anything is written") @(tt : T?) { + var out = "" + tt |> equal(run_tool(["report", "--suite", suite, "--runs", runs_dir], out), 1) + tt |> success(find(out, "are required") >= 0, "the message names what is missing") + } + } +} + +//! Drives `nightly.sh gate` the way an inbound ssh connection does: the verb and its arguments +//! arrive only through SSH_ORIGINAL_COMMAND, and nothing else on that connection exists. +def private run_gate(home, command : string; var output : string&) : int { + var code = -1 + var out = "" + let env <- ["BENCH_STAND_HOME={home}", "SSH_ORIGINAL_COMMAND={command}"] + with_process(["/bin/bash", path_join(TOOL_DIR, "nightly.sh"), "gate"], ".", env) $(var p) { + unsafe { + while (true) { + process_drain(p) $(line) { + out += "{line}\n" + } + code = process_poll(p) + break if (code != process_running) + sleep(20u) + } + process_drain(p) $(line) { + out += "{line}\n" + } + } + } + output = out + return code +} + +[test] +def test_ssh_gate(t : T?) { + if (!fexist("/bin/bash")) { + t |> skip("needs bash") + return + } + with_temp_dir("bench_gate_") $(home) { + mkdir_rec(path_join(home, "site")) + fwrite(path_join(home, "site/status.json"), "\{\"state\":\"finished\",\"exit\":0\}") + fwrite(path_join(home, "site/summary.md"), "# a night\n") + t |> run("the two read-only verbs answer with what the box published") @(tt : T?) { + var out = "" + tt |> equal(run_gate(home, "status", out), 0) + tt |> success(find(out, "\"state\":\"finished\"") >= 0, "status prints status.json: {out}") + tt |> equal(run_gate(home, "summary", out), 0) + tt |> success(find(out, "# a night") >= 0, "summary prints summary.md") + } + t |> run("an argument carrying anything but path characters is refused") @(tt : T?) { + for (hostile in ["start master;rm -rf /", "start $(whoami)", "start `id`", + "start master&touch /tmp/pwned", "follow ../../etc/passwd|cat"]) { + var out = "" + let code = run_gate(home, hostile, out) + tt |> equal(code, 2, "refused: {hostile}") + tt |> success(find(out, "refused") >= 0, "and says so: {out}") + } + } + t |> run("more arguments than any verb takes is refused") @(tt : T?) { + var out = "" + tt |> equal(run_gate(home, "start master fail-on-regression extra", out), 2) + tt |> success(find(out, "too many arguments") >= 0, "the message names the reason: {out}") + } + t |> run("a verb the gate does not carry is refused, naming it") @(tt : T?) { + var out = "" + tt |> equal(run_gate(home, "run", out), 2, "`run` is a local verb, never an ssh one") + tt |> success(find(out, "unknown verb 'run'") >= 0, "the message names the verb: {out}") + tt |> equal(run_gate(home, "bash", out), 2) + } + t |> run("a missing status file still answers, rather than failing the caller") @(tt : T?) { + remove(path_join(home, "site/status.json")) + var out = "" + tt |> equal(run_gate(home, "status", out), 0) + tt |> success(find(out, "never-run") >= 0, "it says the stand has not run: {out}") + } + } +} diff --git a/utils/internal/bench-stand/test_bench_history.das b/utils/internal/bench-stand/test_bench_history.das new file mode 100644 index 0000000000..1e9a530382 --- /dev/null +++ b/utils/internal/bench-stand/test_bench_history.das @@ -0,0 +1,406 @@ +options gen2 + +require dastest/testing_boost public +require bench_history +require bench_runner +require bench_suite +require daslib/fio +require daslib/json_boost +require strings +require daslib/strings_boost +require math + +def private with_temp_dir(prefix : string; blk : block<(dir : string) : void>) { + var err = "" + let dir = create_temp_directory(prefix, err) + verify(!empty(dir), "temp dir created") + invoke(blk, dir) + rmdir_rec(dir) +} + +def private sample(id : string; ns : double) : Sample { + return Sample(id = id, ns = ns, ns_median = ns, spread = 0.05lf, runs = 3) +} + +def private file_ok(id, lane : string; samples : array) : FileResult { + var f <- FileResult(path = "{id}.das", id = id, group = dir_name(id), lane = lane, status = FileStatus.ok, tier = lane == "jit" ? "JIT" : "INTERP") + f.samples := samples + return <- f +} + +def private file_bad(id, lane : string; status : FileStatus; message : string) : FileResult { + return <- FileResult(path = "{id}.das", id = id, group = dir_name(id), lane = lane, status = status, message = message) +} + +//! Night `n` (0-based): commit date and run start both increase with n, so commit order equals run order. +def private night(n : int; files : array) : RunRecord { + var rec <- RunRecord(run_id = "r{n:02}", started = "2026-09-{n + 1:02}T03:00:00Z", finished = "2026-09-{n + 1:02}T04:00:00Z", seconds = 3600.0lf, + commit = CommitInfo(sha = "sha{n:02}abcdef", date = "2026-09-{n + 1:02}T00:00:00Z", subject = "commit {n}", author = "dev"), + machine = MachineInfo(host = "box", cpu = "cpu", cores = 4), build = BuildInfo(status = "ok", seconds = 600.0lf)) + rec.lanes["interp"] = "ok" + rec.lanes["jit"] = "ok" + rec.files := files + return <- rec +} + +def private write_night(dir : string; rec : RunRecord) { + fwrite(path_join(dir, "{rec.run_id}.json"), sprint_json(rec, false)) +} + +def private load(dir : string) : array { + var errors : array + return <- load_runs(dir, errors) +} + +//! The steady week plus an eighth night where a/x jumps +30% (interp) and drops -20% (jit), c/z +//! appears, b/y vanishes, and sort/s fails to compile. +def private eight_nights_with_a_jump(dir : string) { + steady_nights(dir, 7) + write_night(dir, night(7, [ + file_ok("core/hash/t", "interp", [sample("a/x", 130.0lf), sample("c/z", 5.0lf)]), + file_ok("core/hash/t", "jit", [sample("a/x", 40.0lf)]), + file_bad("sort/s", "interp", FileStatus.compile_error, "error[30344]: mismatch")])) +} + +def private cfg() : SuiteConfig { + var c = SuiteConfig() + c.lanes <- ["interp", "jit"] + return <- c +} + +//! `count` nights near 100 ns (interp) and 50 ns (jit) for `core/hash/t#a/x` and near 200 ns for +//! `b/y`, jittered so the baseline has a nonzero MAD - the noise gate needs one to be exercised. +def private steady_nights(dir : string; count : int) { + for (n in range(count)) { + let jitter = double(n % 3) * 0.5lf + write_night(dir, night(n, [ + file_ok("core/hash/t", "interp", [sample("a/x", 100.0lf + jitter), sample("b/y", 200.0lf - jitter)]), + file_ok("core/hash/t", "jit", [sample("a/x", 50.0lf + jitter)])])) + } +} + +[test] +def test_load_runs(t : T?) { + with_temp_dir("bench_hist_") $(dir) { + t |> run("records load in commit order, a corrupt file is named and skipped, status derives") @(tt : T?) { + write_night(dir, night(2, [file_ok("core/t", "interp", [sample("a/x", 1.0lf)])])) + write_night(dir, night(0, [file_ok("core/t", "interp", [sample("a/x", 1.0lf)])])) + let broken <- night(1, [file_bad("core/t", "interp", FileStatus.timeout, "killed")]) + write_night(dir, broken) + fwrite(path_join(dir, "zz_corrupt.json"), "\{ not json") + fwrite(path_join(dir, "notes.txt"), "ignored") + var errors : array + let runs <- load_runs(dir, errors) + tt |> equal(length(runs), 3) + tt |> equal(runs[0].run_id, "r00") + tt |> equal(runs[1].run_id, "r01") + tt |> equal(runs[2].run_id, "r02") + tt |> equal(runs[1].status, RunStatus.bench_failed) + tt |> equal(runs[0].status, RunStatus.ok) + tt |> equal(runs[0].files[0].status, FileStatus.ok) + tt |> equal(length(errors), 1) + tt |> success(find(errors[0], "zz_corrupt.json") >= 0, "corrupt file named") + } + t |> run("a night that measured nothing is not ok, whatever emptied it") @(tt : T?) { + let no_files : array + let empty_night <- night(30, no_files) + tt |> equal(derive_status(empty_night), RunStatus.bench_failed, + "no file ran at all - every lane unavailable, or a filter that matched nothing") + let only_skips <- night(31, [file_bad("core/x", "interp", FileStatus.skipped, "listed")]) + tt |> equal(derive_status(only_skips), RunStatus.bench_failed, "a night of nothing but skips carries no data point") + let one_ok <- night(32, [file_ok("core/x", "interp", [sample("a/x", 1.0lf)])]) + tt |> equal(derive_status(one_ok), RunStatus.ok, "one measured file is a night") + } + t |> run("a build failure written by the driver derives build_failed") @(tt : T?) { + let no_files : array + var rec <- night(5, no_files) + rec.build.status = "failed" + tt |> equal(derive_status(rec), RunStatus.build_failed) + } + } +} + +[test] +def test_statistics(t : T?) { + t |> run("median, relative MAD, geomean") @(tt : T?) { + var odd <- [3.0lf, 1.0lf, 2.0lf] + tt |> success(abs(median(odd) - 2.0lf) < 1e-9lf, "odd median") + var even <- [4.0lf, 1.0lf, 3.0lf, 2.0lf] + tt |> success(abs(median(even) - 2.5lf) < 1e-9lf, "even median") + tt |> success(abs(relative_mad([100.0lf, 110.0lf, 90.0lf, 100.0lf, 105.0lf]) - 0.05lf) < 1e-9lf, "MAD 5 over median 100") + tt |> success(abs(geomean([1.0lf, 4.0lf]) - 2.0lf) < 1e-9lf, "geomean of 1 and 4") + var none : array + tt |> equal(median(none), 0.0lf) + } +} + +[test] +def test_build_dataset(t : T?) { + with_temp_dir("bench_hist_") $(dir) { + eight_nights_with_a_jump(dir) + t |> run("series are keyed by lane and arm, points carry the run index") @(tt : T?) { + let runs <- load(dir) + let ds <- build_dataset(runs, cfg(), "2026-09-08T05:00:00Z", "https://github.com/x/y") + tt |> equal(length(ds.runs), 8) + tt |> equal(ds.latest, 7) + tt |> equal(length(ds.groups), 1) + tt |> equal(ds.groups[0], "core/hash") + tt |> equal(length(ds.series), 4) + tt |> equal(ds.series[0].lane, "interp") + tt |> equal(ds.series[0].id, "core/hash/t#a/x") + tt |> equal(length(ds.series[0].runs), 8) + tt |> equal(ds.series[0].runs[7], 7) + tt |> success(abs(ds.series[0].ns[7] - 130.0lf) < 1e-9lf, "tonight's value") + tt |> equal(ds.series[3].lane, "jit") + } + t |> run("a 30% jump over a quiet baseline is a regression, a 20% drop an improvement, a first sighting is new") @(tt : T?) { + let runs <- load(dir) + let ds <- build_dataset(runs, cfg(), "", "") + let kinds <- { for (ch in ds.changes); "{ch.lane} {ch.id}" => "{ch.kind}" } + tt |> equal(kinds?["interp core/hash/t#a/x"] ?? "", "regression") + tt |> equal(kinds?["jit core/hash/t#a/x"] ?? "", "improvement") + tt |> equal(kinds?["interp core/hash/t#c/z"] ?? "", "first_seen") + tt |> equal(kinds?["interp core/hash/t#b/y"] ?? "", "missing") + tt |> equal(length(ds.changes), 4) + for (ch in ds.changes) { + if (ch.kind == ChangeKind.regression) { + tt |> success(abs(ch.baseline - 100.5lf) < 1e-9lf, "baseline is the median of the 7 prior points: {ch.baseline}") + tt |> success(ch.change > 0.29lf && ch.change < 0.30lf, "change near +29%: {ch.change}") + } + } + tt |> equal(ds.runs[7].regressions, 1) + tt |> equal(ds.runs[7].improvements, 1) + } + t |> run("a series with fewer than three prior points gets no verdict") @(tt : T?) { + var two : array + two |> reserve(3) + for (n in range(3)) { + two |> emplace(night(n, [file_ok("core/hash/t", "interp", [sample("a/x", n == 2 ? 300.0lf : 100.0lf)])])) + } + let ds <- build_dataset(two, cfg(), "", "") + tt |> equal(length(ds.changes), 0) + tt |> equal(ds.runs[2].regressions, 0) + } + t |> run("a run measuring fewer than half a group's series contributes no index point") @(tt : T?) { + var nights : array + nights |> reserve(4) + for (n in range(4)) { + var arms <- [sample("a/x", 100.0lf), sample("b/y", 200.0lf), sample("c/z", 300.0lf)] + if (n == 3) { + arms |> resize(1) + } + nights |> emplace(night(n, [file_ok("core/hash/t", "interp", arms)])) + } + let ds <- build_dataset(nights, cfg(), "", "") + tt |> equal(length(ds.index), 1) + tt |> equal(length(ds.index[0].runs), 3) + tt |> equal(ds.index[0].runs[2], 2) + } + } +} + +[test] +def test_change_gates(t : T?) { + with_temp_dir("bench_hist_") $(dir) { + eight_nights_with_a_jump(dir) + t |> run("a noisy baseline widens the gate past the threshold, so the same jump is no verdict") @(tt : T?) { + // the prior week swings +-20%, so 3x its MAD is a 60% gate and a 30% jump is inside it + var noisy : array + noisy |> reserve(8) + let swing <- [100.0lf, 120.0lf, 80.0lf, 120.0lf, 80.0lf, 120.0lf, 80.0lf] + for (n, v in range(7), swing) { + noisy |> emplace(night(n, [file_ok("core/hash/t", "interp", [sample("a/x", v)])])) + } + noisy |> emplace(night(7, [file_ok("core/hash/t", "interp", [sample("a/x", 130.0lf)])])) + let ds <- build_dataset(noisy, cfg(), "", "") + tt |> equal(length(ds.changes), 0, "a 30% move over a +-20% baseline is noise, not a regression") + var quiet <- cfg() + quiet.noise_multiplier = 0.0lf + let ds_quiet <- build_dataset(noisy, quiet, "", "") + tt |> equal(length(ds_quiet.changes), 1, "with the noise term off, the threshold alone calls it") + tt |> equal(ds_quiet.changes[0].kind, ChangeKind.regression) + tt |> success(ds_quiet.changes[0].noise > 0.15lf, "the recorded noise is the baseline's own: {ds_quiet.changes[0].noise}") + } + t |> run("the baseline is the last baseline_runs points, not every point on record") @(tt : T?) { + // four quiet nights at 10 ns, then three at 100: a window of 3 makes 100 the baseline + var shifted : array + shifted |> reserve(8) + for (n in range(7)) { + shifted |> emplace(night(n, [file_ok("core/hash/t", "interp", [sample("a/x", n < 4 ? 10.0lf : 100.0lf)])])) + } + shifted |> emplace(night(7, [file_ok("core/hash/t", "interp", [sample("a/x", 105.0lf)])])) + var windowed <- cfg() + windowed.baseline_runs = 3 + let ds <- build_dataset(shifted, windowed, "", "") + tt |> equal(length(ds.changes), 0, "105 against a 100 baseline is inside the gate") + let ds_wide <- build_dataset(shifted, cfg(), "", "") + tt |> equal(length(ds_wide.changes), 1, "a seven-run window makes the baseline 10 and the night a regression") + tt |> success(ds_wide.changes[0].baseline < 20.0lf, "the wide baseline is the old level: {ds_wide.changes[0].baseline}") + } + t |> run("a move inside the threshold or inside the noise band is not a change") @(tt : T?) { + let runs <- load(dir) + var c <- cfg() + c.regression_threshold = 0.5lf + let ds <- build_dataset(runs, c, "", "") + var regs = 0 + for (ch in ds.changes) { + regs++ if (ch.kind == ChangeKind.regression || ch.kind == ChangeKind.improvement) + } + tt |> equal(regs, 0) + } + } +} + +[test] +def test_build_dataset_index_and_summary_rows(t : T?) { + with_temp_dir("bench_hist_") $(dir) { + eight_nights_with_a_jump(dir) + t |> run("the group index is 100 at the reference and moves with the group") @(tt : T?) { + let runs <- load(dir) + let ds <- build_dataset(runs, cfg(), "", "") + tt |> equal(length(ds.index), 2) + tt |> equal(ds.index[0].group, "core/hash") + tt |> equal(ds.index[0].lane, "interp") + tt |> equal(length(ds.index[0].runs), 8) + tt |> success(abs(ds.index[0].value[0] - 100.0lf) < 1.0lf, "first night near 100: {ds.index[0].value[0]}") + tt |> success(ds.index[0].value[7] > 105.0lf, "tonight's regression lifts the index: {ds.index[0].value[7]}") + tt |> equal(ds.index[1].lane, "jit") + tt |> success(ds.index[1].value[7] < 90.0lf, "jit improvement lowers its index: {ds.index[1].value[7]}") + } + t |> run("the run summary carries failures with their message and the ok count") @(tt : T?) { + let runs <- load(dir) + let ds <- build_dataset(runs, cfg(), "", "") + tt |> equal(ds.runs[7].status, RunStatus.bench_failed) + tt |> equal(ds.runs[7].files_ok, 2) + tt |> equal(length(ds.runs[7].failures), 1) + tt |> equal(ds.runs[7].failures[0].path, "sort/s.das") + tt |> equal(ds.runs[7].failures[0].status, "compile_error") + tt |> equal(ds.runs[7].failures[0].message, "error[30344]: mismatch") + tt |> equal(ds.runs[7].lanes?["jit"] ?? "", "ok") + } + t |> run("the dataset round-trips through JSON") @(tt : T?) { + let runs <- load(dir) + let ds <- build_dataset(runs, cfg(), "now", "") + var back = Dataset() + tt |> success(sscan_json(sprint_json(ds, false), back), "parses back") + tt |> equal(length(back.series), 4) + tt |> equal(back.latest, 7) + tt |> equal(length(back.changes), 4) + } + } +} + +[test] +def test_summary(t : T?) { + with_temp_dir("bench_hist_") $(dir) { + steady_nights(dir, 7) + t |> run("a clean night renders ok and exits 0") @(tt : T?) { + var errors : array + let runs <- load_runs(dir, errors) + let ds <- build_dataset(runs, cfg(), "", "") + let md = render_summary(ds, "https://daslang.io/bench/") + tt |> success(starts_with(md, "# Benchmark stand - ok - "), "title says ok: {slice(md, 0, 40)}") + tt |> success(find(md, "0 regressions, 0 improvements") >= 0, "counts zero changes") + tt |> success(find(md, "https://daslang.io/bench/") >= 0, "links the site") + tt |> equal(summary_exit_code(ds, true), 0) + } + t |> run("a failed night names the file and exits 1; a regression exits 2 only on request") @(tt : T?) { + write_night(dir, night(7, [ + file_ok("core/hash/t", "interp", [sample("a/x", 150.0lf), sample("b/y", 200.0lf)]), + file_ok("core/hash/t", "jit", [sample("a/x", 50.0lf)]), + file_bad("sort/s", "jit", FileStatus.timeout, "killed after 900 s (timeout_seconds)")])) + var errors : array + let runs <- load_runs(dir, errors) + let ds <- build_dataset(runs, cfg(), "", "") + let md = render_summary(ds, "") + tt |> success(find(md, "`sort/s.das` | jit | timeout | killed after 900 s") >= 0, "failure row present") + tt |> success(find(md, "### Regressions") >= 0, "regression section present") + tt |> success(find(md, "`core/hash/t#a/x` | interp | 100.5 | 150.0 | +49.3%") >= 0, "regression row formatted: {md}") + tt |> equal(summary_exit_code(ds, false), 1) + var clean := runs[7] + clean.files |> erase(2) + write_night(dir, clean) + var errors2 : array + let runs2 <- load_runs(dir, errors2) + let ds2 <- build_dataset(runs2, cfg(), "", "") + tt |> equal(summary_exit_code(ds2, false), 0) + tt |> equal(summary_exit_code(ds2, true), 2) + } + } +} + +[test] +def test_summary_of_a_night_that_failed(t : T?) { + with_temp_dir("bench_hist_") $(dir) { + t |> run("a build failure leads with its reason and the build log tail, and nothing else") @(tt : T?) { + let no_files : array + var rec <- night(7, no_files) + rec.build.status = "failed" + rec.build.seconds = 41.0lf + rec.build.log_tail = "ninja: error: unknown target 'daslang'" + write_night(dir, rec) + let ds <- build_dataset(load(dir), cfg(), "", "https://daslang.io/bench/") + let md = render_summary(ds, "https://daslang.io/bench/") + tt |> success(find(md, "build_failed") >= 0, "the title names the status") + tt |> success(find(md, "## The build failed after 41 s - nothing was measured") >= 0, "the build section leads: {md}") + tt |> success(find(md, "unknown target 'daslang'") >= 0, "the log tail is quoted") + tt |> success(find(md, "## Changes") < 0, "no change section on a night that measured nothing") + tt |> success(find(md, "https://daslang.io/bench/") >= 0, "the charts link survives") + tt |> equal(summary_exit_code(ds, false), 1) + remove(path_join(dir, "{rec.run_id}.json")) + } + t |> run("improvements, first sightings, missing arms, skips and a dark lane all render") @(tt : T?) { + eight_nights_with_a_jump(dir) + var rec <- night(8, [ + file_ok("core/hash/t", "interp", [sample("a/x", 40.0lf), sample("d/w", 7.0lf)]), + file_bad("core/hash/slow", "interp", FileStatus.skipped, "takes an hour")]) + rec.lanes["jit"] = "daslang -jit exited with code 1: no dasLLVM in this build" + write_night(dir, rec) + let ds <- build_dataset(load(dir), cfg(), "", "") + let md = render_summary(ds, "") + tt |> success(find(md, "### Improvements") >= 0, "improvement section") + tt |> success(find(md, "`core/hash/t#a/x` | interp | 100.5 | 40.0 | -60.2%") >= 0, "improvement row: {md}") + tt |> success(find(md, "### Missing tonight") >= 0, "missing section") + tt |> success(find(md, "`core/hash/t#b/y`") >= 0, "the missing arm is named") + tt |> success(find(md, "first seen") >= 0, "first sightings counted") + tt |> success(find(md, "## Skipped (1)") >= 0, "skip section") + tt |> success(find(md, "`core/hash/slow.das` (interp): takes an hour") >= 0, "the skip states its reason") + tt |> success(find(md, "jit = daslang -jit exited with code 1") >= 0, "a dark lane says why") + tt |> success(find(md, "1 file runs ok, 0 failed, 1 skipped") >= 0, "the counts line") + remove(path_join(dir, "{rec.run_id}.json")) + } + t |> run("a night that never benchmarked names the driver as the failure") @(tt : T?) { + var rec <- night(9, [file_ok("core/hash/t", "interp", [sample("a/x", 100.0lf)])]) + mark_failed_run(rec, "2026-09-10T04:00:00Z", "the tool exited 1 without writing a record") + tt |> equal(rec.status, RunStatus.bench_failed) + tt |> equal(length(rec.files), 1) + tt |> equal(rec.files[0].lane, "driver") + tt |> success(find(rec.files[0].message, "exited 1") >= 0, "the reason survives") + tt |> equal(rec.seconds, 0.0lf) + tt |> equal(rec.finished, "2026-09-10T04:00:00Z") + let none_either : array + var build_broke <- night(10, none_either) + build_broke.build.status = "failed" + mark_failed_run(build_broke, "2026-09-11T04:00:00Z", "") + tt |> equal(build_broke.status, RunStatus.build_failed) + tt |> success(empty(build_broke.files), "a failed build carries no file - its log tail is the reason") + } + t |> run("a run with no commit names no commit, rather than empty backticks") @(tt : T?) { + let no_files : array + var bare <- night(20, no_files) + bare.commit = CommitInfo() + bare.machine = MachineInfo() + var one : array + one |> emplace(bare) + let md = render_summary(build_dataset(one, cfg(), "", ""), "") + tt |> success(find(md, "Commit ``") < 0, "no empty commit line: {md}") + tt |> success(find(md, " on ``") < 0, "and no empty host clause") + } + t |> run("no records renders a note and exits 1") @(tt : T?) { + let none : array + let ds <- build_dataset(none, cfg(), "", "") + tt |> success(find(render_summary(ds, ""), "No run records") >= 0, "note") + tt |> equal(summary_exit_code(ds, false), 1) + } + } +} diff --git a/utils/internal/bench-stand/test_bench_runner.das b/utils/internal/bench-stand/test_bench_runner.das new file mode 100644 index 0000000000..59eebdd684 --- /dev/null +++ b/utils/internal/bench-stand/test_bench_runner.das @@ -0,0 +1,358 @@ +options gen2 + +require dastest/testing_boost public +require bench_runner +require bench_suite +require daslib/fio +require strings +require daslib/strings_boost +require math + +def private stats_line(name, sub : string; n : int; time_ns : int64; allocs : int = 0; tier : string = "INTERP"; + string_allocs : int = 0) : string { + return "\{\"name\":\"{name}\",\"sub_name\":\"{sub}\",\"n\":{n},\"time_ns\":{time_ns},\"allocs\":{allocs},\"heap_bytes\":{allocs * 16},\"string_allocs\":{string_allocs},\"string_heap_bytes\":{string_allocs * 32},\"func_type\":\"{tier}\"\}" +} + +def private report(success : bool; names : array; failed : array) : string { + let tests <- [for (n in names); "\{\"name\":\"{n}\",\"passed\":{find_index(failed, n) < 0},\"skipped\":false,\"time\":1,\"messages\":[],\"location\":\"\"\}"] + return "\{\"file\":\"x.das\",\"total\":0,\"passed\":0,\"failed\":{length(failed)},\"errors\":0,\"skipped\":0,\"success\":{success},\"time_usec\":1,\"tests\":[{join(tests, ",")}]\}" +} + +def private fresh_result(status : FileStatus = FileStatus.ok) : FileResult { + return <- FileResult(path = "core/x.das", id = "core/x", group = "core", lane = "interp", status = status) +} + +[test] +def test_parse_bench_output(t : T?) { + t |> run("samples aggregate over repeats: min, median, spread, per-op allocation medians") @(tt : T?) { + var res <- fresh_result() + let lines <- [ + "2026-09-08T14:11:43.241Z run 1/3: benchmarks/core/x.das", + stats_line("bench_a", "insert/100", 100, 10000l, 300, "INTERP", 100), + stats_line("bench_a", "read/100", 1000, 50000l), + stats_line("bench_a", "insert/100", 100, 12000l, 500, "INTERP", 100), + stats_line("bench_a", "read/100", 1000, 40000l), + stats_line("bench_a", "insert/100", 100, 11000l, 400, "INTERP", 100), + stats_line("bench_a", "read/100", 1000, 45000l), + "", + "0 tests, 0 passed, 0 failed, 0 errors, 0 skipped", + "SUCCESS! (3.9s)"] + parse_bench_output(lines, report(true, ["bench_a"], []), 0, res) + tt |> equal(res.status, FileStatus.ok) + tt |> equal(res.exit_code, 0) + tt |> equal(res.tier, "INTERP") + tt |> equal(length(res.samples), 2) + tt |> equal(res.samples[0].id, "bench_a/insert/100") + tt |> equal(res.samples[0].runs, 3) + tt |> success(abs(res.samples[0].ns - 100.0lf) < 0.001lf, "min is 100 ns/op") + tt |> success(abs(res.samples[0].ns_median - 110.0lf) < 0.001lf, "median is 110 ns/op") + tt |> success(abs(res.samples[0].spread - 0.2lf) < 0.001lf, "spread is (120-100)/100") + tt |> equal(res.samples[0].allocs, 4l, "allocs is the per-op median of 3, 4, 5") + tt |> equal(res.samples[0].bytes, 64l, "heap bytes is the per-op median of 48, 64, 80") + tt |> equal(res.samples[0].string_allocs, 1l) + tt |> equal(res.samples[0].string_bytes, 32l) + tt |> equal(res.samples[1].allocs, 0l, "an arm that allocates nothing reports zero") + tt |> equal(res.samples[1].id, "bench_a/read/100") + tt |> success(abs(res.samples[1].ns - 40.0lf) < 0.001lf, "read min is 40 ns/op") + tt |> equal(res.message, "") + } + t |> run("the log tail keeps the non-sample lines, newest last") @(tt : T?) { + var res <- fresh_result() + var lines <- [for (i in range(100)); "line {i}"] + lines |> push(stats_line("b", "s", 10, 100l)) + parse_bench_output(lines, report(true, ["b"], []), 0, res) + tt |> success(find(res.log_tail, "line 0\n") < 0, "oldest lines dropped") + tt |> success(ends_with(res.log_tail, "line 99"), "newest line kept") + tt |> success(find(res.log_tail, "\"sub_name\"") < 0, "sample lines are not log") + } + t |> run("a compile error names the first error line") @(tt : T?) { + var res <- fresh_result() + let lines <- [ + "[E] Failed to compile benchmarks/core/x.das", + "error[30344]: local variable x initialization type mismatch; int const = string const", + "benchmarks/core/x.das:5:8", + "FAIL benchmarks/core/x.das (0.04s)"] + parse_bench_output(lines, "", 1, res) + tt |> equal(res.status, FileStatus.compile_error) + tt |> success(find(res.message, "error[30344]") >= 0, "message carries the error code") + tt |> equal(res.exit_code, 1) + } + t |> run("a failed benchmark function drops its arms and names itself; the others survive") @(tt : T?) { + var res <- fresh_result() + let lines <- [ + "benchmarks/core/x.das:5: values differ", + stats_line("failing", "fail_op", 100, 100l), + "[E] --- FAIL 'failing' in 'fail_op' (1.7s)", + stats_line("ok_after", "ok_op", 100, 100l), + "FAIL benchmarks/core/x.das (3.0s)"] + parse_bench_output(lines, report(false, ["failing", "ok_after"], ["failing"]), 1, res) + tt |> equal(res.status, FileStatus.failed) + tt |> success(find(res.message, "failing") >= 0, "message names the function") + tt |> equal(length(res.samples), 1) + tt |> equal(res.samples[0].id, "ok_after/ok_op") + } +} + +[test] +def test_parse_bench_output_verdicts(t : T?) { + t |> run("the FAIL log line alone marks the function failed when no report was written") @(tt : T?) { + var res <- fresh_result() + let lines <- [stats_line("failing", "fail_op", 100, 100l), "[E] --- FAIL 'failing' in 'fail_op' (1.7s)"] + parse_bench_output(lines, "", 1, res) + tt |> equal(res.status, FileStatus.failed) + tt |> equal(length(res.samples), 0) + } + t |> run("a measured file that dies at shutdown keeps its numbers and says so") @(tt : T?) { + var res <- fresh_result() + let lines <- [ + stats_line("b", "s", 10, 100l), + "SUCCESS! (48.9s)", + "FATAL: exiting with code 1 -- 1 smart pointer(s) still alive at shutdown"] + parse_bench_output(lines, report(true, ["b"], []), 1, res) + tt |> equal(res.status, FileStatus.ok, "the measurement stands: {res.message}") + tt |> equal(length(res.samples), 1) + tt |> success(find(res.message, "at shutdown") >= 0, "the message names when it happened") + tt |> success(find(res.message, "smart pointer") >= 0, "and carries the FATAL line") + } + t |> run("a nonzero exit with no finished report is a failure, samples or not") @(tt : T?) { + var res <- fresh_result() + let lines <- [stats_line("b", "s", 10, 100l)] + parse_bench_output(lines, "", 1, res) + tt |> equal(res.status, FileStatus.exit_nonzero) + tt |> success(find(res.message, "did not run to its end") >= 0, "the message says the suite was cut short: {res.message}") + } + t |> run("a limit verdict survives whatever the output says, keeping the finished arms") @(tt : T?) { + var res <- fresh_result(FileStatus.timeout) + res.message = "killed after 1 s (timeout_seconds)" + let lines <- [stats_line("b", "done", 10, 100l)] + parse_bench_output(lines, "", 9, res) + tt |> equal(res.status, FileStatus.timeout) + tt |> equal(res.message, "killed after 1 s (timeout_seconds)") + tt |> equal(length(res.samples), 1) + tt |> equal(res.exit_code, 9) + } + t |> run("a clean exit with no samples and no report is not ok") @(tt : T?) { + var res <- fresh_result() + let no_lines : array + parse_bench_output(no_lines, "", 0, res) + tt |> equal(res.status, FileStatus.exit_nonzero) + var res2 <- fresh_result() + parse_bench_output(no_lines, report(true, no_lines, no_lines), 0, res2) + tt |> equal(res2.status, FileStatus.failed) + tt |> success(find(res2.message, "no benchmark samples") >= 0, "message says what is missing") + } + t |> run("is_failure separates ok and skipped from the rest") @(tt : T?) { + tt |> success(!is_failure(FileStatus.ok), "ok") + tt |> success(!is_failure(FileStatus.skipped), "skipped") + tt |> success(is_failure(FileStatus.timeout), "timeout") + tt |> success(is_failure(FileStatus.exit_nonzero), "exit_nonzero") + } +} + +def private has_shell() : bool { + return fexist("/bin/sh") +} + +def private bench_file() : BenchFile { + return BenchFile(path = "x/y.das", id = "x/y", group = "x", timeout_seconds = 2, memory_limit_mb = 64) +} + +//! A stand-in for the daslang binary. `body` runs with `$REPORT` set to the path the runner +//! passed after `--json-file` and `$ALL` to the whole argv, so a fixture reads the arguments by +//! name rather than encoding their order. +def private fake_daslang(body : string) : string { + var err = "" + let path = create_temp_file("bench_stand_fake_", ".sh", err) + fwrite(path, "#!/bin/sh\nALL=\"$*\"\nREPORT=\"\"\nwhile [ $# -gt 0 ]; do\n [ \"$1\" = --json-file ] && REPORT=$2\n shift\ndone\n{body}\n") + var out = "" + run_and_capture(["/bin/chmod", "755", path], out) + return path +} + +let private report_text = "\{\"file\":\"x.das\",\"total\":0,\"passed\":0,\"failed\":0,\"errors\":0,\"skipped\":0,\"success\":true,\"time_usec\":1,\"tests\":[\{\"name\":\"b\",\"passed\":true,\"skipped\":false,\"time\":1,\"messages\":[],\"location\":\"\"\}]\}" + +[test] +def test_run_bench_file_limits(t : T?) { + if (!fexist("/bin/sh")) { + t |> skip("needs /bin/sh") + return + } + t |> run("a skipped file never spawns and carries its reason") @(tt : T?) { + var f = bench_file() + f.skip = "too slow" + let res <- run_bench_file("/bin/sh", ".", f, "interp", RunLimits()) + tt |> equal(res.status, FileStatus.skipped) + tt |> equal(res.message, "too slow") + } + t |> run("a child that outlives timeout_seconds is killed; the arms it printed survive") @(tt : T?) { + let fake = fake_daslang("echo '{stats_line("b", "done", 10, 100l)}'; exec sleep 30") + let res <- run_bench_file(fake, ".", bench_file(), "interp", RunLimits(timeout_seconds = 1)) + remove(fake) + tt |> equal(res.status, FileStatus.timeout) + tt |> success(find(res.message, "timeout_seconds") >= 0, "message names the knob") + tt |> success(res.seconds >= 1.0lf && res.seconds < 15.0lf, "killed near the limit, not at the child's own end") + tt |> equal(length(res.samples), 1) + } + t |> run("a child whose resident set passes memory_limit_mb is killed and reported") @(tt : T?) { + if (!fexist("/usr/bin/perl")) { + tt |> skip("needs perl to grow a child's resident set") + return + } + let fake = fake_daslang("exec /usr/bin/perl -e '$x = \"a\" x 200000000; sleep 30'") + let res <- run_bench_file(fake, ".", bench_file(), "interp", RunLimits(timeout_seconds = 20, memory_limit_mb = 64)) + remove(fake) + tt |> equal(res.status, FileStatus.memory) + tt |> success(find(res.message, "memory_limit_mb") >= 0, "message names the knob") + tt |> success(res.seconds < 15.0lf, "killed well before the timeout") + } + t |> run("a child that exits cleanly with samples is ok, from the repo root cwd") @(tt : T?) { + let fake = fake_daslang("echo '{stats_line("b", "done", 10, 100l)}'; echo '{report_text}' > \"$REPORT\"; exit 0") + let res <- run_bench_file(fake, ".", bench_file(), "interp", RunLimits(timeout_seconds = 5)) + remove(fake) + tt |> equal(res.status, FileStatus.ok) + tt |> equal(length(res.samples), 1) + tt |> equal(res.exit_code, 0) + } + t |> run("the jit lane puts -jit ahead of dastest, and the interp lane does not") @(tt : T?) { + let fake = fake_daslang("echo \"ARGV: $ALL\"; exit 1") + let jit <- run_bench_file(fake, ".", bench_file(), "jit", RunLimits(timeout_seconds = 10)) + let interp <- run_bench_file(fake, ".", bench_file(), "interp", RunLimits(timeout_seconds = 10, repeat = 1)) + remove(fake) + tt |> success(find(jit.log_tail, "ARGV: -jit ") >= 0, "the jit lane's first argument is -jit: {jit.log_tail}") + tt |> success(find(jit.log_tail, "dastest/dastest.das") >= 0, "and dastest follows it") + tt |> success(find(interp.log_tail, "-jit") < 0, "the interp lane names no -jit: {interp.log_tail}") + tt |> success(find(interp.log_tail, "--count 1") >= 0, "the repeat count reaches the child: {interp.log_tail}") + } +} + +[test] +def test_run_bench_file_lanes(t : T?) { + if (!has_shell()) { + t |> skip("needs /bin/sh") + return + } + t |> run("the aot lane carries both use-aot flags, or it would measure the interpreter") @(tt : T?) { + let fake = fake_daslang("echo \"ARGV: $ALL\"; exit 1") + let aot <- run_bench_file(fake, ".", bench_file(), "aot", RunLimits(timeout_seconds = 10)) + remove(fake) + // the host flag arms the stub lookup, the dastest flag arms fail_on_no_aot; with only the + // first, every function silently interprets and the lane reports interpreter numbers + tt |> success(find(aot.log_tail, "ARGV: -use-aot ") >= 0, "the host flag comes first: {aot.log_tail}") + tt |> success(find(aot.log_tail, "-- --use-aot ") >= 0, "and dastest's own flag follows the separator") + tt |> success(find(aot.log_tail, "-jit") < 0, "the aot lane is not the jit lane") + } + t |> run("a file skipped in one lane still measures in the others") @(tt : T?) { + var f = bench_file() + f.skip_lanes["aot"] = "the emitter cannot handle this one" + tt |> equal(skip_reason(f, "aot"), "the emitter cannot handle this one") + tt |> equal(skip_reason(f, "interp"), "", "the other lanes are untouched") + let fake = fake_daslang("echo '{stats_line("b", "done", 10, 100l)}'; echo '{report_text}' > \"$REPORT\"; exit 0") + let aot <- run_bench_file(fake, ".", f, "aot", RunLimits(timeout_seconds = 5)) + let interp <- run_bench_file(fake, ".", f, "interp", RunLimits(timeout_seconds = 5)) + remove(fake) + tt |> equal(aot.status, FileStatus.skipped) + tt |> equal(aot.message, "the emitter cannot handle this one") + tt |> equal(interp.status, FileStatus.ok, "and the interpreter still measured it") + f.skip = "broken everywhere" + tt |> equal(skip_reason(f, "interp"), "broken everywhere", "a whole-file skip answers for every lane") + } + t |> run("an unrunnable binary reports that nothing was measured, naming the cause") @(tt : T?) { + let res <- run_bench_file("/nonexistent/daslang", ".", bench_file(), "interp", RunLimits(timeout_seconds = 5)) + tt |> success(res.status == FileStatus.spawn_failed || res.status == FileStatus.exit_nonzero, "spawn failure or an immediate nonzero exit: {res.status}") + tt |> success(empty(res.samples), "nothing measured") + tt |> success(find(res.message, "before measuring anything") >= 0 || find(res.message, "cannot spawn") >= 0, + "the message says nothing ran rather than claiming the benchmarks passed: {res.message}") + } +} + +[test] +def test_process_rss_mb(t : T?) { + t |> run("a live child's resident set reads back as a positive number of megabytes") @(tt : T?) { + if (!fexist("/bin/sh")) { + tt |> skip("needs /bin/sh") + return + } + var mb = -1 + with_process(["/bin/sh", "-c", "sleep 5"]) $(var p) { + unsafe { + mb = process_rss_mb(process_pid(p)) + process_kill(p) + process_wait(p, 5.0) + } + } + tt |> success(mb > 0 && mb < 4096, "a shell's RSS is a small positive figure, got {mb} MB") + tt |> success(mb >= 1, "rounded up, so a process under a megabyte still reads as one") + } + t |> run("a pid nothing owns reads back as -1, which leaves the limit disarmed") @(tt : T?) { + tt |> equal(process_rss_mb(0), -1) + } +} + +[test] +def test_probe_aot(t : T?) { + if (!fexist("/bin/sh")) { + t |> skip("needs /bin/sh") + return + } + t |> run("a missing AOT binary names the target that builds it") @(tt : T?) { + let why = probe_aot("/nonexistent/test_aot") + tt |> success(find(why, "no AOT binary") >= 0, "it says what is absent: {why}") + tt |> success(find(why, "test_aot") >= 0, "and names the target that builds it") + } + t |> run("an AOT binary that runs the probe reports the lane available") @(tt : T?) { + let fake = fake_daslang("echo aot-probe-ok; exit 0") + let why = probe_aot(fake) + remove(fake) + tt |> equal(why, "", "an available lane has no reason") + } + t |> run("an AOT binary that prints its help instead of running is not available") @(tt : T?) { + // what the real binary does when handed a flag it does not take: usage text, exit 255 + let fake = fake_daslang("echo ' -log-compile-time log per-module breakdown'; exit 255") + let why = probe_aot(fake) + remove(fake) + tt |> success(find(why, "ran no program") >= 0, "the reason says no program ran: {why}") + tt |> success(find(why, "exit 255") >= 0, "and names the exit code") + } + t |> run("an AOT binary whose marker is missing is not available even on exit 0") @(tt : T?) { + let fake = fake_daslang("exit 0") + let why = probe_aot(fake) + remove(fake) + tt |> success(!empty(why), "silence is not availability: {why}") + } +} + +[test] +def test_probe_jit(t : T?) { + if (!fexist("/bin/sh")) { + t |> skip("needs /bin/sh") + return + } + t |> run("a binary that runs the probe reports the lane available") @(tt : T?) { + let fake = fake_daslang("echo jit-probe-ok; exit 0") + let why = probe_jit(fake) + remove(fake) + tt |> equal(why, "", "an available lane has no reason") + } + t |> run("a host whose only failure is its exit code still has a working lane") @(tt : T?) { + // what a RelWithDebInfo daslang does: the program runs, then the allocation tracker's + // exit-time report takes the process code to 1 + let fake = fake_daslang("echo jit-probe-ok; echo '=== daslang C++ heap leak report ==='; exit 1") + let why = probe_jit(fake) + remove(fake) + tt |> equal(why, "", "the marker printed, so the lane runs: {why}") + } + t |> run("a binary that fails the probe reports the reason it printed") @(tt : T?) { + let fake = fake_daslang("echo 'error[30812]: undefined function argument type NameLookup? const'; exit 1") + let why = probe_jit(fake) + remove(fake) + tt |> success(find(why, "exit 1") >= 0, "the reason names the exit code: {why}") + tt |> success(find(why, "error[30812]") >= 0, "and the compile error it printed, not some later line") + } + t |> run("a binary that exits 0 without the probe marker is not taken as available") @(tt : T?) { + let fake = fake_daslang("echo nothing to see; exit 0") + let why = probe_jit(fake) + remove(fake) + tt |> success(!empty(why), "silence is not availability: {why}") + tt |> success(find(why, "no jit-compiled run") >= 0, "and the reason says what was missing") + } +} diff --git a/utils/internal/bench-stand/test_bench_suite.das b/utils/internal/bench-stand/test_bench_suite.das new file mode 100644 index 0000000000..cbd7ae93f6 --- /dev/null +++ b/utils/internal/bench-stand/test_bench_suite.das @@ -0,0 +1,137 @@ +options gen2 + +require dastest/testing_boost public +require bench_suite +require daslib/fio +require strings + +def private with_temp_dir(prefix : string; blk : block<(dir : string) : void>) { + var err = "" + let dir = create_temp_directory(prefix, err) + verify(!empty(dir), "temp dir created") + invoke(blk, dir) + rmdir_rec(dir) +} + +def private touch(dir, rel : string) { + let p = path_join(dir, rel) + mkdir_rec(dir_name(p)) + fwrite(p, "options gen2\n") +} + +let private GOOD_CONFIG = "\{ \"root\": \"benchmarks\", \"exclude\": [\"**/tests/**\", \"**/_*.das\"], \"lanes\": [\"interp\"], \"repeat\": 2, \"timeout_seconds\": 30, \"memory_limit_mb\": 512, \"baseline_runs\": 5, \"files\": \{ \"core/slow.das\": \{ \"skip\": \"too slow\" \}, \"core/big.das\": \{ \"timeout_seconds\": 60, \"memory_limit_mb\": 2048 \} \} \}" + +[test] +def test_load_suite_config(t : T?) { + with_temp_dir("bench_stand_") $(dir) { + t |> run("a well-formed config parses with its overrides") @(tt : T?) { + let path = path_join(dir, "suite.json") + fwrite(path, GOOD_CONFIG) + var error = "" + let cfg <- load_suite_config(path, error) + tt |> equal(error, "") + tt |> equal(cfg.repeat, 2) + tt |> equal(cfg.timeout_seconds, 30) + tt |> equal(length(cfg.exclude), 2) + tt |> equal(length(cfg.lanes), 1) + cfg.files |> get("core/slow.das") $(ov) { + tt |> equal(ov.skip, "too slow") + } + cfg.files |> get("core/big.das") $(ov) { + tt |> equal(ov.timeout_seconds, 60) + } + tt |> equal(length(cfg.files), 2) + tt |> success(cfg.regression_threshold > 0.09lf && cfg.regression_threshold < 0.11lf, "threshold keeps its default") + } + t |> run("a missing config fails closed naming the path") @(tt : T?) { + var error = "" + load_suite_config(path_join(dir, "missing.json"), error) + tt |> success(find(error, "missing.json") >= 0, "error names the path") + } + t |> run("an unparseable config fails closed") @(tt : T?) { + let path = path_join(dir, "bad.json") + fwrite(path, "not json") + var error = "" + load_suite_config(path, error) + tt |> success(find(error, "parse") >= 0, "error says it did not parse") + } + t |> run("no lanes, a zero repeat, or an unknown lane are errors") @(tt : T?) { + let path = path_join(dir, "lanes.json") + fwrite(path, "\{ \"lanes\": [] \}") + var error = "" + load_suite_config(path, error) + tt |> success(find(error, "no lanes") >= 0, "empty lanes rejected") + fwrite(path, "\{ \"lanes\": [\"interp\"], \"repeat\": 0 \}") + load_suite_config(path, error) + tt |> success(find(error, "positive") >= 0, "zero repeat rejected") + fwrite(path, "\{ \"lanes\": [\"vulkan\"] \}") + load_suite_config(path, error) + tt |> success(find(error, "vulkan") >= 0, "unknown lane named") + fwrite(path, "\{ \"lanes\": [\"interp\", \"jit\", \"aot\"] \}") + load_suite_config(path, error) + tt |> equal(error, "", "the three real lanes are accepted") + fwrite(path, "\{ \"lanes\": [\"interp\"], \"baseline_runs\": 2 \}") + load_suite_config(path, error) + tt |> success(find(error, "baseline_runs") >= 0, "a baseline of two runs has no median worth gating on") + } + } +} + +def private load_cfg(dir : string) : SuiteConfig { + var error = "" + return <- load_suite_config(path_join(dir, "suite.json"), error) +} + +[test] +def test_discover_files(t : T?) { + with_temp_dir("bench_stand_") $(dir) { + fwrite(path_join(dir, "suite.json"), GOOD_CONFIG) + touch(dir, "benchmarks/core/hash/test02.das") + touch(dir, "benchmarks/core/hash/_common.das") + touch(dir, "benchmarks/core/slow.das") + touch(dir, "benchmarks/core/big.das") + touch(dir, "benchmarks/sql/tests/test_update.das") + touch(dir, "benchmarks/sort/sort.das") + touch(dir, "benchmarks/sort/README.md") + t |> run("helpers, test dirs and non-das files are excluded; the rest is sorted") @(tt : T?) { + let cfg <- load_cfg(dir) + var err = "" + let files <- discover_files(dir, cfg, err) + tt |> equal(err, "") + tt |> equal(length(files), 4) + tt |> equal(files[0].path, "core/big.das") + tt |> equal(files[1].path, "core/hash/test02.das") + tt |> equal(files[2].path, "core/slow.das") + tt |> equal(files[3].path, "sort/sort.das") + } + t |> run("group and id derive from the path") @(tt : T?) { + let cfg <- load_cfg(dir) + var err = "" + let files <- discover_files(dir, cfg, err) + tt |> equal(files[1].group, "core/hash") + tt |> equal(files[1].id, "core/hash/test02") + tt |> equal(files[3].group, "sort") + tt |> equal(group_of("top.das"), "root") + tt |> equal(stem_path("top.das"), "top") + } + t |> run("overrides apply and defaults fill the rest") @(tt : T?) { + let cfg <- load_cfg(dir) + var err = "" + let files <- discover_files(dir, cfg, err) + tt |> equal(files[2].skip, "too slow") + tt |> equal(files[2].timeout_seconds, 30) + tt |> equal(files[0].timeout_seconds, 60) + tt |> equal(files[0].memory_limit_mb, 2048) + tt |> equal(files[1].memory_limit_mb, 512) + tt |> equal(files[1].skip, "") + } + t |> run("an override naming a missing file is reported") @(tt : T?) { + let cfg <- load_cfg(dir) + remove(path_join(dir, "benchmarks/core/slow.das")) + var err = "" + let files <- discover_files(dir, cfg, err) + tt |> equal(length(files), 3) + tt |> success(find(err, "core/slow.das") >= 0, "error names the stale override") + } + } +}