diff --git a/.github/workflows/nightly_bench.yml b/.github/workflows/nightly_bench.yml new file mode 100644 index 0000000000..4c110c849e --- /dev/null +++ b/.github/workflows/nightly_bench.yml @@ -0,0 +1,149 @@ +name: nightly benchmark stand + +# Drives the benchmark stand on the web box (dasweb-1, the daslang.io origin) and +# publishes the night at https://daslang.io/bench/. The box does the work - checkout, +# Release build, every `[benchmark]` file in benchmarks/ in the interpreter and JIT +# lanes, the report - so the numbers come from ONE machine night after night; this +# runner only kicks it off over ssh, streams the log, and posts the summary. +# +# Red means the night is not a usable data point: the build failed, or at least one +# benchmark file failed (compile error, failed assertion, crash, timeout, memory cap). +# A regression is a warning in the step summary and on the site, not a red - flip +# `fail_on_regression` on a dispatch to make it one. +# +# No self-hosted runner: this repository is public, and a self-hosted runner would +# execute pull-request code on the production web box. The key reaches only the +# stand's gate verbs - start, follow, status, summary - never a shell (a forced +# command in the box's authorized_keys). + +on: + schedule: + # 05:00 UTC - after nightly_playground's 04:30, while the box is otherwise idle. + - cron: '0 5 * * *' + workflow_dispatch: + inputs: + ref: + description: 'commit or branch to benchmark (default: master)' + required: false + default: 'master' + fail_on_regression: + description: 'turn a regression into a red run' + required: false + default: 'false' + +defaults: + run: + shell: bash + +# One night at a time: the box holds a run lock, so a second start would only be refused. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +# Empty on purpose: every step of the one job talks to the box over ssh. Nothing checks the +# repository out and nothing calls the GitHub API, so the token needs no scope at all. +permissions: {} + +jobs: + bench: + # The cron only runs on the canonical repo: it drives daslang.io's box, which a fork + # neither owns nor holds the key to, and a red nightly emails the fork owner. + if: github.event_name != 'schedule' || github.repository == 'GaijinEntertainment/daScript' + runs-on: ubuntu-latest + # Build (~20-40 min on the box) + two lanes of the suite (~1-2 h) + report. + timeout-minutes: 360 + # SECRET SCOPE, like pages.yml: DASWEB_BENCH_KEY is an environment secret on + # `github-pages` (the repository has no repo-level secrets). + environment: + name: github-pages + env: + BOX: 89.167.63.131 + BOX_USER: bench + steps: + - name: "Set up the ssh key" + id: key + env: + BENCH_KEY: ${{ secrets.DASWEB_BENCH_KEY }} + run: | + set -euo pipefail + # A fork has no access to the secret and legitimately skips, the way pages.yml's deploy + # step does; on the canonical repo an empty key means the nightly silently stopped + # measuring, so it is loud there. + if [ -z "$BENCH_KEY" ]; then + if [ "$GITHUB_REPOSITORY" = "GaijinEntertainment/daScript" ]; then + echo "::error::DASWEB_BENCH_KEY is empty on the canonical repo - the stand was not driven." + echo "It is an ENVIRONMENT secret on 'github-pages'; check the job still declares that environment." + exit 1 + fi + echo "DASWEB_BENCH_KEY not set (fork) - skipping the run" + echo "skipped=1" >> "$GITHUB_OUTPUT" + exit 0 + fi + mkdir -p ~/.ssh + printf '%s\n' "$BENCH_KEY" > ~/.ssh/bench_key + chmod 600 ~/.ssh/bench_key + # Pinned host key - the same box and key pages.yml deploys to; no trust-on-first-use. + echo "$BOX ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILP2wzOeEGD2r7apldQlTe2gwR1dVTCXCfrD+pJooMmY" >> ~/.ssh/known_hosts + echo "RSH=ssh -i $HOME/.ssh/bench_key -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes -o ServerAliveInterval=30 -o ConnectTimeout=30" >> "$GITHUB_ENV" + + - name: "Start the night on the box" + id: start + if: steps.key.outputs.skipped != '1' + env: + REF: ${{ github.event.inputs.ref || 'master' }} + FAIL_ON_REGRESSION: ${{ github.event.inputs.fail_on_regression || 'false' }} + run: | + set -euo pipefail + # The forced command takes `start [fail-on-regression]`, and that second word is + # the ONLY way the input reaches the box: a forced command carries no environment, so + # exporting BENCH_STAND_FAIL_ON_REGRESSION here would reach nothing. A busy box answers + # exit 3 and names the run it is still working on. + FLAG="" + [ "$FAIL_ON_REGRESSION" = "true" ] && FLAG=" fail-on-regression" + RUN_ID=$($RSH "$BOX_USER@$BOX" "start $REF$FLAG") + echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" + echo "started run $RUN_ID for $REF (regression fails the night: $FAIL_ON_REGRESSION)" + + - name: "Follow the run" + if: steps.key.outputs.skipped != '1' + run: | + set -uo pipefail + # An ssh drop (exit 255) reconnects rather than failing the lane; every other code is + # the run's own, recorded for the summary step to raise. + for attempt in $(seq 1 60); do + $RSH "$BOX_USER@$BOX" "follow ${{ steps.start.outputs.run_id }}" + rc=$? + if [ $rc -ne 255 ]; then + echo "run finished with exit $rc" + echo "$rc" > run_exit + exit 0 + fi + echo "ssh dropped (attempt $attempt) - reconnecting" + sleep 30 + done + echo "::error::could not stay connected to the box" + echo 255 > run_exit + + - name: "Post the summary" + # always(), so a night that ended badly still publishes its summary here. The step's own + # exit code is the lane's verdict: the run's code when there is one, and a red when the + # summary could not be fetched at all - an unfetched summary is the one artifact this + # lane exists to deliver, so swallowing it would leave a green lane saying nothing. + if: always() && steps.key.outputs.skipped != '1' + run: | + set -uo pipefail + summary_rc=0 + $RSH "$BOX_USER@$BOX" "summary" > summary.md || summary_rc=$? + if [ $summary_rc -ne 0 ] || [ ! -s summary.md ]; then + echo "::error::could not fetch the night's summary from the box (ssh exit $summary_rc)" + echo "The night's own verdict, if it reached one, is below." >> "$GITHUB_STEP_SUMMARY" + fi + [ -s summary.md ] && cat summary.md >> "$GITHUB_STEP_SUMMARY" + rc=$(cat run_exit 2>/dev/null || echo 1) + case "$rc" in + 0) echo "night ok" ;; + 2) echo "::error::the night regressed and fail_on_regression is on - see the summary" ;; + *) echo "::error::the night failed (exit $rc) - see the summary and https://daslang.io/bench/" ;; + esac + [ "$rc" = 0 ] && exit "$summary_rc" + exit "$rc" diff --git a/CMakeLists.txt b/CMakeLists.txt index 44a817832d..8c88d11b6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -178,8 +178,17 @@ MACRO(DAS_AOT_EXT input_files genList mainTarget dasAotTool dasAotToolArg) set(all_depends ${dasAotTool} ${PROJECT_SOURCE_DIR}/utils/aot/main.das ${DAS_AOT_DASLIB_DEPENDS} ${DAS_AOT_EXTRA_DEPENDS}) set(all_outputs "") # Per-batch arg buffer; chunked to keep each invocation under Windows' - # 32K command-line limit when worktree paths are long. - set(_DAS_AOT_BATCH_SIZE 60) + # 32K command-line limit when worktree paths are long. A caller may set + # DAS_AOT_BATCH_SIZE to something smaller: one tool run compiles every file it + # is given in one process, so a file whose require graph installs macros can + # perturb the ones after it in its batch (error[20600] / error[30100] on an + # annotation that exists). A batch of 1 costs a process per file and makes that + # impossible. + if(DEFINED DAS_AOT_BATCH_SIZE) + set(_DAS_AOT_BATCH_SIZE ${DAS_AOT_BATCH_SIZE}) + else() + set(_DAS_AOT_BATCH_SIZE 60) + endif() set(_batches "") set(_batch_idx 0) set(_batch_count 0) diff --git a/benchmarks/decs/bench_from_decs_count.das b/benchmarks/decs/bench_from_decs_count.das index 978d96ea50..3157206b62 100644 --- a/benchmarks/decs/bench_from_decs_count.das +++ b/benchmarks/decs/bench_from_decs_count.das @@ -20,7 +20,7 @@ struct BenchCountRow { def fixture(n : int) { restart() - create_entities(n) $(eid : EntityId; i : int; var cmp : ComponentMap) { + create_entities(n) $(_eid : EntityId; i : int; var cmp : ComponentMap) { apply_decs_template(cmp, BenchCountRow(val = i)) } } @@ -32,7 +32,7 @@ def from_decs_count_m1_hand(b : B?) { b |> run("m1_hand_arch_size/{N}", N) { var erq : EcsRequest erq.req |> push("bench_val") - var total = 0 + var total = 0l for_each_archetype(erq) $(arch : Archetype) { total += arch.size } diff --git a/skills/internal/preflight.md b/skills/internal/preflight.md index 504e5fadde..45dada07a5 100644 --- a/skills/internal/preflight.md +++ b/skills/internal/preflight.md @@ -62,6 +62,7 @@ working-tree copy. |---|---|---| | `build.yml` (per-PR) | every PR commit (`pull_request`) + pushes to `master` | `build` matrix (`ci/ci_matrix.py build`: Debug + Release on linux, linux_arm, darwin15, darwin26; windows 32 Release, windows 64 Release), `bundle_smoke`, `build_linux_gcc` | | `build.yml` (nightly) | `schedule` cron (daily 02:00 UTC) | `build_windows_mingw` + `build_windows_clangcl` (gated OFF per-PR) **plus the full build matrix - the per-PR cells, the sanitizer cells (linux Release asan/tsan/ubsan), the fast-math cell (linux Release `-DDAS_FAST_MATH=ON`) and windows 64 Debug - whose Release cells run the full AOT sweep** ("Slow Release Tests"). Breaks surface within ~24 h, not at PR time | +| `nightly_bench.yml` | `schedule` cron (daily 05:00 UTC) + `workflow_dispatch` (`ref`, `fail_on_regression`) | one job that drives the benchmark stand on the daslang.io box over ssh (`utils/internal/bench-stand/nightly.sh`): Release build of `ref` plus `test_aot`, then every `benchmarks/**/*.das` its `suite.json` does not exclude or skip, in the interp, jit and aot lanes, the report to https://daslang.io/bench/; red = build or any benchmark file failed. Local mirror: `bin/daslang utils/internal/bench-stand/main.das -- run --meta --out --filter ` then `... -- report --runs --out-data --out-summary ` (README there) | | `nightly_imgui.yml` | `schedule` cron (daily 03:00 UTC) + `workflow_dispatch` | dasImgui playwright suite on ubuntu + macos - section below | | `extended_checks.yml` (per-PR) | every PR | two darwin15-arm64 jobs, `core` and `modules` (`ci/ci_matrix.py extended`), ALL release modules ON - section below | | `extended_checks.yml` (nightly) | `schedule` cron (daily 04:00 UTC) + `workflow_dispatch` | one job each on linux, darwin15 and windows running every step (role `all`), including the ones too slow for a PR: tutorial dry-runs, the run form of examples, coverage, the nano cross-compile, the AST verify tree sweep, doc-verify | diff --git a/tests/aot/CMakeLists.txt b/tests/aot/CMakeLists.txt index cbd2c5f89e..6693887026 100644 --- a/tests/aot/CMakeLists.txt +++ b/tests/aot/CMakeLists.txt @@ -40,6 +40,26 @@ FILE(GLOB AOT_TESTS_FILES RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS "test # Exclude _-prefixed helper modules (required transitively, not standalone test entry points) list(FILTER AOT_TESTS_FILES EXCLUDE REGEX "/_") +# The nightly benchmark stand's AOT lane (utils/internal/bench-stand): the benchmark bodies get +# their stubs linked into test_aot, so `test_aot -use-aot dastest/dastest.das -- --use-aot --bench` +# measures native code instead of the interpreter. Predefined rather than globbed by the loop +# below, because the tree is nested and the loop's convention is tests//*.das. +FILE(GLOB_RECURSE AOT_BENCHMARKS_FILES RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS "benchmarks/*.das") +# _-prefixed helper modules are required transitively, never standalone; benchmarks/sql/tests is +# a dastest suite, not a benchmark. +list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "/_") +list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "^benchmarks/sql/tests/") +# Suites whose require graph reaches a native module: each would need that module's own AOT half +# linked here, so they measure in the interpreter and JIT lanes only. suite.json states the +# reason per file, and widening this set is what lets a lane skip go away. +list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "^benchmarks/(audio|terminal|sql)/") +list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "benchmarks/core/math/scalar_crt\\.das$") +list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "benchmarks/micro/(join_select|single_last|sort_distinct_take)_shapes\\.das$") +# benchmarks/core/jit_aot rides the language suite on purpose (see AOT_JIT_BENCH_FILES below), so +# its stubs are already in both AOT binaries. Registering it twice emits two TUs for one file and +# the anonymous-module namespaces collide at link (multiple definition of das::_anon_). +list(FILTER AOT_BENCHMARKS_FILES EXCLUDE REGEX "^benchmarks/core/jit_aot/") + FILE(GLOB AOT_BARE_BLOCK_FILES RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS "tests/bare_block/*.das") list(FILTER AOT_BARE_BLOCK_FILES EXCLUDE REGEX "failed_|cant_|invalid_") @@ -394,7 +414,7 @@ set(DAS_AOT_SUITES jobque json jsonrpc language linq lint long_array_table loops lpipe lsp macro_boost macro_call match math mcp md_boost module_cache module_tests option promote quote reader_macro regex rtti safe_addr soa spoof strings stbimage table_packed template - tests type_lattice type_traits typemacro uri with_boost delegate) + tests type_lattice type_traits typemacro uri with_boost delegate benchmarks) foreach(_s IN LISTS DAS_AOT_SUITES) string(TOUPPER ${_s} _u) # suites with an irregular dir / a filter / a curated list define AOT__FILES above; @@ -405,9 +425,18 @@ foreach(_s IN LISTS DAS_AOT_SUITES) add_custom_target(test_aot_${_s}) set(${_u}_AOT_GENERATED_SRC) list(LENGTH AOT_${_u}_FILES _nf) + # One process per benchmark: ten of the core/hash bodies install macros that make a LATER + # file in the same batch fail on `[benchmark]` (error[20600]). Batching is a speed knob, so + # the suite that trips it pays a process per file instead of losing the files. + if(_s STREQUAL "benchmarks") + set(DAS_AOT_BATCH_SIZE 1) + else() + unset(DAS_AOT_BATCH_SIZE) + endif() if(_nf GREATER 0) DAS_AOT("${AOT_${_u}_FILES}" ${_u}_AOT_GENERATED_SRC test_aot_${_s} daslang) endif() + unset(DAS_AOT_BATCH_SIZE) list(APPEND TEST_AOT_TARGETS test_aot_${_s}) list(APPEND TEST_AOT_GENVARS ${_u}_AOT_GENERATED_SRC) list(APPEND TEST_AOT_ALL_DAS ${AOT_${_u}_FILES}) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 6534ff120f..46c9a722ba 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -223,6 +223,7 @@ SET(DAS_UTILS_TO_TEST internal/hygiene internal/preflight/tests internal/ast-fuzz + internal/bench-stand internal/requirefix internal/test-release daspkg/test_daspkg.das diff --git a/utils/internal/bench-stand/README.md b/utils/internal/bench-stand/README.md new file mode 100644 index 0000000000..84e78f79ed --- /dev/null +++ b/utils/internal/bench-stand/README.md @@ -0,0 +1,206 @@ +# bench-stand - the nightly benchmark stand + +Every night one box checks out master, builds it, runs every `[benchmark]` file under +`benchmarks/` in the interpreter and JIT lanes, and publishes the result at +https://daslang.io/bench/ - a chart per benchmark arm over commits, a per-group index, and +a plain statement of what failed and what moved. The GitHub workflow +`.github/workflows/nightly_bench.yml` is the trigger and the red/green; the box is the +measuring instrument. Review rules: `REVIEW.md`. + +## 1. Layout + +- `main.das` - the tool: `run` (benchmark the tree into one run record) and `report` (every + record into `data.json` + `summary.md`, with the exit code the workflow keys on). +- `bench_suite.das` - `suite.json` loading and file discovery: which files, in which group, + under which limits. Pure; no processes. +- `bench_runner.das` - one file in one lane: spawn `dastest --bench`, enforce the wall-clock + and memory limits, turn its output into samples and a verdict. The parser is pure and + tested on captured output shapes. +- `bench_history.das` - records to dataset: series, the group index, regression verdicts, + the markdown summary. Pure. +- `suite.json` - the configuration (section 4). +- `site/` - the viewer: `index.html`, `app.js`, `style.css`. Static, no build step, no + dependencies; reads `data.json` and `status.json` beside it. +- `nightly.sh` - the box-side driver (section 3); `bench-stand-deploy.sh` provisions the box; + `caddy.snippet` is the public route, and the only place a route is written down. +- `REVIEW.md` + `REVIEW.das` - the review checklist and its mechanical half. The gate checks the + key parity of `suite.json` / `SuiteConfig` / section 4 below, that a `skip` states a reason, + that a sibling module is required by bare name, that the viewer writes no markup, that every + `BENCH_STAND_*` knob is documented in both places, that no route literal is copied into + `bench-stand-deploy.sh`, and that every file here has its placement line. +- `test_bench_*.das` - the three dastest suites; `run_utils_tests` (per PR) runs them. + +## 2. Data model + +Benchmark identity: `#/` per lane, where the file id is the path under +`benchmarks/` without `.das` (`core/hash/test02#builtin_table/insert/600000`, lane `interp`). +The group is the file's directory (`core/hash`, `sort`, `sql`); the viewer sections and the +index follow it, so a new benchmark joins its group by living in the right folder. + +`runs/.json` - one per night, `RunRecord` in `bench_history.das`. The run id is +`-`. It carries the commit, the machine (host, CPU, cores, memory, kernel, +compiler, load at start), the build (status, seconds, log tail), the lane states, and one +`FileResult` per file per lane: status, exit code, seconds, message, the last 60 log lines, +and the samples that completed. A sample is one arm aggregated over the night's repeats: +`ns` = the minimum ns/op (the noise-robust statistic), `ns_median`, `spread` = +(max - min) / min, and per-op allocation medians. + +A record's own `status` is `ok`, `build_failed` or `bench_failed`, and `load_runs` re-derives it +from the record's parts on every read, so a hand-edited status cannot mislead the report. + +File statuses: `ok`, `skipped` (listed in `suite.json` with a reason), `compile_error`, +`failed` (a `[benchmark]` function failed an assertion or panicked - its arms are dropped, the +other functions' arms kept), `timeout`, `memory` (killed at the RSS cap), `exit_nonzero` (the +process died without a finished dastest report, so the suite was cut short), `spawn_failed`. + +A file whose dastest report is finished and whose arms are all there stays `ok` even when the +process then exits non-zero: the measurement stands and only the shutdown went wrong, so the +message records it (`measured, then exited with code 1 at shutdown`). Keying the verdict on the +exit code instead would fail every file on a RelWithDebInfo host, where the C++ allocation +tracker's exit-time report takes every process to 1. + +The JIT lane is probed once per night with a trivial program, and the probe believes the marker +that program prints, not the process exit code - for the same reason. + +## 2.3 The AOT lane + +AOT is not a flag on `daslang`: the benchmark bodies are compiled to C++ at build time and linked +into a binary. That binary is `test_aot`, which the benchmark files join through the +`benchmarks` row of `DAS_AOT_SUITES` in `tests/aot/CMakeLists.txt` (repo root); `--aot-bin` names +it and the night builds `--target test_aot` before measuring. The lane's argv carries **both** +`-use-aot` on the host and `--use-aot` on dastest. The second one is what arms `fail_on_no_aot`: +without it every function silently interprets and the lane reports interpreter numbers under an +AOT label, and with it a function missing its stub is `error[50101]`, so a run that finishes is +proof the stubs were live. + +Do not read the tier from dastest's own label: `is_in_aot()` is true only while the AOT emitter +runs, so an AOT run still prints `[INTERP]`. The lane name is the record of what ran. + +A benchmark whose require graph reaches a native module needs that module's own AOT half linked +into the same binary. Those suites are excluded from the AOT set in `tests/aot/CMakeLists.txt` +and carry their reason under `lane_excludes` in `suite.json`; widening the first is what lets the +second go away. + +`site/data.json` - `Dataset`: run summaries, groups, series (columnar: `runs` the run index into +`Dataset.runs`, `ns` the minimum, `spread` the night's spread), the changes for the latest run +(`regression`, `improvement`, `first_seen`, `missing`), the group index (`runs` and `value`). `site/summary.md` is the +same night in markdown; the workflow posts it as its step summary. `site/status.json` is +written by `nightly.sh` at start and end (`running` / `finished`, run id, exit), so a night +whose build failed before any binary existed is still visible. + +### 2.1 Statistics + +A change verdict compares tonight's value with the median of the series' previous +`baseline_runs` points (at least 3). It is a regression when the relative move exceeds both +`regression_threshold` and `noise_multiplier` times the baseline's relative median absolute +deviation; an improvement when it falls by the same gate. A series with no earlier point is +`new`; one with a baseline but no point tonight while its file ran ok is `missing` - an arm +was renamed or removed. The group index is the geometric mean over the group's series of +value / reference, times 100, where each series' reference is the median of its first five +points; a run contributes when at least half the group's series measured that night. + +### 2.2 Exit codes + +`run`: 0 when every file ran ok, 1 otherwise. `report`: 0 when the latest night is ok, 1 when it +failed (build or any file), 2 when it is ok but regressed and `--fail-on-regression` was +given. `nightly.sh run` exits with the run's code, or the report's when the run was clean. + +`run --failed ""` benchmarks nothing: it writes the record of a night that could not run +the suite and exits 1. That is how the driver records a failed build or its own failure without a +second writer of the run-record schema; a build that failed carries no file, since its log tail +already rides the record's `build`. + +## 3. The box + +`dasweb-1` (the daslang.io origin) runs the stand as the unprivileged `bench` user under +`/srv/bench-stand`: `src/` (the clone each night checks out), `runs/`, `site/` (what Caddy +serves at `/bench/`), `logs/`, `last_good/` (the last passing build's binary, `daslib/` and +this tool - renders the report when tonight's build fails, so a broken master publishes a red +night instead of a stale page), `state/` (run lock, current run id). + +The workflow reaches the box over ssh as `bench` with a key whose `authorized_keys` line +forces `nightly.sh gate`: only `start [fail-on-regression]`, `follow [run_id]`, `status` +and `summary` exist on that connection, at most two arguments each and only path characters in +them, no pty, no forwarding. The gate reads the caller's words from `SSH_ORIGINAL_COMMAND`, which +is also why `start`'s second word is the only way the workflow can ask for a regression to fail +the night - a forced command carries no environment. `start` +resolves the ref, launches the night detached and prints the run id; `follow` streams the log +and exits with the run's code once `status.json` says finished, so a dropped ssh session +reconnects instead of failing the lane. There is deliberately no self-hosted runner: the +repository is public, and one would execute pull-request code on the production web box. + +The night builds Release (`-DDAS_SQLITE_DISABLED=OFF -DDAS_PUGIXML_DISABLED=OFF +-DDAS_LLVM_DISABLED=OFF`, GLFW and HV off), the targets the benchmarks need, and `test_aot` for +the AOT lane - that last one is ~1080 translation units and the bulk of the night's build. Release +matters: RelWithDebInfo arms the C++ allocation tracker, whose exit-time leak report turns +every clean process into exit 1. The JIT lane is probed once per night (`daslang -jit` on a +trivial program); when the probe fails the lane is recorded as unavailable with the reason and +the interpreter lane still runs. + +One-time setup, as root on the box, with the public half of a fresh ed25519 key: + +```sh +sudo install -m 755 utils/internal/bench-stand/bench-stand-deploy.sh /usr/local/sbin/ +sudo bench-stand-deploy.sh provision ./bench_key.pub +sudo bench-stand-deploy.sh caddy +``` + +Then store the private half as the `DASWEB_BENCH_KEY` environment secret on `github-pages` +and dispatch `nightly_bench.yml` once. `bench-stand-deploy.sh status` shows the layout, the +last `status.json` and whether Caddy carries every route the snippet declares (it derives that +list from `caddy.snippet`, so a route added there is checked without editing the script; a +vhost holding some routes and not others is reported and never repaired automatically). The +deploy script installs the packages the build needs (git, cmake, ninja, g++, ccache, rsync) - +ccache is what keeps the nightly rebuild at minutes rather than the cold 20-40. + +Environment knobs `nightly.sh` reads, all optional: `BENCH_STAND_HOME` (layout root), +`BENCH_STAND_REPO_URL` (commit-link base), `BENCH_STAND_SITE_URL` (what the summary links), +`BENCH_STAND_JOBS` (build parallelism; the default counts the box's CPUs through `nproc`, `sysctl hw.ncpu`, or 4), `BENCH_STAND_CMAKE_ARGS` (extra configure arguments), +`BENCH_STAND_LANES` / `BENCH_STAND_REPEAT` / `BENCH_STAND_FAIL_ON_REGRESSION` (forwarded to +the tool), `BENCH_STAND_AOT=skip` (leave the AOT binary +unbuilt, so the aot lane reports itself unavailable and the other two still measure), and two for +local dry runs only: `BENCH_STAND_BUILD=skip` (reuse `src/bin/daslang`) and `BENCH_STAND_FILTER` +(a path substring). + +## 4. Configuration - `suite.json` + +`root` (the benchmark tree, `benchmarks`), `exclude` (globs over the path under root; helper +modules `_*.das` and `**/tests/**` by default), `lanes` (`interp`, `jit`, `aot`), +`lane_excludes` (per lane, a glob to the reason those files do not run in it - a file the AOT +binary cannot carry still measures in the other two, and the reason shows on the site every +night), `repeat` (dastest `--count`; the +minimum over repeats is the recorded value), `timeout_seconds` and `memory_limit_mb` per file +(the runner kills the child and records `timeout` / `memory`), `regression_threshold`, +`noise_multiplier`, `baseline_runs` (section 2.1), and `files` - per-file overrides of the +limits or a `skip` with its reason. A skipped file is listed on the site and in the summary +every night, so a skip is visible debt, not a deletion; an override naming a file that no +longer exists is an error, so a renamed benchmark cannot silently lose its skip. + +## 5. Running locally + +```sh +# one night of a slice of the suite, both lanes, into a scratch record +bin/daslang utils/internal/bench-stand/main.das -- run --meta meta.json --out /tmp/stand/runs/n1.json --filter core/math/ --repeat 2 +# the report over every record, then open site/ over any static server +bin/daslang utils/internal/bench-stand/main.das -- report --runs /tmp/stand/runs --out-data /tmp/stand/site/data.json --out-summary /tmp/stand/site/summary.md +cp utils/internal/bench-stand/site/* /tmp/stand/site/ && ln -sfn ../runs /tmp/stand/site/runs +bin/daslang dastest/dastest.das -- --test utils/internal/bench-stand +``` + +`meta.json` is what `nightly.sh` writes: `{"run_id", "started", "commit": {"sha", "date", +"subject", "author"}, "machine": {...}, "build": {"status", "seconds", "log_tail"}}`; any +subset parses. The whole box pipeline runs locally too: point `BENCH_STAND_HOME` at a scratch +layout whose `src/` is a worktree with a built `bin/daslang`, and run +`BENCH_STAND_BUILD=skip BENCH_STAND_FILTER=core/math/ nightly.sh run HEAD`. + +## 6. Reading a red night + +A night whose build failed says so first: the summary leads with the build's own section and the +last lines of its log, and stops there - nothing was measured, so there is nothing else to read. +Otherwise the step summary (and `summary.md`) leads with the status and the commit, then a failures +table - file, lane, status, message - where the message is the thing to act on: the first +`error[...]` line of a compile error, the failed function names, the kill reason with the knob +that set the limit, or the `FATAL` line of a non-zero exit. The run record has the last 60 log +lines of every failed file. A `build_failed` night's record carries the build log tail. A +regression row names the arm, both values, the change and the noise that the gate was +measured against; the chart behind it is one click from the site's latest-night panel. diff --git a/utils/internal/bench-stand/REVIEW.das b/utils/internal/bench-stand/REVIEW.das new file mode 100644 index 0000000000..58c9d04745 --- /dev/null +++ b/utils/internal/bench-stand/REVIEW.das @@ -0,0 +1,254 @@ +options gen2 +options indenting = 4 + +require strings +require daslib/strings_boost +require daslib/fio +require daslib/json_boost +require math +require dastest/review_gate + +// The mechanical half of utils/internal/bench-stand/REVIEW.md (contract: REVIEW_COMMON.md at +// the repo root). Run from the repo root: bin/daslang utils/internal/bench-stand/REVIEW.das - +// exit 0 clean, 1 with findings. + +let TOOL = "utils/internal/bench-stand" + +def private tool_path(rel : string) : string { + return path_join(TOOL, rel) +} + +// Every committed file of the tool. A dot-prefixed path is build or cache output the tree +// ignores (`.jitted_scripts/`), never something a rule reaches. +def private files_here() : array { + var out : array + dir_rec(TOOL) $(name, is_dir) { + return if (is_dir) + let rel = to_generic_path(name) + return if (starts_with(rel, ".") || find(rel, "/.") >= 0) + out |> push(rel) + } + out |> sort + return <- out +} + +// Every `.das` here is one module family: a sibling is required by its bare module name, never +// by a path, so the files stay movable as a set. +def private check_sibling_requires() { + let names = files_here() + var modules : table + for (rel in names) { + if (extension(rel) == ".das" && !starts_with(base_name(rel), "test_") && base_name(rel) != "REVIEW.das") { + modules |> insert(stem(rel)) + } + } + for (rel in names) { + continue if (extension(rel) != ".das") + let path = tool_path(rel) + for (req in das_requires(path)) { + let leaf = stem(base_name(req)) + continue if (req == leaf || !key_exists(modules, leaf)) + gate_finding(path, find_line(fread(path), req), + "requires the sibling module '{leaf}' as '{req}' - spell a sibling by its bare module name") + } + } +} + +// suite.json, SuiteConfig and README section 4 are three spellings of one key set; a key that +// loses any of the three is a knob nobody can set, type or find. +def private check_suite_keys() { + let suite_path = tool_path("suite.json") + let config_path = tool_path("bench_suite.das") + let readme_path = tool_path("README.md") + var error = "" + var root = read_json(fread(suite_path), error) + if (root == null || !empty(error)) { + gate_finding(suite_path, "does not parse as JSON: {error}") + return + } + let config_text = fread(config_path) + let struct_at = find(config_text, "struct SuiteConfig \{") + if (struct_at < 0) { + gate_finding(config_path, "no `struct SuiteConfig \{` - the suite config's field list is what suite.json is checked against") + return + } + let struct_end = find(config_text, "\n\}", struct_at) + let struct_body = slice(config_text, struct_at, struct_end < 0 ? length(config_text) : struct_end) + let readme_section = readme_section_of(readme_path, "## 4.") + if (empty(readme_section)) { + gate_finding(readme_path, "no `## 4.` section - the suite keys are documented there") + } + var json_keys : table + if (!(root.value is _object)) { + gate_finding(suite_path, "is not a JSON object") + return + } + unsafe { + assume obj = root.value as _object + for (key in keys(obj)) { + json_keys |> insert(key) + if (find(struct_body, "\n {key} ") < 0) { + gate_finding(suite_path, "key '{key}' is not a SuiteConfig field in bench_suite.das") + } + if (!empty(readme_section) && find(readme_section, key) < 0) { + gate_finding(readme_path, "section 4 does not mention the suite key '{key}'") + } + } + } + for (line in split(struct_body, "\n")) { + let trimmed = strip(line) + continue if (empty(trimmed) || starts_with(trimmed, "//") || starts_with(trimmed, "struct")) + let colon = find(trimmed, " :") + continue if (colon <= 0) + let field = slice(trimmed, 0, colon) + continue if (find(field, " ") >= 0) + if (!key_exists(json_keys, field)) { + gate_finding(suite_path, "SuiteConfig field '{field}' has no key here - a knob with no line in suite.json is one nobody finds") + } + } +} + +// A skipped benchmark is debt the site shows every night; a skip with no reason hides what it +// is standing in for. +def private check_skip_reasons() { + let suite_path = tool_path("suite.json") + var error = "" + var root = read_json(fread(suite_path), error) + return if (root == null) + var files = root?["files"] + return if (files == null || !(files.value is _object)) + unsafe { + assume entries = files.value as _object + for (name in keys(entries)) { + var over = files?[name] + continue if (over == null) + let skip = over?["skip"] + // a key the entry does not carry comes back as a non-null _null node, never as a + // pointer that tests false - the distinction is what keeps a plain override quiet + continue if (skip == null || skip.value is _null) + if (empty(strip(over?["skip"] ?? ""))) { + gate_finding(suite_path, "the skip of '{name}' states no reason - name the failure it hides") + } + } + } +} + +// The viewer renders series names, commit subjects and failure messages that come from a run +// record; every one of them reaches the DOM as text, never as markup. +def private check_viewer_no_markup() { + for (rel in files_here()) { + continue if (dir_name(rel) != "site" || extension(rel) != ".js") + let path = tool_path(rel) + let text = strip_line_comments(fread(path)) + for (needle in ["innerHTML", "outerHTML", "insertAdjacentHTML", "document.write"]) { + let at = find(text, needle) + continue if (at < 0) + gate_finding(path, find_line(fread(path), needle), + "writes markup with {needle} - a label from a run record reaches the DOM through textContent") + } + } +} + +// nightly.sh's knobs are invisible: one list in its own header, one in README section 3. A knob +// missing from either is one an operator cannot discover, and a documented knob nothing reads is +// one they would set in vain. +def private check_env_knobs() { + let script_path = tool_path("nightly.sh") + let readme_path = tool_path("README.md") + let text = fread(script_path) + let header = slice(text, 0, max(find(text, "\nset -"), 0)) + let body = slice(text, max(find(text, "\nset -"), 0)) + let readme_section = readme_section_of(readme_path, "## 3.") + if (empty(readme_section)) { + gate_finding(readme_path, "no `## 3.` section - nightly.sh's knobs are documented there") + } + var read_names : table + for (name in keys(prefixed_tokens(body, "BENCH_STAND_"))) { + read_names |> insert(name) + if (find(header, name) < 0) { + gate_finding(script_path, "reads {name} without naming it in the script's own header") + } + if (!empty(readme_section) && find(readme_section, name) < 0) { + gate_finding(readme_path, "section 3 does not document {name}, which nightly.sh reads") + } + } + for (name in keys(prefixed_tokens(header, "BENCH_STAND_"))) { + if (!key_exists(read_names, name)) { + gate_finding(script_path, "the header documents {name}, which the script never reads") + } + } +} + +// caddy.snippet is the one place the stand's public routes are written down; the deploy script +// derives its checks from it, so a route literal here is a second copy that can go stale. +def private check_no_route_literals() { + let path = tool_path("bench-stand-deploy.sh") + var in_extractor = false + var lineno = 0 + for (raw in split(fread(path), "\n")) { + lineno++ + let line = strip(raw) + // snippet_routes is the extractor: the directive names ARE its subject, so its body is + // the one place in the script licensed to spell them. + if (starts_with(line, "snippet_routes() \{")) { + in_extractor = true + continue + } + if (in_extractor) { + in_extractor = line != "\}" + continue + } + continue if (empty(line) || starts_with(line, "#")) + let code = find(line, " # ") >= 0 ? slice(line, 0, find(line, " # ")) : line + for (needle in ["handle_path", "redir ", "file_server", "reverse_proxy"]) { + continue if (find(code, needle) < 0) + gate_finding(path, lineno, + "spells the route directive '{strip(needle)}' - read the routes from caddy.snippet instead (snippet_routes)") + } + } +} + +// The placement block is the folder's inventory. A file with no line is a file no rule reaches. +def private check_placement_inventory() { + let review_path = tool_path("REVIEW.md") + let text = fread(review_path) + let block_at = find(text, "**Placement - one file, one line") + if (block_at < 0) { + gate_finding(review_path, "no placement block - the folder's file inventory lives there") + return + } + let inventory = slice(text, block_at) + for (rel in files_here()) { + let leaf = base_name(rel) + continue if (leaf == "REVIEW.md" || leaf == "README.md") + let listed = dir_name(rel) == "site" ? find(inventory, "`site/`") >= 0 : find(inventory, "`{leaf}`") >= 0 + if (!listed) { + gate_finding(review_path, "the placement block has no line for {rel}") + } + } +} + +// A `## N.` section of a markdown file, up to the next same-level heading. +def private readme_section_of(path : string; heading : string) : string { + let text = fread(path) + let at = find(text, heading) + return "" if (at < 0) + let next = find(text, "\n## ", at + 1) + return next < 0 ? slice(text, at) : slice(text, at, next) +} + +[export] +def main() : int { + if (!stat(TOOL).is_dir) { + print("run from the repo root: {TOOL} not found\n") + return 1 + } + check_sibling_requires() + check_suite_keys() + check_skip_reasons() + check_viewer_no_markup() + check_env_knobs() + check_no_route_literals() + check_placement_inventory() + return gate_verdict("bench-stand") +} diff --git a/utils/internal/bench-stand/REVIEW.md b/utils/internal/bench-stand/REVIEW.md new file mode 100644 index 0000000000..5755dbdeae --- /dev/null +++ b/utils/internal/bench-stand/REVIEW.md @@ -0,0 +1,62 @@ +# bench-stand Code Review Checklist + +**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: +`README.md`. A diff here that changes what the nightly workflow invokes - a `main.das` verb or +flag, an exit code, or the `nightly.sh gate` vocabulary - applies `.github/workflows/REVIEW.md` +(repo root) too. + +**Never put a `[test]` file outside this directory - a test of this tool lives beside the file it +tests.** + +**Never add a test that touches the filesystem outside a `temp_directory`-rooted path, or that +leaves behind what it creates.** + +**A failure message a diff adds or changes states the underlying reason, and the summary line +carrying it names the file and the lane it ran in (`interp` or `jit`).** + +**Never add a run-record field without saying in `README.md` section 2 what reads it** - the +summary, the viewer, or a person opening the record. A record is an archive as well as the +viewer's input, so a field only a human reads is fine; a field nobody named is one nobody +notices going wrong. + +**A diff that changes a run-record field keeps the new reader parsing a record written by the old +code, a missing field keeping its declared default.** Records already on the box are never +rewritten. + +**Never let a child's own output overwrite a `timeout` or `memory` status in `run_bench_file` - +a killed child that printed a passing report is still killed. Loosen a value in `suite.json` +instead.** + +**Weakening the limit arms of `test_run_bench_file_limits` (`test_bench_runner.das`) is a defect: +dropping an arm, or relaxing what it asserts about the kill.** + +**A series a diff adds or recolors in `site/app.js` takes its color from its lane, never from its +position in the series list** - a color that moves when a series is added makes two nights +uncomparable. + +**A chart a diff adds to `site/app.js` shows a legend whenever it draws more than one lane.** + +**A chart a diff adds to `site/app.js` also ships its table view - the same points as a table of +numbers, behind the card's `table` toggle.** + +**Never read a benchmark's identity from anywhere but its path under `benchmarks/`** - the group +is the directory, the id is the path without `.das`. + +**Weakening `REVIEW.das` (beside this file) is a defect: dropping a check, narrowing what a check +walks, or rewriting a finding text so it no longer names what failed.** + +**Placement - one file, one line: a diff keeps each file inside its line, and a new file adds its +line here, with its tests, in the same change.** + +- `main.das` - verbs, argv, exit-code mapping. No parsing of dastest output, no statistics. +- `bench_suite.das` - `suite.json` and file discovery. No processes. +- `bench_runner.das` - one file in one lane: spawn, limits, output to samples and verdict. +- `bench_history.das` - records to dataset, verdicts, summary markdown. No processes, no argv. +- `suite.json` - configuration. +- `site/` - the viewer. Zero dependencies, zero build step. +- `nightly.sh` - the box-side driver. +- `bench-stand-deploy.sh` - root-side provisioning. +- `caddy.snippet` - the public route, and the only place a route is written down. +- `REVIEW.das` - the mechanical half of this checklist. +- `test_bench_suite.das`, `test_bench_runner.das`, `test_bench_history.das` - the module suites. +- `test_bench_cli.das` - the two programs, spawned: `main.das`'s verbs and `nightly.sh`'s ssh gate. diff --git a/utils/internal/bench-stand/bench-stand-deploy.sh b/utils/internal/bench-stand/bench-stand-deploy.sh new file mode 100755 index 0000000000..f4c6f22877 --- /dev/null +++ b/utils/internal/bench-stand/bench-stand-deploy.sh @@ -0,0 +1,127 @@ +#!/bin/sh +# bench-stand box-side provisioning for dasweb-1. The ONE privileged surface of the stand: +# installed root-owned at /usr/local/sbin/bench-stand-deploy.sh and run with sudo. Verbs: +# +# provision one-time: packages, the `bench` user, the layout, the clone, +# the forced-command ssh key the GitHub workflow uses +# caddy splice caddy.snippet into the daslang.io vhost, validate, reload +# status user, layout, last status.json, Caddy route presence +# +# Everything the stand does afterwards runs unprivileged as `bench` through nightly.sh, whose +# gate verbs (start / follow / status / summary) are all the key can reach - never a shell. The +# workflow's key (DASWEB_BENCH_KEY, an environment secret on `github-pages`) is generated on +# the operator's machine: `ssh-keygen -t ed25519 -f bench_key -N '' -C bench-stand`; the +# public half is what `provision` installs, the private half goes into the secret. +set -eu + +HOME_DIR=/srv/bench-stand +SVCUSER=bench +REPO=https://github.com/GaijinEntertainment/daScript.git +CADDYFILE=/etc/caddy/Caddyfile +TOOL=utils/internal/bench-stand + +verb="${1:?usage: bench-stand-deploy.sh provision | caddy | status}" + +provision() { + keyfile="${1:?provision needs the public key file the workflow will authenticate with}" + [ -f "$keyfile" ] || { echo "no such key file: $keyfile"; exit 1; } + # Only what the night needs: the build itself, ccache to keep the nightly rebuild + # incremental, rsync for the last_good copy, flock (util-linux) for the run lock. No GLFW or + # X11 development packages - `nightly.sh` configures with -DDAS_GLFW_DISABLED=ON. + apt-get install -y --no-install-recommends git cmake ninja-build g++ ccache rsync curl util-linux >/dev/null + id -u "$SVCUSER" >/dev/null 2>&1 || useradd --system --create-home --home-dir "$HOME_DIR" --shell /bin/bash "$SVCUSER" + install -d -o "$SVCUSER" -g "$SVCUSER" "$HOME_DIR" "$HOME_DIR/runs" "$HOME_DIR/site" "$HOME_DIR/logs" "$HOME_DIR/state" "$HOME_DIR/last_good" + if [ ! -d "$HOME_DIR/src/.git" ]; then + su -s /bin/sh "$SVCUSER" -c "git clone --quiet '$REPO' '$HOME_DIR/src'" + fi + # Forced command: the key can only start/follow/status the stand, never open a shell. + install -d -m 700 -o "$SVCUSER" -g "$SVCUSER" "$HOME_DIR/.ssh" + key=$(head -n 1 "$keyfile") + line="command=\"$HOME_DIR/src/$TOOL/nightly.sh gate\",no-port-forwarding,no-agent-forwarding,no-X11-forwarding,no-pty $key" + auth="$HOME_DIR/.ssh/authorized_keys" + touch "$auth" + grep -qF "$key" "$auth" || echo "$line" >> "$auth" + chown "$SVCUSER:$SVCUSER" "$auth" + chmod 600 "$auth" + echo "provisioned: user=$SVCUSER home=$HOME_DIR clone=$HOME_DIR/src key installed with forced command" + echo "next: sudo bench-stand-deploy.sh caddy; then a workflow_dispatch of nightly_bench.yml" +} + +vhost_has() { + # true when the daslang.io block of the shared Caddyfile carries $1 as a literal - the other + # vhosts may spell the same directive, so a whole-file grep is not the question + awk '/^daslang\.io \{/ { b = 1 } b { print } b && /^\}/ { b = 0 }' "$CADDYFILE" | grep -qF "$1" +} + +# Every route line the snippet declares, each the literal the vhost must carry. Derived from the +# snippet rather than written twice, so a route added there is checked without editing this script. +snippet_routes() { + sed -n 's/ *{$//; s/^\(redir\|handle_path\|handle\|header\|respond\|reverse_proxy\|root\) /\1 /p' "$1" +} + +# Prints "all", "none" or "partial": how much of the snippet the vhost already carries. A partial +# splice is never repaired automatically - re-splicing would duplicate the routes already there. +routes_state() { + local snippet=$1 route have=0 miss=0 + while IFS= read -r route; do + [ -n "$route" ] || continue + if vhost_has "$route"; then have=$((have + 1)); else miss=$((miss + 1)); fi + done < 0) { + if (line ~ /^#/ || line ~ /^[[:space:]]*$/) continue + print "\t" line + } + close(snip); done = 1; next + } + { print } + ' "$CADDYFILE.bak-$ts" > "$CADDYFILE" + if [ "$(routes_state "$snippet")" != all ]; then + echo "caddy: splice did not land every route (daslang.io vhost not matched?) - restoring"; cp "$CADDYFILE.bak-$ts" "$CADDYFILE"; exit 1 + fi + if ! caddy validate --config "$CADDYFILE" --adapter caddyfile >/dev/null 2>&1; then + echo "caddy validate FAILED - restoring $CADDYFILE.bak-$ts"; cp "$CADDYFILE.bak-$ts" "$CADDYFILE"; exit 1 + fi + systemctl reload caddy + echo "caddy: /bench/ spliced + reloaded (backup $CADDYFILE.bak-$ts)" +} + +status() { + id "$SVCUSER" 2>/dev/null || echo "user $SVCUSER: missing" + for d in src runs site logs state last_good; do + [ -d "$HOME_DIR/$d" ] && echo "$HOME_DIR/$d: ok" || echo "$HOME_DIR/$d: missing" + done + [ -f "$HOME_DIR/site/status.json" ] && cat "$HOME_DIR/site/status.json" || echo "status.json: never run" + echo "caddy: routes $(routes_state "$HOME_DIR/src/$TOOL/caddy.snippet" 2>/dev/null || echo "unknown - no snippet")" + ls "$HOME_DIR/runs" 2>/dev/null | wc -l | sed 's/^/runs: /' +} + +case "$verb" in + provision) provision "${2:-}" ;; + caddy) caddy_apply ;; + status) status ;; + *) echo "unknown verb: $verb"; exit 2 ;; +esac diff --git a/utils/internal/bench-stand/bench_history.das b/utils/internal/bench-stand/bench_history.das new file mode 100644 index 0000000000..6b5354f1ac --- /dev/null +++ b/utils/internal/bench-stand/bench_history.das @@ -0,0 +1,544 @@ +options gen2 +options indenting = 4 + +module bench_history public + +require bench_suite +require bench_runner +require daslib/fio +require daslib/json_boost +require daslib/strings_boost +require strings +require math + +//! What a night amounts to. `build_failed` never measured anything; `bench_failed` measured +//! what it could and at least one file failed. json_boost writes an enum as its name, so a +//! record and the viewer both read these spellings. +enum RunStatus { + ok + build_failed + bench_failed +} + +//! A verdict on one benchmark arm at the latest run. `first_seen` had no earlier point to +//! compare with; `missing` had a baseline and no point tonight. +enum ChangeKind { + regression + improvement + first_seen + missing +} + +struct CommitInfo { + sha : string + date : string + subject : string + author : string +} + +struct MachineInfo { + host : string + cpu : string + kernel : string + compiler : string + cores : int + mem_gb : int + load_start : string +} + +//! `status` defaults to `ok` so a meta that says nothing about a build - a local run driven by +//! hand - is not read as a night whose build failed. The driver always writes it explicitly. +struct BuildInfo { + status : string = "ok" + seconds : double + log_tail : string +} + +//! What the box-side driver hands `run`: identity of the night and of the tree it built. +struct RunMeta { + run_id : string + started : string + commit : CommitInfo + machine : MachineInfo + build : BuildInfo = BuildInfo() +} + +//! One night's record (`runs/.json`). `status` is `ok`, `build_failed` or +//! `bench_failed`; `lanes` maps a lane to `ok` or the reason it did not run. +struct RunRecord { + schema : int = 1 + run_id : string + started : string + finished : string + seconds : double + commit : CommitInfo + machine : MachineInfo + build : BuildInfo = BuildInfo() + lanes : table + status : RunStatus + files : array +} + +struct Failure { + path : string + lane : string + status : string + message : string +} + +//! One row of the viewer's run history. `failures` covers every non-ok, non-skipped file. +struct RunSummary { + id : string + sha : string + date : string + subject : string + started : string + status : RunStatus + seconds : double + build_status : string + build_seconds : double + build_log_tail : string + host : string + lanes : table + failures : array + skipped : array + files_ok : int + regressions : int + improvements : int +} + +//! One benchmark arm in one lane over every run that measured it. The three arrays are one +//! point per run, in run order: the run's index into `Dataset.runs`, its ns/op minimum, and its +//! spread over that night's repeats. +struct Series { + id : string + group : string + file : string + lane : string + runs : array + ns : array + spread : array +} + +//! `noise` is the baseline's relative median absolute deviation - the gate `change` had to pass. +struct Change { + id : string + group : string + lane : string + kind : ChangeKind + baseline : double + value : double + change : double + noise : double +} + +//! Geometric-mean index of one group in one lane, 100 = each series at its own reference +//! (the median of its first points). One point per run, in run order. +struct IndexSeries { + group : string + lane : string + runs : array + value : array +} + +//! `site/data.json`: everything the viewer renders, and what the summary is rendered from. +struct Dataset { + schema : int = 1 + generated : string + repo_url : string + latest : int + regression_threshold : double + noise_multiplier : double + baseline_runs : int + runs : array + groups : array + series : array + changes : array + index : array +} + +let INDEX_REFERENCE_POINTS = 5 +let MIN_BASELINE_POINTS = 3 + +//! The median of `vals`, which it does not reorder; 0 for an empty input. +def median(vals : array) : double { + return 0.0lf if (empty(vals)) + var sorted := vals + sort(sorted) + let n = length(sorted) + return (n % 2 == 1) ? sorted[n / 2] : (sorted[n / 2 - 1] + sorted[n / 2]) / 2.0lf +} + +//! Median absolute deviation divided by the median; 0 unless the median is positive. +def relative_mad(vals : array) : double { + return 0.0lf if (empty(vals)) + let med = median(vals) + return 0.0lf if (med <= 0.0lf) + var devs <- [for (x in vals); abs(x - med)] + return median(devs) / med +} + +def geomean(vals : array) : double { + return 0.0lf if (empty(vals)) + var acc = 0.0lf + for (x in vals) { + acc += log(x) + } + return exp(acc / double(length(vals))) +} + +//! The status a record earns from its parts. A night that measured nothing is not `ok`, whatever +//! the reason - every lane unavailable, a filter matching no file, a suite that vanished: an +//! empty night carries no data point and must not read as a good one. +def derive_status(rec : RunRecord) : RunStatus { + return RunStatus.build_failed if (rec.build.status != "ok") + var measured = 0 + for (f in rec.files) { + return RunStatus.bench_failed if (is_failure(f.status)) + measured++ if (f.status == FileStatus.ok) + } + return RunStatus.bench_failed if (measured == 0) + return RunStatus.ok +} + +let DRIVER_FILE_PATH = "utils/internal/bench-stand/nightly.sh" + +//! Fill `rec` in for a night that never benchmarked. A build that failed needs no file - its log +//! tail already rides `build` - while a build that passed and a driver that could not run the +//! suite gets one synthetic failed file carrying `reason`, so the summary and the viewer name it +//! instead of showing an empty night. `rec` keeps the identity the driver's meta gave it. +def mark_failed_run(var rec : RunRecord&; finished, reason : string) { + rec.finished = finished + rec.seconds = 0.0lf + rec.files |> clear() + if (rec.build.status == "ok") { + rec.files |> emplace(FileResult(path = DRIVER_FILE_PATH, id = "bench-stand", group = "stand", + lane = "driver", status = FileStatus.exit_nonzero, exit_code = 1, + message = empty(reason) ? "the driver could not run the suite" : reason)) + } + rec.status = derive_status(rec) +} + +//! Every `*.json` under `dir`, oldest commit first (commit date, then start time). A file that +//! does not parse is named in `errors` and left out, so one corrupt night cannot hide the rest. +def load_runs(runs_dir : string; var errors : array) : array { + var names : array + dir(runs_dir) $(name) { + return if (extension(name) != ".json") + names |> push(name) + } + sort(names) + var runs : array + for (name in names) { + let path = path_join(runs_dir, name) + var rec = RunRecord() + if (!sscan_json(fread(path), rec) || empty(rec.run_id)) { + errors |> push("run record does not parse or has no run_id: {path}") + continue + } + rec.status = derive_status(rec) + runs |> emplace(rec) + } + sort(runs) $(a, b) { + return a.commit.date != b.commit.date ? a.commit.date < b.commit.date : a.started < b.started + } + return <- runs +} + +//! Index of the run that started last - the night the summary and the changes describe; -1 when +//! there are no runs, which every caller branches on. +def latest_run_index(runs : array) : int { + var best = -1 + for (i in range(length(runs))) { + if (best < 0 || runs[i].started >= runs[best].started) { + best = i + } + } + return best +} + +def private summarize_run(rec : RunRecord) : RunSummary { + var s <- RunSummary(id = rec.run_id, sha = rec.commit.sha, date = rec.commit.date, subject = rec.commit.subject, + started = rec.started, status = rec.status, seconds = rec.seconds, build_status = rec.build.status, + build_seconds = rec.build.seconds, build_log_tail = rec.build.log_tail, host = rec.machine.host) + s.lanes := rec.lanes + for (f in rec.files) { + if (f.status == FileStatus.skipped) { + s.skipped |> push(Failure(path = f.path, lane = f.lane, status = "{f.status}", message = f.message)) + } elif (is_failure(f.status)) { + s.failures |> push(Failure(path = f.path, lane = f.lane, status = "{f.status}", message = f.message)) + } else { + s.files_ok++ + } + } + return <- s +} + +def private series_key(f : FileResult; sample_id : string) : string { + return "{f.lane}\t{f.id}#{sample_id}" +} + +//! `groups` collects only the groups that have at least one series - a group whose every file +//! failed every night has no chart to filter, and the failure panel is where it shows. +def private build_series(runs : array; var groups : table) : array { + var index_of : table + var out : array + for (ri, rec in range(length(runs)), runs) { + for (f in rec.files) { + continue if (f.status == FileStatus.skipped) + if (!empty(f.samples)) { + groups |> insert(f.group) + } + for (smp in f.samples) { + let key = series_key(f, smp.id) + var si = index_of?[key] ?? -1 + if (si < 0) { + si = length(out) + index_of[key] = si + out |> emplace(Series(id = "{f.id}#{smp.id}", group = f.group, file = f.id, lane = f.lane)) + } + let n = length(out[si].runs) + continue if (n > 0 && out[si].runs[n - 1] == ri) + out[si].runs |> push(ri) + out[si].ns |> push(smp.ns) + out[si].spread |> push(smp.spread) + } + } + } + sort(out) $(a, b) { + return a.lane != b.lane ? a.lane < b.lane : a.id < b.id + } + return <- out +} + +def private lane_ok_at(rec : RunRecord; lane, file_id : string) : bool { + for (f in rec.files) { + return f.status == FileStatus.ok if (f.lane == lane && f.id == file_id) + } + return false +} + +//! Verdicts for the latest run: each series measured there is compared with the median of its +//! last `baseline_runs` earlier points; a move beyond `regression_threshold` and beyond +//! `noise_multiplier` times the baseline's relative MAD is a regression or an improvement. A +//! series with no earlier point is `new`; one with a baseline but no point tonight, whose file +//! ran ok in that lane, is `missing`. +def detect_changes(runs : array; series : array; latest : int; cfg : SuiteConfig) : array { + var out : array + return <- out if (latest < 0) + for (ser in series) { + var tonight = -1.0lf + var prior : array + for (i in range(length(ser.runs))) { + if (ser.runs[i] == latest) { + tonight = ser.ns[i] + } elif (ser.runs[i] < latest) { + prior |> push(ser.ns[i]) + } + } + if (length(prior) > cfg.baseline_runs) { + prior |> erase(0, length(prior) - cfg.baseline_runs) + } + if (tonight < 0.0lf) { + if (!empty(prior) && lane_ok_at(runs[latest], ser.lane, ser.file)) { + out |> push(Change(id = ser.id, group = ser.group, lane = ser.lane, kind = ChangeKind.missing, baseline = median(prior))) + } + continue + } + if (empty(prior)) { + out |> push(Change(id = ser.id, group = ser.group, lane = ser.lane, kind = ChangeKind.first_seen, value = tonight)) + continue + } + continue if (length(prior) < MIN_BASELINE_POINTS) + let baseline = median(prior) + continue if (baseline <= 0.0lf) + let noise = relative_mad(prior) + let change = tonight / baseline - 1.0lf + let gate = max(cfg.regression_threshold, cfg.noise_multiplier * noise) + continue if (change <= gate && change >= -gate) + let kind = change > gate ? ChangeKind.regression : ChangeKind.improvement + out |> push(Change(id = ser.id, group = ser.group, lane = ser.lane, kind = kind, + baseline = baseline, value = tonight, change = change, noise = noise)) + } + return <- out +} + +def private reference_of(ser : Series) : double { + let firsts <- [for (i in range(min(INDEX_REFERENCE_POINTS, length(ser.ns)))); ser.ns[i]] + return median(firsts) +} + +//! Per group and lane, the geometric mean over its series of value / reference at every run +//! where at least half of the group's series were measured, times 100. +def build_index(series : array; run_count : int) : array { + var index_of : table + var keys_in_order : array + var members : array> + for (si, ser in range(length(series)), series) { + continue if (length(ser.ns) < MIN_BASELINE_POINTS) + let key = "{ser.group}\t{ser.lane}" + var gi = index_of?[key] ?? -1 + if (gi < 0) { + gi = length(members) + index_of[key] = gi + keys_in_order |> push(key) + var fresh_group : array + members |> emplace(fresh_group) + } + members[gi] |> push(si) + } + var out : array + out |> reserve(length(keys_in_order)) + for (key, group_members in keys_in_order, members) { + let tab = find(key, "\t") + var idx <- IndexSeries(group = slice(key, 0, tab), lane = slice(key, tab + 1)) + let refs <- [for (si in group_members); reference_of(series[si])] + for (ri in range(run_count)) { + var ratios : array + for (mi, si in range(length(group_members)), group_members) { + continue if (refs[mi] <= 0.0lf) + for (pi in range(length(series[si].runs))) { + if (series[si].runs[pi] == ri) { + ratios |> push(series[si].ns[pi] / refs[mi]) + break + } + } + } + continue if (length(ratios) * 2 < length(group_members)) + idx.runs |> push(ri) + idx.value |> push(geomean(ratios) * 100.0lf) + } + out |> emplace(idx) + } + sort(out) $(a, b) { + return a.group != b.group ? a.group < b.group : a.lane < b.lane + } + return <- out +} + +def build_dataset(runs : array; cfg : SuiteConfig; generated, repo_url : string) : Dataset { + var ds <- Dataset(generated = generated, repo_url = repo_url, regression_threshold = cfg.regression_threshold, + noise_multiplier = cfg.noise_multiplier, baseline_runs = cfg.baseline_runs) + ds.latest = latest_run_index(runs) + var groups : table + ds.series <- build_series(runs, groups) + ds.groups <- [for (g in keys(groups)); g] + sort(ds.groups) + ds.changes <- detect_changes(runs, ds.series, ds.latest, cfg) + ds.index <- build_index(ds.series, length(runs)) + ds.runs |> reserve(length(runs)) + for (rec in runs) { + ds.runs |> emplace(summarize_run(rec)) + } + if (ds.latest >= 0) { + for (ch in ds.changes) { + if (ch.kind == ChangeKind.regression) { + ds.runs[ds.latest].regressions++ + } elif (ch.kind == ChangeKind.improvement) { + ds.runs[ds.latest].improvements++ + } + } + } + return <- ds +} + +def private count_kind(changes : array; kind : ChangeKind) : int { + var n = 0 + for (ch in changes) { + n++ if (ch.kind == kind) + } + return n +} + +def private write_change_rows(var w : StringBuilderWriter; changes : array; kind : ChangeKind) { + w |> write("| benchmark | lane | baseline ns/op | now ns/op | change | noise |\n|---|---|---:|---:|---:|---:|\n") + for (ch in changes) { + continue if (ch.kind != kind) + w |> write("| `{ch.id}` | {ch.lane} | {ch.baseline:.1f} | {ch.value:.1f} | {ch.change * 100.0lf:+.1f}% | {ch.noise * 100.0lf:.1f}% |\n") + } + w |> write("\n") +} + +//! The markdown the workflow posts as its step summary and the site serves as `summary.md`. +def render_summary(ds : Dataset; site_url : string) : string { + return build_string() $(var w) { + if (ds.latest < 0) { + w |> write("# Benchmark stand\n\nNo run records yet.\n") + return + } + assume run = ds.runs[ds.latest] + w |> write("# Benchmark stand - {run.status} - {run.started}\n\n") + // a hand-driven local run carries no commit and no machine; naming neither beats naming + // both as empty backticks + if (!empty(run.sha) || !empty(run.subject)) { + let short_sha = length(run.sha) > 8 ? slice(run.sha, 0, 8) : run.sha + w |> write("Commit `{short_sha}` {run.subject} ({run.date})") + w |> write(empty(run.host) ? ".\n\n" : " on `{run.host}`.\n\n") + } elif (!empty(run.host)) { + w |> write("On `{run.host}`.\n\n") + } + if (run.build_status != "ok") { + w |> write("## The build failed after {run.build_seconds:.0f} s - nothing was measured\n\n") + w |> write(empty(run.build_log_tail) + ? "The driver recorded no build log; `logs/build-{run.id}.log` on the box has it.\n\n" + : "Last lines of the build log (`logs/build-{run.id}.log` on the box has all of it):\n\n```\n{run.build_log_tail}\n```\n\n") + if (!empty(site_url)) { + w |> write("Charts: {site_url}\n") + } + return + } + w |> write("Build {run.build_seconds:.0f} s, suite {run.seconds:.0f} s; {run.files_ok} file runs ok, {length(run.failures)} failed, {length(run.skipped)} skipped.\n\n") + w |> write("Lanes:") + for (lane, state in keys(run.lanes), values(run.lanes)) { + w |> write(" {lane} = {state};") + } + w |> write("\n\n") + if (!empty(run.failures)) { + w |> write("## Failures ({length(run.failures)})\n\n| file | lane | status | message |\n|---|---|---|---|\n") + for (f in run.failures) { + w |> write("| `{f.path}` | {f.lane} | {f.status} | {replace(f.message, "|", "\\|")} |\n") + } + w |> write("\nEach failure's last log lines are in the run record `runs/{run.id}.json`.\n\n") + } + let regs = count_kind(ds.changes, ChangeKind.regression) + let imps = count_kind(ds.changes, ChangeKind.improvement) + let news = count_kind(ds.changes, ChangeKind.first_seen) + let miss = count_kind(ds.changes, ChangeKind.missing) + w |> write("## Changes: {regs} regressions, {imps} improvements, {news} first seen, {miss} missing\n\n") + w |> write("A change counts when it passes both {ds.regression_threshold * 100.0lf:.0f}% and {ds.noise_multiplier:.1f}x the baseline noise; baseline = median of the previous {ds.baseline_runs} runs.\n\n") + if (regs > 0) { + w |> write("### Regressions\n\n") + write_change_rows(w, ds.changes, ChangeKind.regression) + } + if (imps > 0) { + w |> write("### Improvements\n\n") + write_change_rows(w, ds.changes, ChangeKind.improvement) + } + if (miss > 0) { + w |> write("### Missing tonight (measured before, file ran ok)\n\n") + for (ch in ds.changes) { + continue if (ch.kind != ChangeKind.missing) + w |> write("- `{ch.id}` ({ch.lane})\n") + } + w |> write("\n") + } + if (!empty(run.skipped)) { + w |> write("## Skipped ({length(run.skipped)})\n\n") + for (f in run.skipped) { + w |> write("- `{f.path}` ({f.lane}): {f.message}\n") + } + w |> write("\n") + } + if (!empty(site_url)) { + w |> write("Charts: {site_url}\n") + } + } +} + +//! 0 when the latest run is ok; 1 when it failed (build or any file); 2 when it is ok but +//! regressed and the caller asked regressions to fail. +def summary_exit_code(ds : Dataset; fail_on_regression : bool) : int { + return 1 if (ds.latest < 0 || ds.runs[ds.latest].status != RunStatus.ok) + return 2 if (fail_on_regression && ds.runs[ds.latest].regressions > 0) + return 0 +} diff --git a/utils/internal/bench-stand/bench_runner.das b/utils/internal/bench-stand/bench_runner.das new file mode 100644 index 0000000000..e52430ceb2 --- /dev/null +++ b/utils/internal/bench-stand/bench_runner.das @@ -0,0 +1,432 @@ +options gen2 +options indenting = 4 + +module bench_runner public + +require bench_suite +require daslib/fio +require daslib/json_boost +require daslib/strings_boost +require daslib/strings_convert +require strings + +//! How one benchmark file's process ended. +enum FileStatus { + ok + skipped + spawn_failed + compile_error + failed + timeout + memory + exit_nonzero +} + +//! One `run("name")` arm of one `[benchmark]` function, aggregated over the repeats: +//! `ns` is the minimum ns/op (the noise-robust statistic), `ns_median` the median, +//! `spread` `(max - min) / min`. The four allocation fields are per-op medians. +struct Sample { + id : string + ns : double + ns_median : double + spread : double + runs : int + bytes : int64 + allocs : int64 + string_bytes : int64 + string_allocs : int64 +} + +//! One benchmark file in one lane. `samples` carries only the arms whose `[benchmark]` function +//! passed - a failed function's arms are dropped, so a broken assertion cannot publish a number. +struct FileResult { + path : string + id : string + group : string + lane : string + status : FileStatus + exit_code : int + seconds : double + tier : string + message : string + log_tail : string + samples : array +} + +struct RunLimits { + timeout_seconds : int = 900 + memory_limit_mb : int = 4096 + repeat : int = 3 +} + +//! `dastest --bench-format json` emits one of these per arm per repeat; the report file +//! (`--json-file`) carries the per-function verdicts. +struct BenchStatsLine { + name : string + sub_name : string + n : int + time_ns : int64 + allocs : uint64 + heap_bytes : uint64 + string_allocs : uint64 + string_heap_bytes : uint64 + func_type : string +} + +struct ReportTest { + name : string + passed : bool +} + +struct DastestReport { + success : bool + tests : array +} + +let TAIL_LINES = 60 +let TAIL_LINE_CHARS = 400 + +def is_failure(st : FileStatus) : bool { + return st != FileStatus.ok && st != FileStatus.skipped +} + +def private median(var vals : array) : double { + return 0.0lf if (empty(vals)) + sort(vals) + let n = length(vals) + return (n % 2 == 1) ? vals[n / 2] : (vals[n / 2 - 1] + vals[n / 2]) / 2.0lf +} + +def private median_i64(var vals : array) : int64 { + return 0l if (empty(vals)) + sort(vals) + return vals[length(vals) / 2] +} + +def private keep_tail(var tail : array; line : string) { + if (length(tail) >= TAIL_LINES) { + tail |> erase(0) + } + tail |> push(length(line) > TAIL_LINE_CHARS ? slice(line, 0, TAIL_LINE_CHARS) : line) +} + +def private first_line(text : string) : string { + let nl = find(text, "\n") + return nl < 0 ? text : slice(text, 0, nl) +} + +def private error_or_last_line(text : string) : string { + var last = "" + for (raw in split(text, "\n")) { + let line = strip(raw) + continue if (empty(line)) + return line if (starts_with(line, "FATAL") || starts_with(line, "error") || find(line, "[E]") >= 0) + last = line + } + return last +} + +struct private OutputScan { + by_arm : table> + arm_order : array + tail : array + compile_error : string + failed_to_compile : string + fatal : string + failed_fns : table + tier : string +} + +def private note_diagnostic(var scan : OutputScan; line : string) { + if (empty(scan.compile_error) && find(line, "error[") >= 0 && find(line, "]: ") >= 0) { + scan.compile_error = strip(line) + } + if (empty(scan.failed_to_compile) && find(line, "Failed to compile") >= 0) { + scan.failed_to_compile = strip(line) + } + if (empty(scan.fatal) && starts_with(line, "FATAL")) { + scan.fatal = strip(line) + } + let fail_at = find(line, "--- FAIL '") + return if (fail_at < 0) + let name_start = fail_at + length("--- FAIL '") + let name_end = find(line, "'", name_start) + if (name_end > name_start) { + scan.failed_fns |> insert(slice(line, name_start, name_end)) + } +} + +def private scan_line(var scan : OutputScan; line : string) { + if (starts_with(line, "\{")) { + var st : BenchStatsLine + if (sscan_json(line, st) && st.n > 0) { + let key = "{st.name}/{st.sub_name}" + if (!key_exists(scan.by_arm, key)) { + scan.arm_order |> push(key) + } + scan.by_arm[key] |> push(st) + if (empty(scan.tier)) { + scan.tier = st.func_type + } + return + } + } + keep_tail(scan.tail, line) + note_diagnostic(scan, line) +} + +//! True when the report parsed; its failed functions join the scan's. +def private note_report(var scan : OutputScan; report_text : string) : bool { + return false if (empty(report_text)) + var report : DastestReport + return false if (!sscan_json(report_text, report)) + for (test in report.tests) { + if (!test.passed) { + scan.failed_fns |> insert(test.name) + } + } + return true +} + +def private aggregate_arm(key : string; stats : array) : Sample { + var ns <- [for (st in stats); double(st.time_ns) / double(st.n)] + var bytes <- [for (st in stats); int64(st.heap_bytes) / int64(st.n)] + var allocs <- [for (st in stats); int64(st.allocs) / int64(st.n)] + var sbytes <- [for (st in stats); int64(st.string_heap_bytes) / int64(st.n)] + var sallocs <- [for (st in stats); int64(st.string_allocs) / int64(st.n)] + sort(ns) + let lo = ns[0] + let hi = ns[length(ns) - 1] + return Sample(id = key, ns = lo, ns_median = median(ns), + spread = lo > 0.0lf ? (hi - lo) / lo : 0.0lf, runs = length(ns), + bytes = median_i64(bytes), allocs = median_i64(allocs), + string_bytes = median_i64(sbytes), string_allocs = median_i64(sallocs)) +} + +def private decide_status(scan : OutputScan; report_ok : bool; exit_code : int; var res : FileResult&) { + if (!empty(scan.failed_to_compile)) { + res.status = FileStatus.compile_error + res.message = !empty(scan.compile_error) ? scan.compile_error : scan.failed_to_compile + } elif (!empty(scan.failed_fns)) { + res.status = FileStatus.failed + var names <- [for (n in keys(scan.failed_fns)); n] + sort(names) + res.message = "benchmark function(s) failed: {join(names, ", ")}" + } elif (exit_code != 0 && report_ok && !empty(res.samples)) { + // dastest wrote a finished report and the arms are here, so the measurement stands and + // only the shutdown went wrong - a leak dump, a smart-pointer assert, a signal after the + // last benchmark. Worth saying, never worth discarding a night for: a RelWithDebInfo + // host's allocation tracker takes every process to 1 and would fail the whole suite. + res.status = FileStatus.ok + if (empty(scan.fatal)) { + res.message = "measured, then exited with code {exit_code} at shutdown" + } else { + res.message = "measured, then exited with code {exit_code} at shutdown: {scan.fatal}" + } + } elif (exit_code != 0) { + res.status = FileStatus.exit_nonzero + if (!empty(scan.fatal)) { + res.message = scan.fatal + } elif (empty(res.samples)) { + res.message = "process exited with code {exit_code} before measuring anything, printing no diagnostic (a signal number when it was killed; a missing or unrunnable daslang binary lands here)" + } else { + res.message = "process exited with code {exit_code} without a finished dastest report - the suite did not run to its end" + } + } elif (!report_ok) { + res.status = FileStatus.exit_nonzero + res.message = "dastest wrote no report file - the process ended before the suite finished" + } elif (empty(res.samples)) { + res.status = FileStatus.failed + res.message = "the file ran but produced no benchmark samples (no [benchmark] function, or every arm reported n = 0)" + } else { + res.status = FileStatus.ok + } +} + +//! Turn a finished process's output into the result's samples, status and message. `lines` is +//! the merged stdout+stderr, `report_text` the `--json-file` contents (empty when the process +//! never wrote it). `res.status` on entry is the limit verdict (`timeout` / `memory`) or `ok`, +//! and a limit verdict is kept whatever the output says. +def parse_bench_output(lines : array; report_text : string; exit_code : int; var res : FileResult&) { + var scan <- OutputScan() + for (line in lines) { + scan_line(scan, line) + } + res.log_tail = join(scan.tail, "\n") + res.exit_code = exit_code + res.tier = scan.tier + let report_ok = note_report(scan, report_text) + res.samples |> reserve(length(scan.arm_order)) + for (key in scan.arm_order) { + let fn_name = slice(key, 0, find(key, "/")) + continue if (key_exists(scan.failed_fns, fn_name)) + res.samples |> emplace(aggregate_arm(key, scan.by_arm[key])) + } + return if (res.status == FileStatus.timeout || res.status == FileStatus.memory) + decide_status(scan, report_ok, exit_code, res) +} + +def private rss_mb_from_proc(pid : int) : int { + let text = fread("/proc/{pid}/status") + return -1 if (empty(text)) + let at = find(text, "VmRSS:") + return -1 if (at < 0) + let line = first_line(slice(text, at + length("VmRSS:"))) + let kb = try_to_int(strip(replace(line, "kB", ""))) ?? -1 + return -1 if (kb < 0) + return (kb + 1023) / 1024 +} + +def private rss_mb_from_ps(pid : int) : int { + var out = "" + let code = run_and_capture(["/bin/ps", "-o", "rss=", "-p", "{pid}"], out, 5.0) + return -1 if (code != 0) + let kb = try_to_int(strip(out)) ?? -1 + return -1 if (kb < 0) + return (kb + 1023) / 1024 +} + +//! Resident set size of a live process in MB, rounded up so a process holding memory never +//! reads as 0 and a cap fires a hair early rather than late: `/proc` where it exists, `ps` +//! otherwise (macOS), -1 when neither answers - a caller reading -1 as "under the cap" is what +//! makes a missing source disable the memory limit rather than kill every child. +def process_rss_mb(pid : int) : int { + let from_proc = rss_mb_from_proc(pid) + return from_proc if (from_proc >= 0) + return rss_mb_from_ps(pid) +} + +//! The three lanes differ only in argv. `aot` needs BOTH `-use-aot` on the host and `--use-aot` +//! on dastest: without the second one every function silently interprets and the lane measures +//! the interpreter under an AOT label. With it, a function missing its stub is `error[50101]`, +//! so a run that finishes is proof the stubs were live. +def private bench_argv(bin, repo_root : string; f : BenchFile; lane : string; report_path : string; repeat : int) : array { + var argv <- [bin] + if (lane == "jit") { + argv |> push("-jit") + } elif (lane == "aot") { + argv |> push("-use-aot") + } + argv |> push(to_generic_path(path_join(repo_root, "dastest/dastest.das"))) + argv |> push("--") + if (lane == "aot") { + argv |> push("--use-aot") + } + argv |> push_from(["--bench", "--bench-format", "json", "--json-file", report_path, + "--count", "{repeat}", "--test", to_generic_path(path_join(path_join(repo_root, "benchmarks"), f.path))]) + return <- argv +} + +def private new_result(f : BenchFile; lane : string) : FileResult { + return <- FileResult(path = f.path, id = f.id, group = f.group, lane = lane) +} + +//! Run one benchmark file in one lane under the limits, from the repo root. The child is +//! killed when it outlives `timeout_seconds` or its RSS passes `memory_limit_mb`; the result +//! then says so and keeps the arms that completed before the kill. +def run_bench_file(bin, repo_root : string; f : BenchFile; lane : string; limits : RunLimits) : FileResult { + var res <- new_result(f, lane) + let skipped = skip_reason(f, lane) + if (!empty(skipped)) { + res.status = FileStatus.skipped + res.message = skipped + return <- res + } + var err = "" + let report_path = create_temp_file("bench_stand_", ".json", err) + if (empty(report_path)) { + res.status = FileStatus.spawn_failed + res.message = "cannot create the dastest report file: {err}" + return <- res + } + remove(report_path) + let argv <- bench_argv(bin, repo_root, f, lane, report_path, limits.repeat) + var lines : array + var exit_code = process_running + let t0 = ref_time_ticks() + let noenv : array + with_process(argv, repo_root, noenv) $(var p) { + unsafe { + if (process_pid(p) <= 0) { + res.status = FileStatus.spawn_failed + res.message = "cannot spawn {argv[0]}" + return + } + var checks = 0 + while (true) { + process_drain(p) $(line) { + lines |> push(line) + } + exit_code = process_poll(p) + break if (exit_code != process_running) + let elapsed = double(get_time_usec(t0)) / 1000000.0lf + if (elapsed > double(limits.timeout_seconds)) { + res.status = FileStatus.timeout + res.message = "killed after {limits.timeout_seconds} s (timeout_seconds)" + process_kill(p) + exit_code = process_wait(p, 10.0) + break + } + checks++ + if (checks % 10 == 0) { + let rss = process_rss_mb(process_pid(p)) + if (rss > limits.memory_limit_mb) { + res.status = FileStatus.memory + res.message = "killed at {rss} MB resident (memory_limit_mb = {limits.memory_limit_mb})" + process_kill(p) + exit_code = process_wait(p, 10.0) + break + } + } + sleep(100u) + } + process_drain(p) $(line) { + lines |> push(line) + } + } + } + res.seconds = double(get_time_usec(t0)) / 1000000.0lf + return <- res if (res.status == FileStatus.spawn_failed) + let report_text = fread(report_path) + remove(report_path) + parse_bench_output(lines, report_text, exit_code, res) + return <- res +} + +//! Empty when the AOT binary exists and reports the AOT tier; otherwise the reason. An absent +//! binary is the ordinary case on a tree nobody built the AOT target in. +def probe_aot(aot_bin : string) : string { + return "no AOT binary at {aot_bin} - build the test_aot target first" if (!fexist(aot_bin)) + var err = "" + let probe = create_temp_file("bench_stand_aot_", ".das", err) + return "cannot create the probe file: {err}" if (empty(probe)) + fwrite(probe, "options gen2\n[export]\ndef main() \{\n print(\"aot-probe-ok\\n\")\n\}\n") + var out = "" + let code = run_and_capture([aot_bin, "-use-aot", probe], out, 120.0) + remove(probe) + // The marker, as with the JIT probe: it prints only if the binary got as far as running a + // program. A trivial program has no stubs of its own, so this proves the binary runs, never + // that a benchmark's stubs are present - `--use-aot` and error[50101] prove that, per file. + return "" if (find(out, "aot-probe-ok") >= 0) + return "{aot_bin} -use-aot ran no program (exit {code}): {error_or_last_line(out)}" +} + +//! Empty when `daslang -jit` runs a trivial program on this box; otherwise the reason the +//! JIT lane is unavailable (dasLLVM not built, a missing LLVM library). +def probe_jit(daslang_bin : string) : string { + var err = "" + let probe = create_temp_file("bench_stand_jit_", ".das", err) + return "cannot create the probe file: {err}" if (empty(probe)) + fwrite(probe, "options gen2\n[export]\ndef main() \{\n print(\"jit-probe-ok\\n\")\n\}\n") + var out = "" + let code = run_and_capture([daslang_bin, "-jit", probe], out, 120.0) + remove(probe) + // The marker, not the exit code, is the evidence: it prints only after the JIT compiled and + // ran the function. A RelWithDebInfo host exits 1 from its allocation tracker's report on + // every process, and keying on the code there would delete half the stand's measurements + // on every developer build. + return "" if (find(out, "jit-probe-ok") >= 0) + let said = error_or_last_line(out) + return "daslang -jit produced no jit-compiled run (exit {code}) and printed nothing - is {daslang_bin} runnable?" if (empty(said)) + return "daslang -jit produced no jit-compiled run (exit {code}): {said}" +} diff --git a/utils/internal/bench-stand/bench_suite.das b/utils/internal/bench-stand/bench_suite.das new file mode 100644 index 0000000000..a277fbbbf6 --- /dev/null +++ b/utils/internal/bench-stand/bench_suite.das @@ -0,0 +1,155 @@ +options gen2 +options indenting = 4 + +module bench_suite public + +require daslib/fio +require daslib/json_boost + +//! Per-file knobs from `suite.json` `files`; a zero value means "inherit the suite default". +//! `skip` takes the file out of every lane, `skip_lanes` out of the ones it names - a file the +//! AOT emitter cannot handle still measures in the interpreter. +struct FileOverride { + @optional skip : string + @optional skip_lanes : table + @optional timeout_seconds : int + @optional memory_limit_mb : int +} + +//! Field initializers are the defaults a missing key keeps. +struct SuiteConfig { + root : string = "benchmarks" + exclude : array + lanes : array + repeat : int = 3 + timeout_seconds : int = 900 + memory_limit_mb : int = 4096 + regression_threshold : double = 0.10lf + noise_multiplier : double = 3.0lf + baseline_runs : int = 7 + files : table + lane_excludes : table> +} + +//! One benchmark file the stand runs. `path` is relative to the suite root with `/` separators, +//! `id` is `path` without the extension, `group` is its directory. A non-empty `skip` is the reason +//! the file is listed but not run. +struct BenchFile { + path : string + id : string + group : string + timeout_seconds : int + memory_limit_mb : int + skip : string + skip_lanes : table +} + +//! Why this file does not run in this lane, empty when it does. A whole-file skip answers for +//! every lane; a lane skip answers only for its own. +def skip_reason(f : BenchFile; lane : string) : string { + return f.skip if (!empty(f.skip)) + return f.skip_lanes?[lane] ?? "" +} + +//! Load `suite.json`. Fails closed: any missing, unparseable or out-of-range field sets `error` +//! and returns the defaults, so a typo cannot silently shrink the suite. `exclude` takes a file +//! out of the tree; `lane_excludes` takes it out of one lane, keyed by glob and carrying the +//! reason the site shows every night. +def load_suite_config(path : string; var error : string&) : SuiteConfig { + error = "" + var cfg = SuiteConfig() + if (!stat(path).is_reg) { + error = "suite config not found: {path}" + return <- cfg + } + if (!sscan_json(fread(path), cfg)) { + error = "suite config does not parse as JSON: {path}" + return <- cfg + } + if (empty(cfg.lanes)) { + error = "suite config lists no lanes: {path}" + } elif (cfg.repeat <= 0 || cfg.timeout_seconds <= 0 || cfg.memory_limit_mb <= 0) { + error = "suite config needs positive repeat, timeout_seconds and memory_limit_mb: {path}" + } elif (cfg.baseline_runs < 3) { + error = "suite config needs baseline_runs >= 3: {path}" + } + for (lane in keys(cfg.lane_excludes)) { + if (find_index(cfg.lanes, lane) < 0) { + error = "suite config excludes files from lane \"{lane}\", which it does not run: {path}" + } + } + for (lane in cfg.lanes) { + if (lane != "interp" && lane != "jit" && lane != "aot") { + error = "suite config names an unknown lane \"{lane}\" (interp, jit or aot): {path}" + } + } + return <- cfg +} + +def private is_excluded(rel : string; excludes : array) : bool { + for (pattern in excludes) { + return true if (match_glob(pattern, rel)) + } + return false +} + +//! Every `.das` file under `/` that no `exclude` glob matches, sorted by +//! path, with the per-file overrides applied. A `files` override naming a path that does not +//! exist is reported through `error` so a renamed benchmark cannot silently lose its skip. +def discover_files(repo_root : string; cfg : SuiteConfig; var error : string&) : array { + error = "" + let root = path_join(repo_root, cfg.root) + var rels : array + dir_rec(root) $(name, is_dir) { + let rel = to_generic_path(name) + return if (is_dir || extension(rel) != ".das" || is_excluded(rel, cfg.exclude)) + rels |> push(rel) + } + sort(rels) + var out : array + out |> reserve(length(rels)) + for (rel in rels) { + var f = BenchFile(path = rel, id = stem_path(rel), group = group_of(rel), + timeout_seconds = cfg.timeout_seconds, memory_limit_mb = cfg.memory_limit_mb) + for (lane in keys(cfg.lane_excludes)) { + cfg.lane_excludes |> get(lane) $(globs) { + for (pattern in keys(globs)) { + continue if (!match_glob(pattern, rel)) + f.skip_lanes[lane] = globs?[pattern] ?? "" + } + } + } + cfg.files |> get(rel) $(ov) { + f.skip = ov.skip + for (lane in keys(ov.skip_lanes)) { + f.skip_lanes[lane] = ov.skip_lanes?[lane] ?? "" + } + if (ov.timeout_seconds > 0) { + f.timeout_seconds = ov.timeout_seconds + } + if (ov.memory_limit_mb > 0) { + f.memory_limit_mb = ov.memory_limit_mb + } + } + out |> emplace(f) + } + for (k in keys(cfg.files)) { + if (!stat(path_join(root, k)).is_reg) { + error = "suite config overrides \"{k}\", which is not a file under {root}" + } + } + return <- out +} + +//! `core/hash/test02.das` -> `core/hash`; a file at the root has group `root`. +def group_of(rel : string) : string { + let d = to_generic_path(dir_name(rel)) + return (empty(d) || d == ".") ? "root" : d +} + +//! `core/hash/test02.das` -> `core/hash/test02`. +def stem_path(rel : string) : string { + let d = to_generic_path(dir_name(rel)) + let s = stem(rel) + return (empty(d) || d == ".") ? s : "{d}/{s}" +} diff --git a/utils/internal/bench-stand/caddy.snippet b/utils/internal/bench-stand/caddy.snippet new file mode 100644 index 0000000000..3f500be9d9 --- /dev/null +++ b/utils/internal/bench-stand/caddy.snippet @@ -0,0 +1,17 @@ +# bench-stand routes for the daslang.io vhost in /etc/caddy/Caddyfile. +# This file is the authoritative copy of the stand's public boundary — the +# deployed Caddyfile is edited to match it, never the other way round. +# +# Paste inside the `daslang.io { ... }` block, ahead of `root`/`file_server`. +# `bench-stand-deploy.sh caddy` does the splice, validates, and reloads. + +# The nightly benchmark stand: a static tree the `bench` user rewrites every +# night (viewer, data.json, status.json, summary.md, runs/). No service behind +# it. no-cache: the viewer fetches data.json on every visit and a day-old +# cached copy would show yesterday's night as tonight's. +redir /bench /bench/ 308 +handle_path /bench/* { + root * /srv/bench-stand/site + header Cache-Control "no-cache" + file_server +} diff --git a/utils/internal/bench-stand/main.das b/utils/internal/bench-stand/main.das new file mode 100644 index 0000000000..20cea7aeb3 --- /dev/null +++ b/utils/internal/bench-stand/main.das @@ -0,0 +1,240 @@ +options gen2 +options indenting = 4 + +require bench_suite +require bench_runner +require bench_history +require daslib/clargs +require daslib/fio +require daslib/json_boost +require daslib/strings_boost +require strings + +//! The daslang benchmark stand - the tool half of the nightly. Two verbs, both run from the +//! repo root with the tree's own binary: +//! daslang utils/internal/bench-stand/main.das -- run --meta meta.json --out runs/.json +//! daslang utils/internal/bench-stand/main.das -- report --runs runs/ --out-data site/data.json --out-summary site/summary.md +//! The box-side driver is `nightly.sh`; the exit codes are stated where they are produced. + +[CommandLineArgs] +struct Config { + @clarg_positional + @clarg_doc = "run | report" + verb : string + + @clarg_doc = "Repo root (default: .)" + root : string + + @clarg_doc = "suite.json to use (default: the one beside this tool)" + suite : string + + @clarg_doc = "run: daslang binary the interp and jit lanes use (default: bin/daslang under --root)" + bin : string + + @clarg_doc = "run: binary the aot lane uses - daslang with the benchmarks' AOT stubs linked in (default: bin/test_aot under --root)" + aot_bin : string + + @clarg_doc = "run: JSON with run_id, started, commit, machine, build - written by nightly.sh" + meta : string + + @clarg_doc = "run: where to write the run record" + out : string + + @clarg_doc = "run: only files whose path contains this substring" + filter : string + + @clarg_doc = "run: comma-separated lanes to run (default: suite.json lanes)" + lanes : string + + @clarg_doc = "run: benchmark nothing - write the record of a night that could not run the suite, with this reason (non-empty), and exit 1" + failed : string + + @clarg_doc = "run: repeats per file (default: suite.json repeat)" + repeat : int + + @clarg_doc = "report: directory of run records" + runs : string + + @clarg_doc = "report: where to write data.json" + out_data : string + + @clarg_doc = "report: where to write summary.md" + out_summary : string + + @clarg_doc = "report: ISO timestamp stamped into data.json (default: now, UTC)" + generated : string + + @clarg_doc = "report: repository URL for commit links" + repo_url : string + + @clarg_doc = "report: site URL the summary links to" + site_url : string + + @clarg_doc = "report: exit 2 when the latest run regressed" + fail_on_regression : bool + + @clarg_short = "?" + @clarg_name = "show-help" + @clarg_doc = "Show this help and exit" + help : bool +} + +def private iso_now() : string { + return "{iso8601_now()}" +} + +def private write_text_file(path, text : string) : bool { + if (!fwrite(path, text)) { + to_log(LOG_ERROR, "cannot write {path}\n") + return false + } + return true +} + +def private suite_path_of(cfg : Config) : string { + return empty(cfg.suite) ? path_join(get_das_root(), "utils/internal/bench-stand/suite.json") : cfg.suite +} + +def private read_meta(path : string; var rec : RunRecord&) : bool { + var meta = RunMeta() + return false if (!sscan_json(fread(path), meta) || empty(meta.run_id)) + rec.run_id = meta.run_id + rec.started = meta.started + rec.commit = meta.commit + rec.machine = meta.machine + rec.build = meta.build + return true +} + +//! The binary a lane measures with: the AOT lane needs its own, the other two share daslang. +def private lane_binary(lane, bin, aot_bin : string) : string { + return lane == "aot" ? aot_bin : bin +} + +def private probe_lanes(var rec : RunRecord&; lanes : array; bin, aot_bin : string) { + for (lane in lanes) { + if (lane == "jit") { + let why = probe_jit(bin) + rec.lanes[lane] = empty(why) ? "ok" : why + } elif (lane == "aot") { + let why = probe_aot(aot_bin) + rec.lanes[lane] = empty(why) ? "ok" : why + } else { + rec.lanes[lane] = "ok" + } + to_log(LOG_INFO, "lane {lane}: {rec.lanes[lane]}\n") + } +} + +def private matches_filter(f : BenchFile; filter : string) : bool { + return empty(filter) || find(f.path, filter) >= 0 +} + +def private verb_run(cfg : Config; suite : SuiteConfig) : int { + if (empty(cfg.meta) || empty(cfg.out)) { + to_log(LOG_ERROR, "bench-stand run: --meta and --out are required\n") + return 1 + } + let root = empty(cfg.root) ? "." : cfg.root + let bin = empty(cfg.bin) ? path_join(root, "bin/daslang") : cfg.bin + let aot_bin = empty(cfg.aot_bin) ? path_join(root, "bin/test_aot") : cfg.aot_bin + var rec = RunRecord() + if (!read_meta(cfg.meta, rec)) { + to_log(LOG_ERROR, "bench-stand run: --meta {cfg.meta} does not parse or has no run_id\n") + return 1 + } + if (!empty(cfg.failed)) { + mark_failed_run(rec, iso_now(), cfg.failed) + return 1 if (!write_text_file(cfg.out, sprint_json(rec, true))) + to_log(LOG_ERROR, "bench-stand run: {rec.status} - {cfg.failed}; record {cfg.out}\n") + return 1 + } + var error = "" + let all_files <- discover_files(root, suite, error) + if (!empty(error)) { + to_log(LOG_ERROR, "bench-stand run: {error}\n") + return 1 + } + let lanes <- empty(cfg.lanes) ? clone_to_move(suite.lanes) : split(cfg.lanes, ",") + probe_lanes(rec, lanes, bin, aot_bin) + let repeat = cfg.repeat > 0 ? cfg.repeat : suite.repeat + let t0 = ref_time_ticks() + let failed = benchmark_every_file(rec, all_files, cfg.filter, lanes, bin, aot_bin, root, repeat) + rec.seconds = double(get_time_usec(t0)) / 1000000.0lf + rec.finished = iso_now() + rec.status = derive_status(rec) + return 1 if (!write_text_file(cfg.out, sprint_json(rec, true))) + let ok = rec.status == RunStatus.ok + to_log(ok ? LOG_INFO : LOG_ERROR, "bench-stand run: {rec.status} - {length(rec.files)} file runs, {failed} failed, {rec.seconds:.0f} s; record {cfg.out}\n") + return ok ? 0 : 1 +} + +//! Runs every file in every runnable lane into `rec.files`, one progress line each; returns how +//! many of those runs failed. A lane the probe marked unavailable contributes no file. +def private benchmark_every_file(var rec : RunRecord&; files : array; filter : string; + lanes : array; bin, aot_bin, root : string; repeat : int) : int { + var limits = RunLimits(repeat = repeat) + var matching = 0 + for (f in files) { + matching++ if (matches_filter(f, filter)) + } + let total = matching * length(lanes) + var done = 0 + var failed = 0 + rec.files |> reserve(total) + for (f in files) { + continue if (!matches_filter(f, filter)) + for (lane in lanes) { + done++ + continue if (rec.lanes[lane] != "ok") + limits.timeout_seconds = f.timeout_seconds + limits.memory_limit_mb = f.memory_limit_mb + var res <- run_bench_file(lane_binary(lane, bin, aot_bin), root, f, lane, limits) + failed++ if (is_failure(res.status)) + to_log(is_failure(res.status) ? LOG_ERROR : LOG_INFO, + "[{done}/{total}] {f.path} {lane}: {res.status} {res.seconds:.1f}s {length(res.samples)} arms{empty(res.message) ? "" : " - " + res.message}\n") + rec.files |> emplace(res) + } + } + return failed +} + +def private verb_report(cfg : Config; suite : SuiteConfig) : int { + if (empty(cfg.runs) || empty(cfg.out_data) || empty(cfg.out_summary)) { + to_log(LOG_ERROR, "bench-stand report: --runs, --out-data and --out-summary are required\n") + return 1 + } + var errors : array + let runs <- load_runs(cfg.runs, errors) + for (e in errors) { + to_log(LOG_WARNING, "bench-stand report: {e}\n") + } + let ds <- build_dataset(runs, suite, empty(cfg.generated) ? iso_now() : cfg.generated, cfg.repo_url) + return 1 if (!write_text_file(cfg.out_data, sprint_json(ds, false))) + let md = render_summary(ds, cfg.site_url) + return 1 if (!write_text_file(cfg.out_summary, md)) + to_log(LOG_INFO, "bench-stand report: {length(runs)} runs, {length(ds.series)} series, {length(ds.changes)} changes -> {cfg.out_data}, {cfg.out_summary}\n") + return summary_exit_code(ds, cfg.fail_on_regression) +} + +[export] +def main() : int { + var r <- parse_args(type) + if (r |> is_err) { + to_log(LOG_ERROR, "error: {r |> unwrap_err}\n") + print_help(get_command_info(type), "bench-stand") + return 1 + } + let cfg <- r |> move_unwrap + if (cfg.help || (cfg.verb != "run" && cfg.verb != "report")) { + print_help(get_command_info(type), "bench-stand") + return cfg.help ? 0 : 1 + } + var error = "" + let suite <- load_suite_config(suite_path_of(cfg), error) + if (!empty(error)) { + to_log(LOG_ERROR, "bench-stand: {error}\n") + return 1 + } + return cfg.verb == "run" ? verb_run(cfg, suite) : verb_report(cfg, suite) +} diff --git a/utils/internal/bench-stand/nightly.sh b/utils/internal/bench-stand/nightly.sh new file mode 100755 index 0000000000..6fd2366c2b --- /dev/null +++ b/utils/internal/bench-stand/nightly.sh @@ -0,0 +1,363 @@ +#!/bin/bash +# bench-stand nightly driver - the box side of the daslang benchmark stand. Runs as the +# `bench` user on the web box; the GitHub workflow (.github/workflows/nightly_bench.yml) +# reaches it over ssh with a forced command (`gate`). Verbs: +# +# start [fail-on-regression] +# fetch, resolve , launch `run` detached; prints the run id +# run the whole night, synchronous: checkout, build, benchmark, report, publish +# follow [run_id] stream the run's log until it finishes; exits with the run's exit code +# status print status.json +# gate ssh forced command: dispatches start/follow/status from SSH_ORIGINAL_COMMAND +# +# Layout under BENCH_STAND_HOME (default /srv/bench-stand): +# src/ clone of the repository - the tree each night checks out, builds and benchmarks +# runs/ one JSON record per night (utils/internal/bench-stand/README.md - data model) +# site/ what Caddy serves at /bench/: viewer, data.json, status.json, summary.md, runs -> ../runs +# logs/ nightly-.log +# last_good/ bin/daslang + daslib + the tool, from the last night whose build passed; renders +# the report when tonight's build fails, so a broken master still publishes a red night +# state/ lock and current run id +# +# Environment knobs (all optional; documented in the README): +# BENCH_STAND_HOME layout root (default /srv/bench-stand) +# BENCH_STAND_REPO_URL commit-link base (default https://github.com/GaijinEntertainment/daScript) +# BENCH_STAND_SITE_URL what the summary links to (default https://daslang.io/bench/) +# BENCH_STAND_JOBS build parallelism (default nproc) +# BENCH_STAND_CMAKE_ARGS extra configure arguments +# BENCH_STAND_BUILD `skip` reuses the existing build - local dry runs only +# BENCH_STAND_AOT `skip` leaves the AOT binary unbuilt; the aot lane then reports itself +# unavailable and the other lanes still measure +# BENCH_STAND_FILTER forwarded as `run --filter` - local dry runs only +# BENCH_STAND_LANES forwarded as `run --lanes` (comma-separated) +# BENCH_STAND_REPEAT forwarded as `run --repeat` +# BENCH_STAND_FAIL_ON_REGRESSION non-empty: `report --fail-on-regression` (exit 2 on a regression); +# `start`'s second word is the same request from a caller with no environment +# +# One variable this script reads that nothing here sets: SSH_ORIGINAL_COMMAND, which the `gate` +# verb parses. An inbound ssh connection whose authorized_keys line forces `nightly.sh gate` +# arrives with the caller's words there and nowhere else. +set -euo pipefail + +HOME_DIR=${BENCH_STAND_HOME:-/srv/bench-stand} +SRC=$HOME_DIR/src +RUNS=$HOME_DIR/runs +SITE=$HOME_DIR/site +LOGS=$HOME_DIR/logs +LAST_GOOD=$HOME_DIR/last_good +STATE=$HOME_DIR/state +TOOL=utils/internal/bench-stand +REPO_URL=${BENCH_STAND_REPO_URL:-https://github.com/GaijinEntertainment/daScript} +SITE_URL=${BENCH_STAND_SITE_URL:-https://daslang.io/bench/} +# Portable, because every verb of this script runs wherever a developer sits: nproc is +# GNU-only, and on a box without it the bare $(nproc) failed the whole script with 127 +# before it ever reached a verb. +cpu_count() { + if command -v nproc >/dev/null 2>&1; then nproc + elif command -v sysctl >/dev/null 2>&1 && sysctl -n hw.ncpu >/dev/null 2>&1; then sysctl -n hw.ncpu + else echo 4 + fi +} +JOBS=${BENCH_STAND_JOBS:-$(cpu_count)} +# The same module set the sql/xml/audio/terminal benchmarks require; dasLLVM for the jit lane. +# Release: RelWithDebInfo arms the C++ allocation tracker whose exit-time leak report turns +# every clean run into exit 1. +CMAKE_ARGS=(-G Ninja -DCMAKE_BUILD_TYPE=Release -DDAS_SQLITE_DISABLED=OFF -DDAS_PUGIXML_DISABLED=OFF + -DDAS_LLVM_DISABLED=OFF -DDAS_GLFW_DISABLED=ON -DDAS_HV_DISABLED=ON) +BUILD_TARGETS=(daslang dasModuleSQLITE dasModulePUGIXML dasModuleAudio dasModuleMinfft dasModuleTerminal dasModuleUnitTest dasModuleLLVM) +# The AOT lane measures native code, so its binary is built, not flagged: test_aot carries the +# benchmark bodies' stubs (the `benchmarks` row of DAS_AOT_SUITES). It is EXCLUDE_FROM_ALL and +# ~1080 TUs, so it is its own target and its own opt-out - a stand running only interp and jit +# should not pay for it. +AOT_TARGET=test_aot + +now_iso() { date -u +%Y-%m-%dT%H:%M:%SZ; } +log() { echo "[$(now_iso)] $*"; } + +# JSON string escaper for meta.json. Multi-byte UTF-8 passes through untouched - it is legal +# inside a JSON string, and stripping it would mangle a commit subject or an author name written +# in any non-ASCII script. +json_str() { + printf '%s' "$1" | tr -d '\r' \ + | LC_ALL=C awk 'BEGIN{ORS=""} {gsub(/\\/,"\\\\"); gsub(/"/,"\\\""); gsub(/\t/,"\\t"); if (NR>1) printf "\\n"; printf "%s", $0}' \ + | tr '\000-\010\013\014\016-\037' ' ' +} + +write_status() { + local state=$1 run_id=$2 started=$3 sha=$4 exit_code=${5:-null} tmp=$SITE/status.json.tmp + printf '{"state":"%s","run_id":"%s","started":"%s","finished":"%s","sha":"%s","exit":%s}\n' \ + "$state" "$run_id" "$started" "$(now_iso)" "$sha" "$exit_code" > "$tmp" + mv "$tmp" "$SITE/status.json" +} + +machine_json() { + local cpu cores mem kernel compiler load + cpu=$(lscpu 2>/dev/null | sed -n 's/^Model name:[[:space:]]*//p' | head -1) + [ -n "$cpu" ] || cpu=$(sysctl -n machdep.cpu.brand_string 2>/dev/null || true) + cores=$(cpu_count) + mem=$(free -g 2>/dev/null | awk '/^Mem:/{print $2}') + [ -n "$mem" ] || mem=$(( $(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824 )) + kernel=$(uname -r) + compiler=$( (c++ --version 2>/dev/null || cc --version 2>/dev/null) | head -1) + load=$(cut -d' ' -f1-3 /proc/loadavg 2>/dev/null || echo "") + printf '{"host":"%s","cpu":"%s","kernel":"%s","compiler":"%s","cores":%s,"mem_gb":%s,"load_start":"%s"}' \ + "$(json_str "$(hostname)")" "$(json_str "$cpu")" "$(json_str "$kernel")" "$(json_str "$compiler")" "${cores:-0}" "${mem:-0}" "$(json_str "$load")" +} + +commit_json() { + local sha date subject author + sha=$(git -C "$SRC" rev-parse HEAD) + date=$(TZ=UTC git -C "$SRC" log -1 --format=%cd --date=iso-strict-local | sed 's/+00:00$/Z/') + subject=$(git -C "$SRC" log -1 --format=%s) + author=$(git -C "$SRC" log -1 --format=%an) + printf '{"sha":"%s","date":"%s","subject":"%s","author":"%s"}' "$sha" "$date" "$(json_str "$subject")" "$(json_str "$author")" +} + +resolve_sha() { + git -C "$SRC" fetch --quiet origin "+refs/heads/*:refs/remotes/origin/*" 2>/dev/null || git -C "$SRC" fetch --quiet origin + git -C "$SRC" rev-parse --verify --quiet "$1^{commit}" \ + || git -C "$SRC" rev-parse --verify --quiet "origin/$1^{commit}" \ + || { echo "cannot resolve '$1' to a commit" >&2; return 1; } +} + +publish_viewer() { + local from=$1 + cp "$from/site/index.html" "$from/site/app.js" "$from/site/style.css" "$SITE/" + ln -sfn ../runs "$SITE/runs" +} + +report_with() { + local root=$1 bin=$2 rc=0 + local extra=() + [ -n "${BENCH_STAND_FAIL_ON_REGRESSION:-}" ] && extra+=(--fail-on-regression) + "$bin" "$root/$TOOL/main.das" -- report --suite "$root/$TOOL/suite.json" --runs "$RUNS" \ + --out-data "$SITE/data.json.tmp" --out-summary "$SITE/summary.md.tmp" \ + --repo-url "$REPO_URL" --site-url "$SITE_URL" "${extra[@]}" || rc=$? + if [ -s "$SITE/data.json.tmp" ] && [ -s "$SITE/summary.md.tmp" ]; then + mv "$SITE/data.json.tmp" "$SITE/data.json" + mv "$SITE/summary.md.tmp" "$SITE/summary.md" + REPORT_RENDERED=1 + fi + rm -f "$SITE/data.json.tmp" "$SITE/summary.md.tmp" + return $rc +} + +run_night() { + local run_id=$1 sha=$2 started rc=0 build_rc=0 build_start build_end + started=$(now_iso) + mkdir -p "$RUNS" "$SITE" "$LOGS" "$STATE" "$LAST_GOOD" + exec 9>"$STATE/lock" + if ! flock -n 9; then + log "another run holds $STATE/lock - refusing to start" + return 3 + fi + echo "$run_id" > "$STATE/current" + write_status running "$run_id" "$started" "$sha" + # a run that dies on an unexpected error still closes status.json, so `follow` returns + trap 'write_status finished "$run_id" "$started" "$sha" 1; log "run $run_id: aborted"' ERR + log "run $run_id: checkout $sha" + git -C "$SRC" checkout --quiet --detach "$sha" + git -C "$SRC" submodule update --quiet --init --recursive || log "submodule update failed - continuing with what is checked out" + # build/, lib/ (the LLVM download + shared modules) and the module cache survive so the rebuild stays incremental + git -C "$SRC" clean -fdxq --exclude=build --exclude=bin --exclude=lib --exclude=.jitted_scripts + + local meta=$STATE/meta.json build_log=$LOGS/build-$run_id.log + build_start=$(date +%s) + if [ "${BENCH_STAND_BUILD:-}" = "skip" ]; then + log "BENCH_STAND_BUILD=skip - reusing $SRC/bin/daslang" + [ -x "$SRC/bin/daslang" ] || build_rc=1 + : > "$build_log" + else + log "build: cmake ${CMAKE_ARGS[*]} ${BENCH_STAND_CMAKE_ARGS:-} -> ${BUILD_TARGETS[*]} (-j $JOBS)" + local launcher=() + command -v ccache >/dev/null && launcher=(-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache) + local targets=("${BUILD_TARGETS[@]}") + if [ "${BENCH_STAND_AOT:-}" = skip ]; then + log "BENCH_STAND_AOT=skip - the aot lane will report itself unavailable" + else + targets+=("$AOT_TARGET") + fi + # shellcheck disable=SC2086 - BENCH_STAND_CMAKE_ARGS must word-split into separate arguments + if ! ( cd "$SRC" && cmake --no-warn-unused-cli -B build "${CMAKE_ARGS[@]}" "${launcher[@]}" ${BENCH_STAND_CMAKE_ARGS:-} \ + && cmake --build build --parallel "$JOBS" --target "${targets[@]}" ) > "$build_log" 2>&1; then + build_rc=1 + fi + fi + build_end=$(date +%s) + local build_status=ok + [ $build_rc -eq 0 ] || build_status=failed + log "build: $build_status in $((build_end - build_start)) s (log $build_log)" + printf '{"run_id":"%s","started":"%s","commit":%s,"machine":%s,"build":{"status":"%s","seconds":%s,"log_tail":"%s"}}\n' \ + "$run_id" "$started" "$(commit_json)" "$(machine_json)" "$build_status" "$((build_end - build_start))" \ + "$(json_str "$(tail -n 40 "$build_log" | cut -c1-400)")" > "$meta" + + local record=$RUNS/$run_id.json + if [ $build_rc -eq 0 ]; then + local run_args=() + [ -n "${BENCH_STAND_FILTER:-}" ] && run_args+=(--filter "$BENCH_STAND_FILTER") + [ -n "${BENCH_STAND_LANES:-}" ] && run_args+=(--lanes "$BENCH_STAND_LANES") + [ -n "${BENCH_STAND_REPEAT:-}" ] && run_args+=(--repeat "$BENCH_STAND_REPEAT") + log "benchmarks: start" + local tool_rc=0 + ( cd "$SRC" && bin/daslang "$TOOL/main.das" -- run --root . --meta "$meta" --out "$record" "${run_args[@]}" ) || tool_rc=$? + log "benchmarks: exit $tool_rc" + rc=$tool_rc + if [ ! -s "$record" ]; then + log "the tool wrote no run record - writing a failed one" + rc=1 + record_failed "$record" "$meta" "the bench-stand tool exited $tool_rc without writing a record; see logs/nightly-$run_id.log on the box" + fi + else + rc=1 + record_failed "$record" "$meta" "the build failed after $((build_end - build_start)) s - logs/build-$run_id.log on the box has the whole log" + fi + + local report_root=$SRC report_bin=$SRC/bin/daslang report_rc=0 + if [ $build_rc -ne 0 ] || [ ! -x "$report_bin" ]; then + if [ -x "$LAST_GOOD/bin/daslang" ]; then + report_root=$LAST_GOOD; report_bin=$LAST_GOOD/bin/daslang + log "report: rendering with last_good's binary" + else + log "report: no binary at all (first night, build failed) - status.json is the only publication" + report_bin="" + fi + fi + if [ -n "$report_bin" ]; then + REPORT_RENDERED=0 + report_with "$report_root" "$report_bin" || report_rc=$? + if [ "$REPORT_RENDERED" = 0 ] && [ "$report_root" != "$LAST_GOOD" ] && [ -x "$LAST_GOOD/bin/daslang" ]; then + # tonight's tree built but its copy of this tool cannot render: the night is still + # published, by the last good tool, so the site shows the red night rather than yesterday's + log "report: tonight's tool failed to render (exit $report_rc) - retrying with last_good" + report_root=$LAST_GOOD; report_bin=$LAST_GOOD/bin/daslang; report_rc=0 + report_with "$report_root" "$report_bin" || report_rc=$? + fi + publish_viewer "$report_root/$TOOL" + log "report: exit $report_rc (rendered=$REPORT_RENDERED)" + fi + # last_good advances only when tonight's own tree rendered the report: a tree whose build + # passed but whose copy of this tool is broken must not become the fallback + if [ $build_rc -eq 0 ] && [ "${REPORT_RENDERED:-0}" = 1 ] && [ "$report_root" = "$SRC" ]; then + mkdir -p "$LAST_GOOD/bin" + cp "$SRC/bin/daslang" "$LAST_GOOD/bin/daslang" + rsync -a --delete "$SRC/daslib/" "$LAST_GOOD/daslib/" + mkdir -p "$LAST_GOOD/$TOOL" + rsync -a --delete "$SRC/$TOOL/" "$LAST_GOOD/$TOOL/" + fi + local final=$rc + [ $final -eq 0 ] && final=$report_rc + trap - ERR + write_status finished "$run_id" "$started" "$sha" "$final" + log "run $run_id: finished, exit $final" + return $final +} + +record_failed() { + # The record of a night that never benchmarked, written by the tool so the run-record schema + # has exactly one writer. `reason` must be non-empty - an empty one would read to the tool as + # "no --failed given" and start a real run. Neither binary available (a first night whose + # build failed) leaves status.json as the night's only publication. + local record=$1 meta=$2 reason=$3 root out + for root in "$SRC" "$LAST_GOOD"; do + [ -x "$root/bin/daslang" ] || continue + out=$("$root/bin/daslang" "$root/$TOOL/main.das" -- run --suite "$root/$TOOL/suite.json" \ + --meta "$meta" --out "$record" --failed "$reason" 2>&1) || true + if [ -s "$record" ]; then + log "failed record written by $root/bin/daslang" + return 0 + fi + # the one path whose failure is otherwise invisible: it runs when everything else already + # went wrong, so its own output is the only thing left to read + log "record_failed: $root/bin/daslang wrote nothing - $(printf '%s' "$out" | tail -n 3 | tr '\n' ' ')" + done + log "no binary could write the failed record - status.json is the night's only publication" + return 1 +} + +verb_start() { + # start [fail-on-regression]. The second word is how the caller asks for a regression + # to fail the night: an ssh forced command carries no environment, so BENCH_STAND_* cannot + # reach here from the workflow - the verb's own argument is the only channel. + local ref=${1:-master} regression=${2:-} sha run_id + mkdir -p "$LOGS" "$STATE" "$SITE" + case "$regression" in + ""|fail-on-regression) ;; + *) echo "start: second argument is 'fail-on-regression' or nothing, got '$regression'" >&2; return 2 ;; + esac + sha=$(resolve_sha "$ref") + run_id="$(date -u +%Y%m%dT%H%M%SZ)-${sha:0:8}" + if [ -e "$STATE/lock" ] && ! flock -n "$STATE/lock" true; then + echo "busy: $(cat "$STATE/current" 2>/dev/null) is still running" >&2 + return 3 + fi + echo "$run_id" > "$STATE/current" + local fail_flag=${BENCH_STAND_FAIL_ON_REGRESSION:-} + [ "$regression" = fail-on-regression ] && fail_flag=1 + BENCH_STAND_FAIL_ON_REGRESSION=$fail_flag \ + setsid nohup bash "$0" run-id "$run_id" "$sha" > "$LOGS/nightly-$run_id.log" 2>&1 < /dev/null & + echo "$run_id" +} + +verb_follow() { + local run_id=${1:-$(cat "$STATE/current" 2>/dev/null || true)} + [ -n "$run_id" ] || { echo "no current run" >&2; return 1; } + local logf=$LOGS/nightly-$run_id.log + local waited=0 + while [ ! -f "$logf" ] && [ $waited -lt 60 ]; do sleep 1; waited=$((waited + 1)); done + if [ ! -f "$logf" ]; then + # a run driven by `run` (no detached log) still answers through status.json + if grep -q "\"run_id\":\"$run_id\"" "$SITE/status.json" 2>/dev/null && grep -q '"state":"finished"' "$SITE/status.json"; then + cat "$SITE/status.json" + return "$(sed -n 's/.*"exit":\([0-9]*\).*/\1/p' "$SITE/status.json")" + fi + echo "no log for $run_id" >&2; return 1 + fi + local pos=0 + while :; do + local size + size=$(stat -c %s "$logf") + if [ "$size" -gt "$pos" ]; then + tail -c +"$((pos + 1))" "$logf" | head -c "$((size - pos))" + pos=$size + fi + if grep -q "\"run_id\":\"$run_id\"" "$SITE/status.json" 2>/dev/null && grep -q '"state":"finished"' "$SITE/status.json"; then + local exit_code + exit_code=$(sed -n 's/.*"exit":\([0-9]*\).*/\1/p' "$SITE/status.json") + echo "--- run $run_id finished with exit ${exit_code:-?} ---" + return "${exit_code:-1}" + fi + sleep 20 + done +} + +verb_gate() { + # SSH forced command: only these verbs, at most two arguments, only path characters in each. + local cmd=${SSH_ORIGINAL_COMMAND:-status} + # shellcheck disable=SC2086 - the forced command's one string is split into words on purpose + set -- $cmd + if [ $# -gt 3 ]; then + echo "refused: too many arguments" >&2; exit 2 + fi + local verb=${1:-status} arg=${2:-} arg2=${3:-} + case "$arg$arg2" in + *[!A-Za-z0-9._/-]*) echo "refused: argument '$arg$arg2' has characters outside A-Za-z0-9._/-" >&2; exit 2 ;; + esac + case "$verb" in + start) verb_start "$arg" "$arg2" ;; + follow) verb_follow "$arg" ;; + status) cat "$SITE/status.json" 2>/dev/null || echo '{"state":"never-run"}' ;; + summary) cat "$SITE/summary.md" 2>/dev/null || echo "no summary yet" ;; + *) echo "refused: unknown verb '$verb'" >&2; exit 2 ;; + esac +} + +case "${1:-}" in + start) verb_start "${2:-master}" "${3:-}" ;; + run) sha=$(resolve_sha "${2:-master}"); run_night "$(date -u +%Y%m%dT%H%M%SZ)-${sha:0:8}" "$sha" ;; + run-id) run_night "$2" "$3" ;; + follow) verb_follow "${2:-}" ;; + status) cat "$SITE/status.json" 2>/dev/null || echo '{"state":"never-run"}' ;; + summary) cat "$SITE/summary.md" 2>/dev/null || echo "no summary yet" ;; + gate) verb_gate ;; + *) echo "usage: nightly.sh start [fail-on-regression] | run | follow [run_id] | status | summary | gate" >&2; exit 2 ;; +esac diff --git a/utils/internal/bench-stand/site/app.js b/utils/internal/bench-stand/site/app.js new file mode 100644 index 0000000000..000f6519a8 --- /dev/null +++ b/utils/internal/bench-stand/site/app.js @@ -0,0 +1,581 @@ +// daslang benchmark stand viewer. Reads data.json (built by bench-stand `report`) and +// status.json (written by nightly.sh) from the same directory and renders everything +// client-side: the latest night, the group index, one chart per benchmark arm, run history. +"use strict"; + +const LANES = ["interp", "jit"]; +const state = { + data: null, + status: null, + runWindow: 365, + lanes: new Set(LANES), + group: "", + search: "", + onlyChanged: false, +}; + +const $ = (sel, root) => (root || document).querySelector(sel); +const el = (tag, cls, text) => { + const e = document.createElement(tag); + if (cls) e.className = cls; + if (text !== undefined) e.textContent = text; + return e; +}; + +function fmtNs(v) { + if (v >= 1e6) return (v / 1e6).toFixed(2) + " ms"; + if (v >= 1e3) return (v / 1e3).toFixed(2) + " us"; + if (v >= 100) return v.toFixed(0) + " ns"; + return v.toFixed(v >= 10 ? 1 : 2) + " ns"; +} +const fmtPct = (x) => (x >= 0 ? "+" : "") + (x * 100).toFixed(1) + "%"; +const fmtSec = (s) => (s >= 3600 ? (s / 3600).toFixed(1) + " h" : s >= 60 ? Math.round(s / 60) + " min" : Math.round(s) + " s"); +const shortSha = (sha) => (sha || "").slice(0, 8); +const dateOf = (iso) => (iso || "").slice(0, 10); +function commitUrl(sha) { + const base = state.data && state.data.repo_url; + return base && sha ? base.replace(/\/$/, "") + "/commit/" + sha : null; +} + +// ---- data loading ---------------------------------------------------------------------------- + +async function load() { + const [dataRes, statusRes] = await Promise.allSettled([ + fetch("data.json", { cache: "no-store" }).then((r) => (r.ok ? r.json() : Promise.reject(new Error("data.json " + r.status)))), + fetch("status.json", { cache: "no-store" }).then((r) => (r.ok ? r.json() : null)), + ]); + state.status = statusRes.status === "fulfilled" ? statusRes.value : null; + if (dataRes.status === "fulfilled") { + state.data = dataRes.value; + } else { + const main = $("main"); + main.replaceChildren(el("p", "notice notice--error", "No data.json yet: the stand has not published a report (" + dataRes.reason.message + ").")); + } + renderStatus(); + if (state.data) renderAll(); +} + +// ---- visible runs (the range filter) --------------------------------------------------------- + +function visibleRunIndices() { + const runs = state.data.runs; + const n = runs.length; + const from = state.runWindow > 0 ? Math.max(0, n - state.runWindow) : 0; + const out = []; + for (let i = from; i < n; i++) out.push(i); + return out; +} + +// ---- status header --------------------------------------------------------------------------- + +function renderStatus() { + const box = $("#status"); + box.replaceChildren(); + const st = state.status; + if (st && st.state === "running") { + const pill = el("span", "pill pill--running", "running " + (st.run_id || "")); + box.append(pill, el("span", "muted", "started " + (st.started || ""))); + } + if (!state.data || state.data.latest < 0) { + if (!st) box.append(el("span", "pill", "no runs")); + return; + } + const run = state.data.runs[state.data.latest]; + box.append(el("span", "pill pill--" + run.status, run.status.replace("_", " "))); + const link = el("a", null, shortSha(run.sha) + " " + run.subject); + link.href = commitUrl(run.sha) || "#"; + link.title = run.date; + box.append(link); + box.append(el("span", "muted", dateOf(run.started) + " on " + run.host + ", build " + fmtSec(run.build_seconds) + ", suite " + fmtSec(run.seconds))); + const ageDays = (Date.now() - Date.parse(run.started)) / 86400000; + if (ageDays > 2 && !(st && st.state === "running")) { + box.append(el("span", "pill pill--stale", "last run " + Math.floor(ageDays) + " days ago")); + } + $("#generated").textContent = "report generated " + (state.data.generated || ""); +} + +// ---- filters --------------------------------------------------------------------------------- + +function wireFilters() { + document.querySelectorAll(".chip[data-range]").forEach((b) => { + b.addEventListener("click", () => { + document.querySelectorAll(".chip[data-range]").forEach((x) => x.classList.remove("is-on")); + b.classList.add("is-on"); + state.runWindow = Number(b.dataset.range); + renderAll(); + }); + }); + document.querySelectorAll("input[data-lane]").forEach((c) => { + c.addEventListener("change", () => { + if (c.checked) state.lanes.add(c.dataset.lane); else state.lanes.delete(c.dataset.lane); + renderAll(); + }); + }); + $("#group").addEventListener("change", (e) => { state.group = e.target.value; renderSeries(); }); + let timer = 0; + $("#search").addEventListener("input", (e) => { + clearTimeout(timer); + timer = setTimeout(() => { state.search = e.target.value.trim().toLowerCase(); renderSeries(); }, 120); + }); + $("#only-changed").addEventListener("change", (e) => { state.onlyChanged = e.target.checked; renderSeries(); }); +} + +function fillGroups() { + const sel = $("#group"); + const keep = sel.value; + sel.replaceChildren(el("option", null, "all groups")); + sel.firstChild.value = ""; + for (const g of state.data.groups) { + const o = el("option", null, g); + o.value = g; + sel.append(o); + } + sel.value = state.data.groups.includes(keep) ? keep : ""; +} + +// ---- latest night ---------------------------------------------------------------------------- + +function renderNight() { + const box = $("#night"); + box.replaceChildren(); + const d = state.data; + if (d.latest < 0) return; + const run = d.runs[d.latest]; + + const fails = el("div", "card"); + if (run.build_status && run.build_status !== "ok") { + fails.append(header("The build failed", "", "nothing was measured")); + const why = el("div", "msg"); + why.append(el("div", null, "after " + fmtSec(run.build_seconds) + "; last lines of the build log:")); + const pre = el("pre", "logtail"); + pre.textContent = run.build_log_tail || "the driver recorded no build log"; + why.append(pre); + fails.append(why); + const rec = el("a", "msg", "run record"); + rec.href = "runs/" + encodeURIComponent(run.id) + ".json"; + fails.append(rec); + box.append(fails); + return; + } + fails.append(header("Failures", run.failures.length)); + fails.append(list(run.failures, (f) => { + const li = el("li"); + li.append(el("span", "tag tag--failure", f.status.replace("_", " ")), el("span", "tag tag--" + f.lane, f.lane), el("code", null, f.path)); + const m = el("div", "msg"); + m.append(el("code", null, f.message)); + li.append(m); + return li; + }, "every file ran")); + const lanes = el("div", "msg"); + for (const lane of Object.keys(run.lanes)) { + if (run.lanes[lane] !== "ok") lanes.append(el("div", null, lane + " lane did not run: " + run.lanes[lane])); + } + if (run.skipped.length) { + const s = el("div", "msg"); + s.append(el("div", null, run.skipped.length + " skipped:")); + for (const f of run.skipped) { + const line = el("div"); + line.append(el("code", null, f.path), " (" + f.lane + "): " + f.message); + s.append(line); + } + lanes.append(s); + } + fails.append(lanes); + const link = el("a", "msg", "run record"); + link.href = "runs/" + encodeURIComponent(run.id) + ".json"; + fails.append(link); + + const regs = d.changes.filter((c) => c.kind === "regression"); + const imps = d.changes.filter((c) => c.kind === "improvement"); + const newOrMissing = d.changes.filter((c) => c.kind === "first_seen" || c.kind === "missing"); + const changeItem = (c) => { + const li = el("li"); + const a = el("a", null, c.id); + a.href = "#" + anchorId(c.id); + li.append(el("span", "tag tag--" + c.lane, c.lane), a); + if (c.kind === "regression" || c.kind === "improvement") { + li.append(el("span", "delta " + (c.change > 0 ? "delta--up" : "delta--down"), fmtPct(c.change))); + li.append(el("span", "muted", fmtNs(c.baseline) + " -> " + fmtNs(c.value) + ", noise " + (c.noise * 100).toFixed(1) + "%")); + } else { + li.append(el("span", "tag", c.kind)); + } + return li; + }; + const regCard = el("div", "card"); + regCard.append(header("Regressions", regs.length, "slower than " + fmtPct(d.regression_threshold) + " and " + d.noise_multiplier + "x noise over the median of " + d.baseline_runs + " runs")); + regCard.append(list(regs, changeItem, "nothing slower")); + const impCard = el("div", "card"); + impCard.append(header("Improvements", imps.length)); + impCard.append(list(imps, changeItem, "nothing faster")); + if (newOrMissing.length) { + impCard.append(header("First seen or missing", newOrMissing.length)); + impCard.append(list(newOrMissing, changeItem, "")); + } + box.append(fails, regCard, impCard); +} + +function header(title, count, sub) { + const h = el("h3"); + h.append(title); + if (count !== "") h.append(el("span", "count", String(count))); + if (sub) h.append(el("span", "muted", sub)); + return h; +} +function list(items, render, emptyText) { + const ul = el("ul"); + if (!items.length) { + if (emptyText) ul.append(el("li", "empty", emptyText)); + return ul; + } + for (const it of items) ul.append(render(it)); + return ul; +} + +// ---- charts ---------------------------------------------------------------------------------- + +const W = 420, H = 150, PAD = { l: 44, r: 54, t: 10, b: 22 }; + +function niceTicks(lo, hi, count) { + if (!(hi > lo)) { hi = lo + 1; } + const span = hi - lo; + const step0 = span / Math.max(1, count); + const mag = Math.pow(10, Math.floor(Math.log10(step0))); + const step = [1, 2, 2.5, 5, 10].map((m) => m * mag).find((s) => s >= step0) || 10 * mag; + const start = Math.floor(lo / step) * step; + const out = []; + for (let v = start; v <= hi + step * 0.5; v += step) out.push(+v.toPrecision(12)); + return out; +} + +// lines: [{lane, points: [{r, v, s}]}], runIdx: visible run indices (x positions) +function drawChart(lines, runIdx, opts) { + const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg"); + svg.setAttribute("viewBox", `0 0 ${W} ${H}`); + svg.setAttribute("class", "chart"); + svg.setAttribute("role", "img"); + svg.setAttribute("aria-label", opts.label); + const xOf = new Map(); + const n = runIdx.length; + runIdx.forEach((ri, i) => xOf.set(ri, PAD.l + (n === 1 ? (W - PAD.l - PAD.r) / 2 : (i * (W - PAD.l - PAD.r)) / (n - 1)))); + let lo = Infinity, hi = -Infinity; + for (const ln of lines) for (const p of ln.points) { + if (!xOf.has(p.r)) continue; + const top = p.v * (1 + (p.s || 0)); + if (p.v < lo) lo = p.v; + if (top > hi) hi = top; + } + if (!isFinite(lo)) { lo = 0; hi = 1; } + if (opts.zeroBased) lo = Math.min(lo, 0); + const pad = (hi - lo) * 0.12 || hi * 0.1 || 1; + lo = Math.max(opts.zeroBased ? 0 : -Infinity, lo - pad); + hi = hi + pad; + const ticks = niceTicks(lo, hi, 4); + lo = Math.min(lo, ticks[0]); + hi = Math.max(hi, ticks[ticks.length - 1]); + const yOf = (v) => PAD.t + (H - PAD.t - PAD.b) * (1 - (v - lo) / (hi - lo)); + const svgEl = (tag, attrs) => { + const e = document.createElementNS("http://www.w3.org/2000/svg", tag); + for (const k in attrs) e.setAttribute(k, attrs[k]); + return e; + }; + for (const tv of ticks) { + if (tv < lo || tv > hi) continue; + const y = yOf(tv); + svg.append(svgEl("line", { class: "grid-line", x1: PAD.l, x2: W - PAD.r, y1: y, y2: y })); + const t = svgEl("text", { x: PAD.l - 6, y: y + 3.5, "text-anchor": "end" }); + t.textContent = opts.fmt(tv); + svg.append(t); + } + svg.append(svgEl("line", { class: "axis-line", x1: PAD.l, x2: W - PAD.r, y1: H - PAD.b, y2: H - PAD.b })); + const runs = state.data.runs; + const labelEvery = Math.max(1, Math.ceil(n / 5)); + runIdx.forEach((ri, i) => { + if (i % labelEvery !== 0 && i !== n - 1) return; + const t = svgEl("text", { x: xOf.get(ri), y: H - 6, "text-anchor": i === n - 1 ? "end" : i === 0 ? "start" : "middle" }); + t.textContent = dateOf(runs[ri].date).slice(5); + svg.append(t); + }); + if (opts.failedRuns) { + for (const ri of opts.failedRuns) { + if (!xOf.has(ri)) continue; + svg.append(svgEl("rect", { class: "fail-mark", x: xOf.get(ri) - 2, y: H - PAD.b - 5, width: 4, height: 5 })); + } + } + for (const ln of lines) { + const pts = ln.points.filter((p) => xOf.has(p.r)); + if (!pts.length) continue; + if (opts.band) { + let d = ""; + for (const p of pts) d += (d ? "L" : "M") + xOf.get(p.r).toFixed(1) + "," + yOf(p.v * (1 + p.s)).toFixed(1); + for (let i = pts.length - 1; i >= 0; i--) d += "L" + xOf.get(pts[i].r).toFixed(1) + "," + yOf(pts[i].v).toFixed(1); + svg.append(svgEl("path", { class: "band band--" + ln.lane, d: d + "Z" })); + } + let d = ""; + let prev = -2; + for (const p of pts) { + const gap = runIdx.indexOf(p.r) - prev > 1; + d += (gap ? "M" : "L") + xOf.get(p.r).toFixed(1) + "," + yOf(p.v).toFixed(1); + prev = runIdx.indexOf(p.r); + } + svg.append(svgEl("path", { class: "series series--" + ln.lane, d })); + if (pts.length === 1) svg.append(svgEl("circle", { class: "marker marker--" + ln.lane, cx: xOf.get(pts[0].r), cy: yOf(pts[0].v), r: 4 })); + const last = pts[pts.length - 1]; + const lbl = svgEl("text", { class: "end-label", x: W - PAD.r + 6, y: yOf(last.v) + 3.5 }); + lbl.textContent = opts.fmt(last.v); + svg.append(lbl); + } + attachHover(svg, svgEl, lines, runIdx, opts, xOf, yOf); + return svg; +} + +// The hover layer: a crosshair that snaps to the nearest run, one tooltip listing every lane at +// that run, and a click that opens the commit. The hit rectangle spans the plot and overhangs it, +// so a reader aims at a date rather than at a 2px line. +function attachHover(svg, svgEl, lines, runIdx, opts, xOf, yOf) { + const runs = state.data.runs; + const cross = svgEl("line", { class: "crosshair", x1: 0, x2: 0, y1: PAD.t, y2: H - PAD.b, visibility: "hidden" }); + svg.append(cross); + const markers = lines.map((ln) => { + const m = svgEl("circle", { class: "marker marker--" + ln.lane, r: 4, visibility: "hidden" }); + svg.append(m); + return m; + }); + const hit = svgEl("rect", { class: "hit", x: PAD.l - 10, y: 0, width: W - PAD.l - PAD.r + 20, height: H }); + svg.append(hit); + const tooltip = $("#tooltip"); + const nearestRun = (evt) => { + const rect = svg.getBoundingClientRect(); + const px = ((evt.clientX - rect.left) / rect.width) * W; + let best = -1, bestD = Infinity; + for (const ri of runIdx) { + const dx = Math.abs(xOf.get(ri) - px); + if (dx < bestD) { bestD = dx; best = ri; } + } + return best; + }; + const show = (evt) => { + const best = nearestRun(evt); + if (best < 0) return; + const x = xOf.get(best); + cross.setAttribute("x1", x); cross.setAttribute("x2", x); cross.setAttribute("visibility", "visible"); + tooltip.replaceChildren(); + const run = runs[best]; + tooltip.append(el("div", "tt-head", dateOf(run.date) + " " + shortSha(run.sha) + " " + run.subject)); + lines.forEach((ln, i) => { + const p = ln.points.find((q) => q.r === best); + if (!p) { markers[i].setAttribute("visibility", "hidden"); return; } + markers[i].setAttribute("cx", x); markers[i].setAttribute("cy", yOf(p.v)); markers[i].setAttribute("visibility", "visible"); + const row = el("div", "tt-row"); + row.append(el("span", "tt-key tt-key--" + ln.lane), el("span", "tt-val", opts.fmt(p.v)), el("span", null, ln.lane)); + if (p.s !== undefined && opts.band) row.append(el("span", "tt-sub", "spread " + (p.s * 100).toFixed(1) + "%")); + tooltip.append(row); + }); + if (opts.failedRuns && opts.failedRuns.has(best)) tooltip.append(el("div", "tt-sub", "this file failed that night")); + tooltip.hidden = false; + const tw = tooltip.offsetWidth, th = tooltip.offsetHeight; + let left = evt.clientX + 14, top = evt.clientY + 14; + if (left + tw > window.innerWidth - 8) left = evt.clientX - tw - 14; + if (top + th > window.innerHeight - 8) top = evt.clientY - th - 14; + tooltip.style.left = left + "px"; tooltip.style.top = top + "px"; + }; + const hide = () => { + cross.setAttribute("visibility", "hidden"); + markers.forEach((m) => m.setAttribute("visibility", "hidden")); + tooltip.hidden = true; + }; + hit.addEventListener("pointermove", show); + hit.addEventListener("pointerleave", hide); + hit.addEventListener("click", (evt) => { + const best = nearestRun(evt); + const url = best >= 0 ? commitUrl(runs[best].sha) : null; + if (url) window.open(url, "_blank", "noopener"); + }); +} + +function legend(lanes) { + const lg = el("div", "legend"); + for (const lane of lanes) lg.append(el("span", "legend--" + lane, lane)); + return lg; +} + +function tableView(lines, runIdx, fmt) { + const t = el("table", "mini"); + const thead = el("tr"); + thead.append(el("th", null, "run")); + for (const ln of lines) thead.append(el("th", "num", ln.lane)); + t.append(thead); + const shown = runIdx.slice(-12).reverse(); + for (const ri of shown) { + const tr = el("tr"); + const run = state.data.runs[ri]; + tr.append(el("td", null, dateOf(run.date) + " " + shortSha(run.sha))); + for (const ln of lines) { + const p = ln.points.find((q) => q.r === ri); + tr.append(el("td", "num", p ? fmt(p.v) : "-")); + } + t.append(tr); + } + return t; +} + +function chartCard(title, fileLabel, lines, runIdx, opts) { + const card = el("div", "card chart-card"); + if (opts.anchor) card.id = opts.anchor; + const head = el("div", "chart-card__head"); + const ttl = el("div", "chart-card__title"); + if (fileLabel) ttl.append(el("span", "file", fileLabel + " ")); + ttl.append(title); + head.append(ttl); + const tools = el("div", "chart-card__tools"); + if (opts.badges) for (const b of opts.badges) tools.append(b); + const toggle = el("button", null, "table"); + tools.append(toggle); + head.append(tools); + card.append(head); + const svg = drawChart(lines, runIdx, opts); + card.append(svg); + if (lines.length > 1) card.append(legend(lines.map((l) => l.lane))); + let table = null; + toggle.addEventListener("click", () => { + if (table) { table.remove(); table = null; toggle.textContent = "table"; return; } + table = tableView(lines, runIdx, opts.fmt); + card.append(table); + toggle.textContent = "chart only"; + }); + return card; +} + +const anchorId = (id) => "b-" + id.replace(/[^A-Za-z0-9_-]/g, "_"); + +function renderIndex() { + const box = $("#index"); + box.replaceChildren(); + const runIdx = visibleRunIndices(); + const byGroup = new Map(); + for (const ix of state.data.index) { + if (!state.lanes.has(ix.lane)) continue; + if (!byGroup.has(ix.group)) byGroup.set(ix.group, []); + byGroup.get(ix.group).push({ lane: ix.lane, points: ix.runs.map((r, i) => ({ r, v: ix.value[i], s: 0 })) }); + } + if (!byGroup.size) { box.append(el("p", "notice", "The index needs at least three runs per series.")); return; } + for (const [group, lines] of [...byGroup.entries()].sort((a, b) => a[0].localeCompare(b[0]))) { + lines.sort((a, b) => LANES.indexOf(a.lane) - LANES.indexOf(b.lane)); + box.append(chartCard(group, "", lines, runIdx, { label: "index of " + group, fmt: (v) => v.toFixed(0), band: false, zeroBased: false })); + } +} + +function failedRunsByFile() { + const out = new Map(); + state.data.runs.forEach((run, ri) => { + for (const f of run.failures) { + const key = f.lane + "\t" + f.path.replace(/\.das$/, ""); + if (!out.has(key)) out.set(key, new Set()); + out.get(key).add(ri); + } + }); + return out; +} + +function renderSeries() { + const box = $("#series"); + box.replaceChildren(); + const d = state.data; + const runIdx = visibleRunIndices(); + const visible = new Set(runIdx); + const changeOf = new Map(); + for (const c of d.changes) changeOf.set(c.lane + "\t" + c.id, c); + const failed = failedRunsByFile(); + const arms = new Map(); + for (const s of d.series) { + if (!state.lanes.has(s.lane)) continue; + if (state.group && s.group !== state.group) continue; + if (state.search && !s.id.toLowerCase().includes(state.search)) continue; + const ch = changeOf.get(s.lane + "\t" + s.id); + if (state.onlyChanged && !(ch && (ch.kind === "regression" || ch.kind === "improvement"))) continue; + if (!s.runs.some((r) => visible.has(r))) continue; + if (!arms.has(s.id)) arms.set(s.id, { group: s.group, file: s.file, lines: [], changes: [] }); + const arm = arms.get(s.id); + arm.lines.push({ lane: s.lane, points: s.runs.map((r, i) => ({ r, v: s.ns[i], s: s.spread[i] })) }); + if (ch) arm.changes.push(ch); + } + $("#series-count").textContent = arms.size + " of " + new Set(d.series.map((s) => s.id)).size + " arms"; + if (!arms.size) { box.append(el("p", "notice", "Nothing matches the filters.")); return; } + const byGroup = new Map(); + for (const [id, arm] of arms) { + if (!byGroup.has(arm.group)) byGroup.set(arm.group, []); + byGroup.get(arm.group).push([id, arm]); + } + for (const [group, entries] of [...byGroup.entries()].sort((a, b) => a[0].localeCompare(b[0]))) { + const section = el("div", "group"); + const h = el("h3"); + h.append(group, el("span", "muted", entries.length + " arms")); + section.append(h); + const grid = el("div", "grid grid--series"); + entries.sort((a, b) => a[0].localeCompare(b[0])); + for (const [id, arm] of entries) { + arm.lines.sort((a, b) => LANES.indexOf(a.lane) - LANES.indexOf(b.lane)); + const badges = arm.changes.filter((c) => c.kind === "regression" || c.kind === "improvement") + .map((c) => el("span", "tag tag--" + c.kind, c.lane + " " + fmtPct(c.change))); + const failedRuns = new Set(); + for (const ln of arm.lines) for (const ri of failed.get(ln.lane + "\t" + arm.file) || []) failedRuns.add(ri); + const title = id.slice(arm.file.length + 1); + grid.append(chartCard(title, arm.file, arm.lines, runIdx, { + label: id, fmt: fmtNs, band: true, zeroBased: true, anchor: anchorId(id), badges, failedRuns, + })); + } + section.append(grid); + box.append(section); + } +} + +function renderRuns() { + const table = $("#runs"); + table.replaceChildren(); + const head = el("tr"); + for (const [txt, cls] of [["date", ""], ["commit", ""], ["status", ""], ["lanes", ""], ["build", "num"], ["suite", "num"], ["ok", "num"], ["failed", "num"], ["regressions", "num"], ["improvements", "num"], ["record", ""]]) { + head.append(el("th", cls, txt)); + } + table.append(head); + const runIdx = visibleRunIndices().slice().reverse(); + for (const ri of runIdx) { + const run = state.data.runs[ri]; + const tr = el("tr"); + tr.append(el("td", null, dateOf(run.started))); + const c = el("td"); + const a = el("a", null, shortSha(run.sha)); + a.href = commitUrl(run.sha) || "#"; + a.title = run.subject; + c.append(a, " ", el("span", "muted", run.subject.length > 60 ? run.subject.slice(0, 60) + "..." : run.subject)); + tr.append(c); + const s = el("td"); + s.append(el("span", "pill pill--" + run.status, run.status.replace("_", " "))); + tr.append(s); + const lanes = el("td"); + for (const lane of Object.keys(run.lanes)) { + const t = el("span", "tag tag--" + lane, lane); + if (run.lanes[lane] !== "ok") { t.textContent = lane + " off"; t.title = run.lanes[lane]; t.className = "tag"; } + lanes.append(t, " "); + } + tr.append(lanes); + tr.append(el("td", "num", fmtSec(run.build_seconds)), el("td", "num", fmtSec(run.seconds))); + tr.append(el("td", "num", String(run.files_ok)), el("td", "num", String(run.failures.length))); + tr.append(el("td", "num", ri === state.data.latest ? String(run.regressions) : ""), el("td", "num", ri === state.data.latest ? String(run.improvements) : "")); + const rec = el("td"); + const link = el("a", null, "json"); + link.href = "runs/" + encodeURIComponent(run.id) + ".json"; + rec.append(link); + tr.append(rec); + table.append(tr); + } +} + +function renderAll() { + fillGroups(); + renderNight(); + renderIndex(); + renderSeries(); + renderRuns(); +} + +wireFilters(); +load(); diff --git a/utils/internal/bench-stand/site/index.html b/utils/internal/bench-stand/site/index.html new file mode 100644 index 0000000000..119d7275ec --- /dev/null +++ b/utils/internal/bench-stand/site/index.html @@ -0,0 +1,66 @@ + + + + + +daslang benchmark stand + + + + +
+
+ daslang +

benchmark stand

+
+
+
+ +
+
+
+ + + + +
+
+ + +
+
+ + +
+
+ +
+
+ +
+ +
+

Group index geometric mean of each series against its own first runs, 100 = reference

+
+
+ +
+

Benchmarks

+
+
+ +
+

Run history

+
+
+
+ +
+ Every point is the minimum ns/op over the night's repeats of a dastest --bench run; the tooltip shows the spread. Source: utils/internal/bench-stand in the daslang repository. + +
+ + + + + diff --git a/utils/internal/bench-stand/site/style.css b/utils/internal/bench-stand/site/style.css new file mode 100644 index 0000000000..37424f3dab --- /dev/null +++ b/utils/internal/bench-stand/site/style.css @@ -0,0 +1,181 @@ +:root { + color-scheme: light; + --page: #f9f9f7; + --surface: #fcfcfb; + --ink: #0b0b0b; + --ink-2: #52514e; + --muted: #898781; + --grid: #e1e0d9; + --axis: #c3c2b7; + --border: rgba(11, 11, 11, 0.10); + --interp: #2a78d6; + --jit: #eb6834; + --good: #0ca30c; + --good-text: #006300; + --warning: #fab219; + --serious: #ec835a; + --critical: #d03b3b; + --chip-on: #e8eef9; +} +@media (prefers-color-scheme: dark) { + :root { + color-scheme: dark; + --page: #0d0d0d; + --surface: #1a1a19; + --ink: #ffffff; + --ink-2: #c3c2b7; + --muted: #898781; + --grid: #2c2c2a; + --axis: #383835; + --border: rgba(255, 255, 255, 0.10); + --interp: #3987e5; + --jit: #d95926; + --good-text: #0ca30c; + --chip-on: #22314a; + } +} + +* { box-sizing: border-box; } +html, body { margin: 0; } +body { + background: var(--page); + color: var(--ink); + font: 14px/1.45 system-ui, -apple-system, "Segoe UI", sans-serif; +} +a { color: inherit; } +code { font: 12px/1.4 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; } +h1, h2, h3 { font-weight: 600; margin: 0; } +h2 { font-size: 16px; margin: 0 0 12px; } +.muted { color: var(--muted); font-weight: 400; font-size: 13px; } + +.top { + display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: 12px; + padding: 18px 24px 8px; +} +.top__title { display: flex; align-items: baseline; gap: 10px; } +.top__brand { text-decoration: none; color: var(--ink-2); font-weight: 600; } +.top__title h1 { font-size: 20px; } +.top__status { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; color: var(--ink-2); } + +main { padding: 0 24px 32px; max-width: 1500px; } +.section { margin-top: 28px; } + +.filters { + display: flex; flex-wrap: wrap; gap: 18px; align-items: center; + padding: 10px 0 14px; border-bottom: 1px solid var(--grid); +} +.filters__group { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; } +.chip { + background: var(--surface); color: var(--ink-2); border: 1px solid var(--border); + border-radius: 999px; padding: 4px 10px; cursor: pointer; font: inherit; font-size: 13px; +} +.chip.is-on { background: var(--chip-on); color: var(--ink); border-color: transparent; font-weight: 600; } +.lane { display: inline-flex; align-items: center; gap: 6px; cursor: pointer; padding: 4px 8px; } +.lane input { accent-color: var(--ink-2); } +.lane--interp input { accent-color: var(--interp); } +.lane--jit input { accent-color: var(--jit); } +.select select, .search { + font: inherit; font-size: 13px; color: var(--ink); background: var(--surface); + border: 1px solid var(--border); border-radius: 6px; padding: 4px 8px; +} +.search { width: 260px; } + +.pill { + display: inline-flex; align-items: center; gap: 6px; padding: 2px 10px; border-radius: 999px; + font-size: 13px; font-weight: 600; background: var(--surface); border: 1px solid var(--border); +} +.pill::before { content: ""; width: 8px; height: 8px; border-radius: 50%; background: var(--muted); } +.pill--ok::before { background: var(--good); } +.pill--bench_failed::before, .pill--build_failed::before { background: var(--critical); } +.pill--running::before { background: var(--warning); } +.pill--stale::before { background: var(--serious); } + +.night { margin-top: 18px; display: grid; gap: 14px; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); } +.card { + background: var(--surface); border: 1px solid var(--border); border-radius: 10px; padding: 12px 14px; +} +.card h3 { font-size: 14px; margin-bottom: 8px; display: flex; gap: 8px; align-items: baseline; } +.card h3 .count { color: var(--muted); font-weight: 400; } +.card ul { margin: 0; padding: 0; list-style: none; } +.card li { padding: 4px 0; border-top: 1px solid var(--grid); display: flex; gap: 8px; flex-wrap: wrap; align-items: baseline; } +.card li:first-child { border-top: 0; } +.card .empty { color: var(--muted); } +.tag { font-size: 12px; padding: 0 6px; border-radius: 4px; border: 1px solid var(--border); color: var(--ink-2); } +.tag--regression { color: var(--critical); border-color: var(--critical); } +.tag--improvement { color: var(--good-text); border-color: var(--good-text); } +.tag--failure { color: var(--critical); border-color: var(--critical); } +.tag--interp { color: var(--interp); border-color: var(--interp); } +.tag--jit { color: var(--jit); border-color: var(--jit); } +.delta { font-variant-numeric: tabular-nums; font-weight: 600; } +.delta--up { color: var(--critical); } +.delta--down { color: var(--good-text); } +.msg { color: var(--ink-2); font-size: 13px; } +.msg code { white-space: pre-wrap; word-break: break-word; } +pre.logtail { + margin: 6px 0 0; padding: 8px 10px; max-height: 220px; overflow: auto; + background: var(--page); border: 1px solid var(--grid); border-radius: 6px; + font: 12px/1.45 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; color: var(--ink-2); + white-space: pre-wrap; word-break: break-word; +} + +.grid { display: grid; gap: 14px; } +.grid--index { grid-template-columns: repeat(auto-fill, minmax(320px, 1fr)); } +.group { margin-top: 18px; } +.group > h3 { font-size: 15px; margin: 0 0 8px; display: flex; gap: 10px; align-items: baseline; } +.group > h3 .muted { font-size: 13px; } +.grid--series { grid-template-columns: repeat(auto-fill, minmax(360px, 1fr)); } + +.chart-card { padding: 10px 12px 6px; } +.chart-card__head { display: flex; justify-content: space-between; gap: 8px; align-items: baseline; } +.chart-card__title { font-size: 13px; font-weight: 600; word-break: break-all; } +.chart-card__title .file { color: var(--muted); font-weight: 400; } +.chart-card__tools { display: flex; gap: 6px; align-items: center; flex-shrink: 0; } +.chart-card__tools button { + background: none; border: 1px solid var(--border); color: var(--ink-2); border-radius: 6px; + font: inherit; font-size: 12px; padding: 1px 7px; cursor: pointer; +} +.legend { display: flex; gap: 12px; font-size: 12px; color: var(--ink-2); margin: 4px 0 0; } +.legend span::before { content: ""; display: inline-block; width: 14px; height: 2px; vertical-align: middle; margin-right: 5px; border-radius: 1px; background: var(--muted); } +.legend .legend--interp::before { background: var(--interp); } +.legend .legend--jit::before { background: var(--jit); } +svg.chart { display: block; width: 100%; height: auto; overflow: visible; touch-action: pan-y; } +svg.chart text { fill: var(--muted); font-size: 11px; font-variant-numeric: tabular-nums; } +svg.chart .grid-line { stroke: var(--grid); stroke-width: 1; } +svg.chart .axis-line { stroke: var(--axis); stroke-width: 1; } +svg.chart .series { fill: none; stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; } +svg.chart .series--interp { stroke: var(--interp); } +svg.chart .series--jit { stroke: var(--jit); } +svg.chart .band { opacity: 0.10; } +svg.chart .band--interp { fill: var(--interp); } +svg.chart .band--jit { fill: var(--jit); } +svg.chart .end-label { fill: var(--ink-2); font-weight: 600; } +svg.chart .crosshair { stroke: var(--axis); stroke-width: 1; } +svg.chart .marker { stroke: var(--surface); stroke-width: 2; } +svg.chart .marker--interp { fill: var(--interp); } +svg.chart .marker--jit { fill: var(--jit); } +svg.chart .fail-mark { fill: var(--critical); } +svg.chart .hit { fill: transparent; cursor: crosshair; } + +.table-wrap { overflow-x: auto; } +table { border-collapse: collapse; width: 100%; font-size: 13px; } +th, td { text-align: left; padding: 6px 8px; border-top: 1px solid var(--grid); vertical-align: top; } +th { color: var(--muted); font-weight: 500; border-top: 0; } +td.num, th.num { text-align: right; font-variant-numeric: tabular-nums; } +table.mini { font-size: 12px; margin-top: 6px; } + +.tooltip { + position: fixed; z-index: 10; pointer-events: none; max-width: 360px; + background: var(--surface); color: var(--ink); border: 1px solid var(--border); + border-radius: 8px; padding: 8px 10px; box-shadow: 0 6px 24px rgba(0, 0, 0, 0.12); font-size: 12px; +} +.tooltip .tt-head { color: var(--ink-2); margin-bottom: 4px; } +.tooltip .tt-row { display: flex; gap: 8px; align-items: baseline; } +.tooltip .tt-key { display: inline-block; width: 14px; height: 2px; border-radius: 1px; background: var(--muted); } +.tooltip .tt-key--interp { background: var(--interp); } +.tooltip .tt-key--jit { background: var(--jit); } +.tooltip .tt-val { font-weight: 600; font-variant-numeric: tabular-nums; } +.tooltip .tt-sub { color: var(--muted); } + +.foot { display: flex; flex-wrap: wrap; justify-content: space-between; gap: 12px; padding: 16px 24px 28px; color: var(--muted); font-size: 12px; border-top: 1px solid var(--grid); } +.notice { padding: 16px; color: var(--ink-2); } +.notice--error { color: var(--critical); } diff --git a/utils/internal/bench-stand/suite.json b/utils/internal/bench-stand/suite.json new file mode 100644 index 0000000000..75afb1bccc --- /dev/null +++ b/utils/internal/bench-stand/suite.json @@ -0,0 +1,32 @@ +{ + "root": "benchmarks", + "exclude": [ + "**/tests/**", + "**/_*.das" + ], + "lanes": [ + "interp", + "jit", + "aot" + ], + "repeat": 3, + "timeout_seconds": 900, + "memory_limit_mb": 4096, + "regression_threshold": 0.1, + "noise_multiplier": 3.0, + "baseline_runs": 7, + "files": { + "core/array/test01.das": { + "skip": "allocates ~19 GB under persistent_heap and gets OOM-killed; unskip once the benchmark is fixed" + } + }, + "lane_excludes": { + "aot": { + "audio/**": "needs dasAudio's own AOT half, which the benchmark AOT set does not carry", + "terminal/**": "needs dasTerminal's own AOT half, which the benchmark AOT set does not carry", + "sql/**": "needs dasSQLITE and dasPUGIXML's own AOT halves, which the benchmark AOT set does not carry", + "micro/*_shapes.das": "the XML-source shapes need dasPUGIXML's own AOT half", + "core/math/scalar_crt.das": "requires the UnitTest module, which the benchmark AOT set does not carry" + } + } +} \ No newline at end of file diff --git a/utils/internal/bench-stand/test_bench_cli.das b/utils/internal/bench-stand/test_bench_cli.das new file mode 100644 index 0000000000..5414511b35 --- /dev/null +++ b/utils/internal/bench-stand/test_bench_cli.das @@ -0,0 +1,321 @@ +options gen2 + +require dastest/testing_boost public +require bench_history +require bench_runner +require daslib/fio +require daslib/json_boost +require strings +require daslib/strings_boost + +//! The two programs the box drives: `main.das`'s verbs and `nightly.sh`'s ssh gate. Both are +//! spawned rather than called, because their contract with the driver is argv in, exit code and +//! written file out. Every arm skips where it has no shell. + +let private TOOL_DIR = "utils/internal/bench-stand" + +def private has_shell() : bool { + return fexist("/bin/sh") +} + +def private with_temp_dir(prefix : string; blk : block<(dir : string) : void>) { + var err = "" + let dir = create_temp_directory(prefix, err) + verify(!empty(dir), "temp dir created") + invoke(blk, dir) + rmdir_rec(dir) +} + +//! A `meta.json` of the shape `nightly.sh` writes, with the build's own verdict. +def private write_meta(dir : string; build_status : string) : string { + let path = path_join(dir, "meta.json") + fwrite(path, "\{\"run_id\":\"20260908T030000Z-abcdef12\",\"started\":\"2026-09-08T03:00:00Z\"," + + "\"commit\":\{\"sha\":\"abcdef1234567890\",\"date\":\"2026-09-08T01:00:00Z\",\"subject\":\"a commit\",\"author\":\"dev\"\}," + + "\"machine\":\{\"host\":\"box\",\"cores\":4\}," + + "\"build\":\{\"status\":\"{build_status}\",\"seconds\":12,\"log_tail\":\"ninja: build stopped\"\}\}") + return path +} + +//! A stand-in daslang: prints one benchmark sample, writes the report the runner asked for, and +//! answers the JIT probe, so `main.das run` completes without a real build. +def private write_fake_daslang(dir : string) : string { + let path = path_join(dir, "fake_daslang.sh") + fwrite(path, "#!/bin/sh\n" + + "REPORT=\"\"\n" + + "for a in \"$@\"; do case \"$PREV\" in --json-file) REPORT=$a;; esac; PREV=$a; done\n" + + "case \"$*\" in *jit_probe*|*-jit*.das) echo jit-probe-ok; exit 0;; esac\n" + + "echo '\{\"name\":\"b\",\"sub_name\":\"arm\",\"n\":10,\"time_ns\":1000,\"allocs\":0,\"heap_bytes\":0,\"string_allocs\":0,\"string_heap_bytes\":0,\"func_type\":\"INTERP\"\}'\n" + + "[ -n \"$REPORT\" ] && echo '\{\"file\":\"x\",\"total\":0,\"passed\":0,\"failed\":0,\"errors\":0,\"skipped\":0,\"success\":true,\"time_usec\":1,\"tests\":[\{\"name\":\"b\",\"passed\":true,\"skipped\":false,\"time\":1,\"messages\":[],\"location\":\"\"\}]\}' > \"$REPORT\"\n" + + "exit 0\n") + var out = "" + run_and_capture(["/bin/chmod", "755", path], out) + return path +} + +def private write_suite(dir : string; lanes : string) : string { + let path = path_join(dir, "suite.json") + fwrite(path, "\{ \"root\": \"benchmarks\", \"exclude\": [\"**/_*.das\"], \"lanes\": [{lanes}], \"repeat\": 1, \"baseline_runs\": 3 \}") + return path +} + +def private seed_benchmarks(dir : string) { + mkdir_rec(path_join(dir, "benchmarks/core")) + fwrite(path_join(dir, "benchmarks/core/one.das"), "options gen2\n") + fwrite(path_join(dir, "benchmarks/core/two.das"), "options gen2\n") +} + +//! Runs the tool under the host binary and returns its exit code; `output` gets stdout+stderr. +def private run_tool(args : array; var output : string&) : int { + var argv <- ["bin/daslang", path_join(TOOL_DIR, "main.das"), "--"] + argv |> push_from(args) + return run_and_capture(argv, output, 240.0) +} + +[test] +def test_verb_run(t : T?) { + if (!has_shell()) { + t |> skip("needs /bin/sh") + return + } + with_temp_dir("bench_cli_") $(dir) { + seed_benchmarks(dir) + let fake = write_fake_daslang(dir) + let suite = write_suite(dir, "\"interp\"") + t |> run("a clean run records one file per lane and exits 0") @(tt : T?) { + let meta = write_meta(dir, "ok") + let record = path_join(dir, "run.json") + var out = "" + let code = run_tool(["run", "--root", dir, "--bin", fake, "--suite", suite, + "--meta", meta, "--out", record], out) + tt |> equal(code, 0, "exit 0 when every file ran: {out}") + var rec = RunRecord() + tt |> success(sscan_json(fread(record), rec), "the record parses") + tt |> equal(rec.run_id, "20260908T030000Z-abcdef12") + tt |> equal(rec.status, RunStatus.ok) + tt |> equal(length(rec.files), 2, "one file per lane, two benchmark files") + tt |> equal(rec.lanes?["interp"] ?? "", "ok") + tt |> success(rec.seconds >= 0.0lf, "the suite's duration is recorded") + tt |> success(!empty(rec.finished), "the finish stamp is written") + } + t |> run("a meta with no build section is a night that ran, not a night whose build failed") @(tt : T?) { + let bare = path_join(dir, "bare_meta.json") + fwrite(bare, "\{\"run_id\":\"local-1\",\"started\":\"2026-09-09T10:00:00Z\"\}") + let record = path_join(dir, "bare.json") + var out = "" + let code = run_tool(["run", "--root", dir, "--bin", fake, "--suite", suite, + "--meta", bare, "--out", record], out) + tt |> equal(code, 0, "a hand-driven local run exits 0: {out}") + var rec = RunRecord() + tt |> success(sscan_json(fread(record), rec), "the record parses") + tt |> equal(rec.status, RunStatus.ok) + tt |> equal(rec.build.status, "ok", "an absent build section reads as a build that was fine") + } + t |> run("--filter narrows the run to the files whose path contains it") @(tt : T?) { + let meta = write_meta(dir, "ok") + let record = path_join(dir, "filtered.json") + var out = "" + run_tool(["run", "--root", dir, "--bin", fake, "--suite", suite, "--meta", meta, + "--out", record, "--filter", "one"], out) + var rec = RunRecord() + tt |> success(sscan_json(fread(record), rec), "the record parses") + tt |> equal(length(rec.files), 1) + tt |> equal(rec.files[0].path, "core/one.das") + } + t |> run("--lanes overrides the suite, and an unavailable lane runs no file") @(tt : T?) { + let meta = write_meta(dir, "ok") + let record = path_join(dir, "lanes.json") + var out = "" + run_tool(["run", "--root", dir, "--bin", "/nonexistent/daslang", "--suite", suite, + "--meta", meta, "--out", record, "--lanes", "jit"], out) + var rec = RunRecord() + tt |> success(sscan_json(fread(record), rec), "the record parses") + tt |> success(!key_exists(rec.lanes, "interp"), "the suite's lane list was overridden") + tt |> success(find(rec.lanes?["jit"] ?? "", "no jit-compiled run") >= 0, + "the jit lane names why it is unavailable: {rec.lanes?["jit"] ?? ""}") + tt |> success(find(rec.lanes?["jit"] ?? "", "/nonexistent/daslang") >= 0, + "and names the binary it could not run, since that binary printed nothing") + tt |> success(empty(rec.files), "an unavailable lane contributes no file") + } + t |> run("--failed writes the record of a night that never benchmarked and exits 1") @(tt : T?) { + let record = path_join(dir, "failed.json") + var out = "" + let code = run_tool(["run", "--root", dir, "--bin", fake, "--suite", suite, + "--meta", write_meta(dir, "failed"), "--out", record, "--failed", "the build failed"], out) + tt |> equal(code, 1, "a night that could not run is a failure") + var rec = RunRecord() + tt |> success(sscan_json(fread(record), rec), "the record parses") + tt |> equal(rec.status, RunStatus.build_failed) + tt |> success(empty(rec.files), "a failed build carries no file") + } + } +} + +[test] +def test_verb_run_refusals(t : T?) { + if (!has_shell()) { + t |> skip("needs /bin/sh") + return + } + with_temp_dir("bench_cli_") $(dir) { + seed_benchmarks(dir) + let fake = write_fake_daslang(dir) + let suite = write_suite(dir, "\"interp\"") + t |> run("a missing --meta, an unparseable one, and missing arguments each exit 1") @(tt : T?) { + var out = "" + tt |> equal(run_tool(["run", "--root", dir, "--out", path_join(dir, "x.json")], out), 1, "no --meta") + tt |> success(find(out, "--meta and --out are required") >= 0, "the message names what is missing") + let bad = path_join(dir, "bad_meta.json") + fwrite(bad, "not json") + tt |> equal(run_tool(["run", "--root", dir, "--bin", fake, "--suite", suite, + "--meta", bad, "--out", path_join(dir, "y.json")], out), 1, "unparseable meta") + tt |> success(find(out, "does not parse") >= 0, "and says so: {out}") + tt |> equal(run_tool(["nonsense"], out), 1, "an unknown verb prints help and exits 1") + } + t |> run("a suite config that fails to load stops the run before it spawns anything") @(tt : T?) { + let broken = path_join(dir, "broken_suite.json") + fwrite(broken, "\{ \"lanes\": [] \}") + var out = "" + tt |> equal(run_tool(["run", "--root", dir, "--bin", fake, "--suite", broken, + "--meta", write_meta(dir, "ok"), "--out", path_join(dir, "z.json")], out), 1) + tt |> success(find(out, "no lanes") >= 0, "the config's own error reaches the operator") + tt |> success(!fexist(path_join(dir, "z.json")), "and no record is written") + } + } +} + +[test] +def test_verb_report(t : T?) { + if (!has_shell()) { + t |> skip("needs /bin/sh") + return + } + with_temp_dir("bench_cli_") $(dir) { + let suite = write_suite(dir, "\"interp\"") + let runs_dir = path_join(dir, "runs") + mkdir_rec(runs_dir) + t |> run("report writes both artifacts and exits on the latest night's verdict") @(tt : T?) { + for (n in range(4)) { + var rec <- RunRecord(run_id = "r{n}", started = "2026-09-0{n + 1}T03:00:00Z", + commit = CommitInfo(sha = "sha{n}", date = "2026-09-0{n + 1}T01:00:00Z", subject = "c{n}"), + machine = MachineInfo(host = "box"), build = BuildInfo(status = "ok", seconds = 1.0lf), + status = RunStatus.ok) + rec.lanes["interp"] = "ok" + var f <- FileResult(path = "core/one.das", id = "core/one", group = "core", + lane = "interp", status = FileStatus.ok) + f.samples |> emplace(Sample(id = "b/arm", ns = 100.0lf, ns_median = 100.0lf, runs = 1)) + rec.files |> emplace(f) + fwrite(path_join(runs_dir, "r{n}.json"), sprint_json(rec, false)) + } + let data = path_join(dir, "data.json") + let summary = path_join(dir, "summary.md") + var out = "" + let code = run_tool(["report", "--suite", suite, "--runs", runs_dir, + "--out-data", data, "--out-summary", summary, "--generated", "2026-09-09T00:00:00Z", + "--repo-url", "https://example.invalid/repo", "--site-url", "https://example.invalid/bench/"], out) + tt |> equal(code, 0, "a clean latest night exits 0: {out}") + var ds = Dataset() + tt |> success(sscan_json(fread(data), ds), "data.json parses") + tt |> equal(length(ds.runs), 4) + tt |> equal(ds.latest, 3) + tt |> equal(ds.generated, "2026-09-09T00:00:00Z") + tt |> equal(ds.repo_url, "https://example.invalid/repo") + let md = fread(summary) + tt |> success(find(md, "# Benchmark stand - ok") >= 0, "summary.md leads with the verdict") + tt |> success(find(md, "https://example.invalid/bench/") >= 0, "and links the site") + } + t |> run("a corrupt record is warned about and left out, not fatal") @(tt : T?) { + fwrite(path_join(runs_dir, "zz_corrupt.json"), "\{ not json") + var out = "" + let code = run_tool(["report", "--suite", suite, "--runs", runs_dir, + "--out-data", path_join(dir, "d2.json"), "--out-summary", path_join(dir, "s2.md")], out) + tt |> equal(code, 0, "the other nights still publish") + tt |> success(find(out, "zz_corrupt.json") >= 0, "the corrupt file is named: {out}") + remove(path_join(runs_dir, "zz_corrupt.json")) + } + t |> run("an empty runs directory exits 1 and says there is nothing yet") @(tt : T?) { + let empty_dir = path_join(dir, "no_runs") + mkdir_rec(empty_dir) + var out = "" + let code = run_tool(["report", "--suite", suite, "--runs", empty_dir, + "--out-data", path_join(dir, "d3.json"), "--out-summary", path_join(dir, "s3.md")], out) + tt |> equal(code, 1) + tt |> success(find(fread(path_join(dir, "s3.md")), "No run records") >= 0, "the summary says so") + } + t |> run("missing arguments exit 1 before anything is written") @(tt : T?) { + var out = "" + tt |> equal(run_tool(["report", "--suite", suite, "--runs", runs_dir], out), 1) + tt |> success(find(out, "are required") >= 0, "the message names what is missing") + } + } +} + +//! Drives `nightly.sh gate` the way an inbound ssh connection does: the verb and its arguments +//! arrive only through SSH_ORIGINAL_COMMAND, and nothing else on that connection exists. +def private run_gate(home, command : string; var output : string&) : int { + var code = -1 + var out = "" + let env <- ["BENCH_STAND_HOME={home}", "SSH_ORIGINAL_COMMAND={command}"] + with_process(["/bin/bash", path_join(TOOL_DIR, "nightly.sh"), "gate"], ".", env) $(var p) { + unsafe { + while (true) { + process_drain(p) $(line) { + out += "{line}\n" + } + code = process_poll(p) + break if (code != process_running) + sleep(20u) + } + process_drain(p) $(line) { + out += "{line}\n" + } + } + } + output = out + return code +} + +[test] +def test_ssh_gate(t : T?) { + if (!fexist("/bin/bash")) { + t |> skip("needs bash") + return + } + with_temp_dir("bench_gate_") $(home) { + mkdir_rec(path_join(home, "site")) + fwrite(path_join(home, "site/status.json"), "\{\"state\":\"finished\",\"exit\":0\}") + fwrite(path_join(home, "site/summary.md"), "# a night\n") + t |> run("the two read-only verbs answer with what the box published") @(tt : T?) { + var out = "" + tt |> equal(run_gate(home, "status", out), 0) + tt |> success(find(out, "\"state\":\"finished\"") >= 0, "status prints status.json: {out}") + tt |> equal(run_gate(home, "summary", out), 0) + tt |> success(find(out, "# a night") >= 0, "summary prints summary.md") + } + t |> run("an argument carrying anything but path characters is refused") @(tt : T?) { + for (hostile in ["start master;rm -rf /", "start $(whoami)", "start `id`", + "start master&touch /tmp/pwned", "follow ../../etc/passwd|cat"]) { + var out = "" + let code = run_gate(home, hostile, out) + tt |> equal(code, 2, "refused: {hostile}") + tt |> success(find(out, "refused") >= 0, "and says so: {out}") + } + } + t |> run("more arguments than any verb takes is refused") @(tt : T?) { + var out = "" + tt |> equal(run_gate(home, "start master fail-on-regression extra", out), 2) + tt |> success(find(out, "too many arguments") >= 0, "the message names the reason: {out}") + } + t |> run("a verb the gate does not carry is refused, naming it") @(tt : T?) { + var out = "" + tt |> equal(run_gate(home, "run", out), 2, "`run` is a local verb, never an ssh one") + tt |> success(find(out, "unknown verb 'run'") >= 0, "the message names the verb: {out}") + tt |> equal(run_gate(home, "bash", out), 2) + } + t |> run("a missing status file still answers, rather than failing the caller") @(tt : T?) { + remove(path_join(home, "site/status.json")) + var out = "" + tt |> equal(run_gate(home, "status", out), 0) + tt |> success(find(out, "never-run") >= 0, "it says the stand has not run: {out}") + } + } +} diff --git a/utils/internal/bench-stand/test_bench_history.das b/utils/internal/bench-stand/test_bench_history.das new file mode 100644 index 0000000000..1e9a530382 --- /dev/null +++ b/utils/internal/bench-stand/test_bench_history.das @@ -0,0 +1,406 @@ +options gen2 + +require dastest/testing_boost public +require bench_history +require bench_runner +require bench_suite +require daslib/fio +require daslib/json_boost +require strings +require daslib/strings_boost +require math + +def private with_temp_dir(prefix : string; blk : block<(dir : string) : void>) { + var err = "" + let dir = create_temp_directory(prefix, err) + verify(!empty(dir), "temp dir created") + invoke(blk, dir) + rmdir_rec(dir) +} + +def private sample(id : string; ns : double) : Sample { + return Sample(id = id, ns = ns, ns_median = ns, spread = 0.05lf, runs = 3) +} + +def private file_ok(id, lane : string; samples : array) : FileResult { + var f <- FileResult(path = "{id}.das", id = id, group = dir_name(id), lane = lane, status = FileStatus.ok, tier = lane == "jit" ? "JIT" : "INTERP") + f.samples := samples + return <- f +} + +def private file_bad(id, lane : string; status : FileStatus; message : string) : FileResult { + return <- FileResult(path = "{id}.das", id = id, group = dir_name(id), lane = lane, status = status, message = message) +} + +//! Night `n` (0-based): commit date and run start both increase with n, so commit order equals run order. +def private night(n : int; files : array) : RunRecord { + var rec <- RunRecord(run_id = "r{n:02}", started = "2026-09-{n + 1:02}T03:00:00Z", finished = "2026-09-{n + 1:02}T04:00:00Z", seconds = 3600.0lf, + commit = CommitInfo(sha = "sha{n:02}abcdef", date = "2026-09-{n + 1:02}T00:00:00Z", subject = "commit {n}", author = "dev"), + machine = MachineInfo(host = "box", cpu = "cpu", cores = 4), build = BuildInfo(status = "ok", seconds = 600.0lf)) + rec.lanes["interp"] = "ok" + rec.lanes["jit"] = "ok" + rec.files := files + return <- rec +} + +def private write_night(dir : string; rec : RunRecord) { + fwrite(path_join(dir, "{rec.run_id}.json"), sprint_json(rec, false)) +} + +def private load(dir : string) : array { + var errors : array + return <- load_runs(dir, errors) +} + +//! The steady week plus an eighth night where a/x jumps +30% (interp) and drops -20% (jit), c/z +//! appears, b/y vanishes, and sort/s fails to compile. +def private eight_nights_with_a_jump(dir : string) { + steady_nights(dir, 7) + write_night(dir, night(7, [ + file_ok("core/hash/t", "interp", [sample("a/x", 130.0lf), sample("c/z", 5.0lf)]), + file_ok("core/hash/t", "jit", [sample("a/x", 40.0lf)]), + file_bad("sort/s", "interp", FileStatus.compile_error, "error[30344]: mismatch")])) +} + +def private cfg() : SuiteConfig { + var c = SuiteConfig() + c.lanes <- ["interp", "jit"] + return <- c +} + +//! `count` nights near 100 ns (interp) and 50 ns (jit) for `core/hash/t#a/x` and near 200 ns for +//! `b/y`, jittered so the baseline has a nonzero MAD - the noise gate needs one to be exercised. +def private steady_nights(dir : string; count : int) { + for (n in range(count)) { + let jitter = double(n % 3) * 0.5lf + write_night(dir, night(n, [ + file_ok("core/hash/t", "interp", [sample("a/x", 100.0lf + jitter), sample("b/y", 200.0lf - jitter)]), + file_ok("core/hash/t", "jit", [sample("a/x", 50.0lf + jitter)])])) + } +} + +[test] +def test_load_runs(t : T?) { + with_temp_dir("bench_hist_") $(dir) { + t |> run("records load in commit order, a corrupt file is named and skipped, status derives") @(tt : T?) { + write_night(dir, night(2, [file_ok("core/t", "interp", [sample("a/x", 1.0lf)])])) + write_night(dir, night(0, [file_ok("core/t", "interp", [sample("a/x", 1.0lf)])])) + let broken <- night(1, [file_bad("core/t", "interp", FileStatus.timeout, "killed")]) + write_night(dir, broken) + fwrite(path_join(dir, "zz_corrupt.json"), "\{ not json") + fwrite(path_join(dir, "notes.txt"), "ignored") + var errors : array + let runs <- load_runs(dir, errors) + tt |> equal(length(runs), 3) + tt |> equal(runs[0].run_id, "r00") + tt |> equal(runs[1].run_id, "r01") + tt |> equal(runs[2].run_id, "r02") + tt |> equal(runs[1].status, RunStatus.bench_failed) + tt |> equal(runs[0].status, RunStatus.ok) + tt |> equal(runs[0].files[0].status, FileStatus.ok) + tt |> equal(length(errors), 1) + tt |> success(find(errors[0], "zz_corrupt.json") >= 0, "corrupt file named") + } + t |> run("a night that measured nothing is not ok, whatever emptied it") @(tt : T?) { + let no_files : array + let empty_night <- night(30, no_files) + tt |> equal(derive_status(empty_night), RunStatus.bench_failed, + "no file ran at all - every lane unavailable, or a filter that matched nothing") + let only_skips <- night(31, [file_bad("core/x", "interp", FileStatus.skipped, "listed")]) + tt |> equal(derive_status(only_skips), RunStatus.bench_failed, "a night of nothing but skips carries no data point") + let one_ok <- night(32, [file_ok("core/x", "interp", [sample("a/x", 1.0lf)])]) + tt |> equal(derive_status(one_ok), RunStatus.ok, "one measured file is a night") + } + t |> run("a build failure written by the driver derives build_failed") @(tt : T?) { + let no_files : array + var rec <- night(5, no_files) + rec.build.status = "failed" + tt |> equal(derive_status(rec), RunStatus.build_failed) + } + } +} + +[test] +def test_statistics(t : T?) { + t |> run("median, relative MAD, geomean") @(tt : T?) { + var odd <- [3.0lf, 1.0lf, 2.0lf] + tt |> success(abs(median(odd) - 2.0lf) < 1e-9lf, "odd median") + var even <- [4.0lf, 1.0lf, 3.0lf, 2.0lf] + tt |> success(abs(median(even) - 2.5lf) < 1e-9lf, "even median") + tt |> success(abs(relative_mad([100.0lf, 110.0lf, 90.0lf, 100.0lf, 105.0lf]) - 0.05lf) < 1e-9lf, "MAD 5 over median 100") + tt |> success(abs(geomean([1.0lf, 4.0lf]) - 2.0lf) < 1e-9lf, "geomean of 1 and 4") + var none : array + tt |> equal(median(none), 0.0lf) + } +} + +[test] +def test_build_dataset(t : T?) { + with_temp_dir("bench_hist_") $(dir) { + eight_nights_with_a_jump(dir) + t |> run("series are keyed by lane and arm, points carry the run index") @(tt : T?) { + let runs <- load(dir) + let ds <- build_dataset(runs, cfg(), "2026-09-08T05:00:00Z", "https://github.com/x/y") + tt |> equal(length(ds.runs), 8) + tt |> equal(ds.latest, 7) + tt |> equal(length(ds.groups), 1) + tt |> equal(ds.groups[0], "core/hash") + tt |> equal(length(ds.series), 4) + tt |> equal(ds.series[0].lane, "interp") + tt |> equal(ds.series[0].id, "core/hash/t#a/x") + tt |> equal(length(ds.series[0].runs), 8) + tt |> equal(ds.series[0].runs[7], 7) + tt |> success(abs(ds.series[0].ns[7] - 130.0lf) < 1e-9lf, "tonight's value") + tt |> equal(ds.series[3].lane, "jit") + } + t |> run("a 30% jump over a quiet baseline is a regression, a 20% drop an improvement, a first sighting is new") @(tt : T?) { + let runs <- load(dir) + let ds <- build_dataset(runs, cfg(), "", "") + let kinds <- { for (ch in ds.changes); "{ch.lane} {ch.id}" => "{ch.kind}" } + tt |> equal(kinds?["interp core/hash/t#a/x"] ?? "", "regression") + tt |> equal(kinds?["jit core/hash/t#a/x"] ?? "", "improvement") + tt |> equal(kinds?["interp core/hash/t#c/z"] ?? "", "first_seen") + tt |> equal(kinds?["interp core/hash/t#b/y"] ?? "", "missing") + tt |> equal(length(ds.changes), 4) + for (ch in ds.changes) { + if (ch.kind == ChangeKind.regression) { + tt |> success(abs(ch.baseline - 100.5lf) < 1e-9lf, "baseline is the median of the 7 prior points: {ch.baseline}") + tt |> success(ch.change > 0.29lf && ch.change < 0.30lf, "change near +29%: {ch.change}") + } + } + tt |> equal(ds.runs[7].regressions, 1) + tt |> equal(ds.runs[7].improvements, 1) + } + t |> run("a series with fewer than three prior points gets no verdict") @(tt : T?) { + var two : array + two |> reserve(3) + for (n in range(3)) { + two |> emplace(night(n, [file_ok("core/hash/t", "interp", [sample("a/x", n == 2 ? 300.0lf : 100.0lf)])])) + } + let ds <- build_dataset(two, cfg(), "", "") + tt |> equal(length(ds.changes), 0) + tt |> equal(ds.runs[2].regressions, 0) + } + t |> run("a run measuring fewer than half a group's series contributes no index point") @(tt : T?) { + var nights : array + nights |> reserve(4) + for (n in range(4)) { + var arms <- [sample("a/x", 100.0lf), sample("b/y", 200.0lf), sample("c/z", 300.0lf)] + if (n == 3) { + arms |> resize(1) + } + nights |> emplace(night(n, [file_ok("core/hash/t", "interp", arms)])) + } + let ds <- build_dataset(nights, cfg(), "", "") + tt |> equal(length(ds.index), 1) + tt |> equal(length(ds.index[0].runs), 3) + tt |> equal(ds.index[0].runs[2], 2) + } + } +} + +[test] +def test_change_gates(t : T?) { + with_temp_dir("bench_hist_") $(dir) { + eight_nights_with_a_jump(dir) + t |> run("a noisy baseline widens the gate past the threshold, so the same jump is no verdict") @(tt : T?) { + // the prior week swings +-20%, so 3x its MAD is a 60% gate and a 30% jump is inside it + var noisy : array + noisy |> reserve(8) + let swing <- [100.0lf, 120.0lf, 80.0lf, 120.0lf, 80.0lf, 120.0lf, 80.0lf] + for (n, v in range(7), swing) { + noisy |> emplace(night(n, [file_ok("core/hash/t", "interp", [sample("a/x", v)])])) + } + noisy |> emplace(night(7, [file_ok("core/hash/t", "interp", [sample("a/x", 130.0lf)])])) + let ds <- build_dataset(noisy, cfg(), "", "") + tt |> equal(length(ds.changes), 0, "a 30% move over a +-20% baseline is noise, not a regression") + var quiet <- cfg() + quiet.noise_multiplier = 0.0lf + let ds_quiet <- build_dataset(noisy, quiet, "", "") + tt |> equal(length(ds_quiet.changes), 1, "with the noise term off, the threshold alone calls it") + tt |> equal(ds_quiet.changes[0].kind, ChangeKind.regression) + tt |> success(ds_quiet.changes[0].noise > 0.15lf, "the recorded noise is the baseline's own: {ds_quiet.changes[0].noise}") + } + t |> run("the baseline is the last baseline_runs points, not every point on record") @(tt : T?) { + // four quiet nights at 10 ns, then three at 100: a window of 3 makes 100 the baseline + var shifted : array + shifted |> reserve(8) + for (n in range(7)) { + shifted |> emplace(night(n, [file_ok("core/hash/t", "interp", [sample("a/x", n < 4 ? 10.0lf : 100.0lf)])])) + } + shifted |> emplace(night(7, [file_ok("core/hash/t", "interp", [sample("a/x", 105.0lf)])])) + var windowed <- cfg() + windowed.baseline_runs = 3 + let ds <- build_dataset(shifted, windowed, "", "") + tt |> equal(length(ds.changes), 0, "105 against a 100 baseline is inside the gate") + let ds_wide <- build_dataset(shifted, cfg(), "", "") + tt |> equal(length(ds_wide.changes), 1, "a seven-run window makes the baseline 10 and the night a regression") + tt |> success(ds_wide.changes[0].baseline < 20.0lf, "the wide baseline is the old level: {ds_wide.changes[0].baseline}") + } + t |> run("a move inside the threshold or inside the noise band is not a change") @(tt : T?) { + let runs <- load(dir) + var c <- cfg() + c.regression_threshold = 0.5lf + let ds <- build_dataset(runs, c, "", "") + var regs = 0 + for (ch in ds.changes) { + regs++ if (ch.kind == ChangeKind.regression || ch.kind == ChangeKind.improvement) + } + tt |> equal(regs, 0) + } + } +} + +[test] +def test_build_dataset_index_and_summary_rows(t : T?) { + with_temp_dir("bench_hist_") $(dir) { + eight_nights_with_a_jump(dir) + t |> run("the group index is 100 at the reference and moves with the group") @(tt : T?) { + let runs <- load(dir) + let ds <- build_dataset(runs, cfg(), "", "") + tt |> equal(length(ds.index), 2) + tt |> equal(ds.index[0].group, "core/hash") + tt |> equal(ds.index[0].lane, "interp") + tt |> equal(length(ds.index[0].runs), 8) + tt |> success(abs(ds.index[0].value[0] - 100.0lf) < 1.0lf, "first night near 100: {ds.index[0].value[0]}") + tt |> success(ds.index[0].value[7] > 105.0lf, "tonight's regression lifts the index: {ds.index[0].value[7]}") + tt |> equal(ds.index[1].lane, "jit") + tt |> success(ds.index[1].value[7] < 90.0lf, "jit improvement lowers its index: {ds.index[1].value[7]}") + } + t |> run("the run summary carries failures with their message and the ok count") @(tt : T?) { + let runs <- load(dir) + let ds <- build_dataset(runs, cfg(), "", "") + tt |> equal(ds.runs[7].status, RunStatus.bench_failed) + tt |> equal(ds.runs[7].files_ok, 2) + tt |> equal(length(ds.runs[7].failures), 1) + tt |> equal(ds.runs[7].failures[0].path, "sort/s.das") + tt |> equal(ds.runs[7].failures[0].status, "compile_error") + tt |> equal(ds.runs[7].failures[0].message, "error[30344]: mismatch") + tt |> equal(ds.runs[7].lanes?["jit"] ?? "", "ok") + } + t |> run("the dataset round-trips through JSON") @(tt : T?) { + let runs <- load(dir) + let ds <- build_dataset(runs, cfg(), "now", "") + var back = Dataset() + tt |> success(sscan_json(sprint_json(ds, false), back), "parses back") + tt |> equal(length(back.series), 4) + tt |> equal(back.latest, 7) + tt |> equal(length(back.changes), 4) + } + } +} + +[test] +def test_summary(t : T?) { + with_temp_dir("bench_hist_") $(dir) { + steady_nights(dir, 7) + t |> run("a clean night renders ok and exits 0") @(tt : T?) { + var errors : array + let runs <- load_runs(dir, errors) + let ds <- build_dataset(runs, cfg(), "", "") + let md = render_summary(ds, "https://daslang.io/bench/") + tt |> success(starts_with(md, "# Benchmark stand - ok - "), "title says ok: {slice(md, 0, 40)}") + tt |> success(find(md, "0 regressions, 0 improvements") >= 0, "counts zero changes") + tt |> success(find(md, "https://daslang.io/bench/") >= 0, "links the site") + tt |> equal(summary_exit_code(ds, true), 0) + } + t |> run("a failed night names the file and exits 1; a regression exits 2 only on request") @(tt : T?) { + write_night(dir, night(7, [ + file_ok("core/hash/t", "interp", [sample("a/x", 150.0lf), sample("b/y", 200.0lf)]), + file_ok("core/hash/t", "jit", [sample("a/x", 50.0lf)]), + file_bad("sort/s", "jit", FileStatus.timeout, "killed after 900 s (timeout_seconds)")])) + var errors : array + let runs <- load_runs(dir, errors) + let ds <- build_dataset(runs, cfg(), "", "") + let md = render_summary(ds, "") + tt |> success(find(md, "`sort/s.das` | jit | timeout | killed after 900 s") >= 0, "failure row present") + tt |> success(find(md, "### Regressions") >= 0, "regression section present") + tt |> success(find(md, "`core/hash/t#a/x` | interp | 100.5 | 150.0 | +49.3%") >= 0, "regression row formatted: {md}") + tt |> equal(summary_exit_code(ds, false), 1) + var clean := runs[7] + clean.files |> erase(2) + write_night(dir, clean) + var errors2 : array + let runs2 <- load_runs(dir, errors2) + let ds2 <- build_dataset(runs2, cfg(), "", "") + tt |> equal(summary_exit_code(ds2, false), 0) + tt |> equal(summary_exit_code(ds2, true), 2) + } + } +} + +[test] +def test_summary_of_a_night_that_failed(t : T?) { + with_temp_dir("bench_hist_") $(dir) { + t |> run("a build failure leads with its reason and the build log tail, and nothing else") @(tt : T?) { + let no_files : array + var rec <- night(7, no_files) + rec.build.status = "failed" + rec.build.seconds = 41.0lf + rec.build.log_tail = "ninja: error: unknown target 'daslang'" + write_night(dir, rec) + let ds <- build_dataset(load(dir), cfg(), "", "https://daslang.io/bench/") + let md = render_summary(ds, "https://daslang.io/bench/") + tt |> success(find(md, "build_failed") >= 0, "the title names the status") + tt |> success(find(md, "## The build failed after 41 s - nothing was measured") >= 0, "the build section leads: {md}") + tt |> success(find(md, "unknown target 'daslang'") >= 0, "the log tail is quoted") + tt |> success(find(md, "## Changes") < 0, "no change section on a night that measured nothing") + tt |> success(find(md, "https://daslang.io/bench/") >= 0, "the charts link survives") + tt |> equal(summary_exit_code(ds, false), 1) + remove(path_join(dir, "{rec.run_id}.json")) + } + t |> run("improvements, first sightings, missing arms, skips and a dark lane all render") @(tt : T?) { + eight_nights_with_a_jump(dir) + var rec <- night(8, [ + file_ok("core/hash/t", "interp", [sample("a/x", 40.0lf), sample("d/w", 7.0lf)]), + file_bad("core/hash/slow", "interp", FileStatus.skipped, "takes an hour")]) + rec.lanes["jit"] = "daslang -jit exited with code 1: no dasLLVM in this build" + write_night(dir, rec) + let ds <- build_dataset(load(dir), cfg(), "", "") + let md = render_summary(ds, "") + tt |> success(find(md, "### Improvements") >= 0, "improvement section") + tt |> success(find(md, "`core/hash/t#a/x` | interp | 100.5 | 40.0 | -60.2%") >= 0, "improvement row: {md}") + tt |> success(find(md, "### Missing tonight") >= 0, "missing section") + tt |> success(find(md, "`core/hash/t#b/y`") >= 0, "the missing arm is named") + tt |> success(find(md, "first seen") >= 0, "first sightings counted") + tt |> success(find(md, "## Skipped (1)") >= 0, "skip section") + tt |> success(find(md, "`core/hash/slow.das` (interp): takes an hour") >= 0, "the skip states its reason") + tt |> success(find(md, "jit = daslang -jit exited with code 1") >= 0, "a dark lane says why") + tt |> success(find(md, "1 file runs ok, 0 failed, 1 skipped") >= 0, "the counts line") + remove(path_join(dir, "{rec.run_id}.json")) + } + t |> run("a night that never benchmarked names the driver as the failure") @(tt : T?) { + var rec <- night(9, [file_ok("core/hash/t", "interp", [sample("a/x", 100.0lf)])]) + mark_failed_run(rec, "2026-09-10T04:00:00Z", "the tool exited 1 without writing a record") + tt |> equal(rec.status, RunStatus.bench_failed) + tt |> equal(length(rec.files), 1) + tt |> equal(rec.files[0].lane, "driver") + tt |> success(find(rec.files[0].message, "exited 1") >= 0, "the reason survives") + tt |> equal(rec.seconds, 0.0lf) + tt |> equal(rec.finished, "2026-09-10T04:00:00Z") + let none_either : array + var build_broke <- night(10, none_either) + build_broke.build.status = "failed" + mark_failed_run(build_broke, "2026-09-11T04:00:00Z", "") + tt |> equal(build_broke.status, RunStatus.build_failed) + tt |> success(empty(build_broke.files), "a failed build carries no file - its log tail is the reason") + } + t |> run("a run with no commit names no commit, rather than empty backticks") @(tt : T?) { + let no_files : array + var bare <- night(20, no_files) + bare.commit = CommitInfo() + bare.machine = MachineInfo() + var one : array + one |> emplace(bare) + let md = render_summary(build_dataset(one, cfg(), "", ""), "") + tt |> success(find(md, "Commit ``") < 0, "no empty commit line: {md}") + tt |> success(find(md, " on ``") < 0, "and no empty host clause") + } + t |> run("no records renders a note and exits 1") @(tt : T?) { + let none : array + let ds <- build_dataset(none, cfg(), "", "") + tt |> success(find(render_summary(ds, ""), "No run records") >= 0, "note") + tt |> equal(summary_exit_code(ds, false), 1) + } + } +} diff --git a/utils/internal/bench-stand/test_bench_runner.das b/utils/internal/bench-stand/test_bench_runner.das new file mode 100644 index 0000000000..59eebdd684 --- /dev/null +++ b/utils/internal/bench-stand/test_bench_runner.das @@ -0,0 +1,358 @@ +options gen2 + +require dastest/testing_boost public +require bench_runner +require bench_suite +require daslib/fio +require strings +require daslib/strings_boost +require math + +def private stats_line(name, sub : string; n : int; time_ns : int64; allocs : int = 0; tier : string = "INTERP"; + string_allocs : int = 0) : string { + return "\{\"name\":\"{name}\",\"sub_name\":\"{sub}\",\"n\":{n},\"time_ns\":{time_ns},\"allocs\":{allocs},\"heap_bytes\":{allocs * 16},\"string_allocs\":{string_allocs},\"string_heap_bytes\":{string_allocs * 32},\"func_type\":\"{tier}\"\}" +} + +def private report(success : bool; names : array; failed : array) : string { + let tests <- [for (n in names); "\{\"name\":\"{n}\",\"passed\":{find_index(failed, n) < 0},\"skipped\":false,\"time\":1,\"messages\":[],\"location\":\"\"\}"] + return "\{\"file\":\"x.das\",\"total\":0,\"passed\":0,\"failed\":{length(failed)},\"errors\":0,\"skipped\":0,\"success\":{success},\"time_usec\":1,\"tests\":[{join(tests, ",")}]\}" +} + +def private fresh_result(status : FileStatus = FileStatus.ok) : FileResult { + return <- FileResult(path = "core/x.das", id = "core/x", group = "core", lane = "interp", status = status) +} + +[test] +def test_parse_bench_output(t : T?) { + t |> run("samples aggregate over repeats: min, median, spread, per-op allocation medians") @(tt : T?) { + var res <- fresh_result() + let lines <- [ + "2026-09-08T14:11:43.241Z run 1/3: benchmarks/core/x.das", + stats_line("bench_a", "insert/100", 100, 10000l, 300, "INTERP", 100), + stats_line("bench_a", "read/100", 1000, 50000l), + stats_line("bench_a", "insert/100", 100, 12000l, 500, "INTERP", 100), + stats_line("bench_a", "read/100", 1000, 40000l), + stats_line("bench_a", "insert/100", 100, 11000l, 400, "INTERP", 100), + stats_line("bench_a", "read/100", 1000, 45000l), + "", + "0 tests, 0 passed, 0 failed, 0 errors, 0 skipped", + "SUCCESS! (3.9s)"] + parse_bench_output(lines, report(true, ["bench_a"], []), 0, res) + tt |> equal(res.status, FileStatus.ok) + tt |> equal(res.exit_code, 0) + tt |> equal(res.tier, "INTERP") + tt |> equal(length(res.samples), 2) + tt |> equal(res.samples[0].id, "bench_a/insert/100") + tt |> equal(res.samples[0].runs, 3) + tt |> success(abs(res.samples[0].ns - 100.0lf) < 0.001lf, "min is 100 ns/op") + tt |> success(abs(res.samples[0].ns_median - 110.0lf) < 0.001lf, "median is 110 ns/op") + tt |> success(abs(res.samples[0].spread - 0.2lf) < 0.001lf, "spread is (120-100)/100") + tt |> equal(res.samples[0].allocs, 4l, "allocs is the per-op median of 3, 4, 5") + tt |> equal(res.samples[0].bytes, 64l, "heap bytes is the per-op median of 48, 64, 80") + tt |> equal(res.samples[0].string_allocs, 1l) + tt |> equal(res.samples[0].string_bytes, 32l) + tt |> equal(res.samples[1].allocs, 0l, "an arm that allocates nothing reports zero") + tt |> equal(res.samples[1].id, "bench_a/read/100") + tt |> success(abs(res.samples[1].ns - 40.0lf) < 0.001lf, "read min is 40 ns/op") + tt |> equal(res.message, "") + } + t |> run("the log tail keeps the non-sample lines, newest last") @(tt : T?) { + var res <- fresh_result() + var lines <- [for (i in range(100)); "line {i}"] + lines |> push(stats_line("b", "s", 10, 100l)) + parse_bench_output(lines, report(true, ["b"], []), 0, res) + tt |> success(find(res.log_tail, "line 0\n") < 0, "oldest lines dropped") + tt |> success(ends_with(res.log_tail, "line 99"), "newest line kept") + tt |> success(find(res.log_tail, "\"sub_name\"") < 0, "sample lines are not log") + } + t |> run("a compile error names the first error line") @(tt : T?) { + var res <- fresh_result() + let lines <- [ + "[E] Failed to compile benchmarks/core/x.das", + "error[30344]: local variable x initialization type mismatch; int const = string const", + "benchmarks/core/x.das:5:8", + "FAIL benchmarks/core/x.das (0.04s)"] + parse_bench_output(lines, "", 1, res) + tt |> equal(res.status, FileStatus.compile_error) + tt |> success(find(res.message, "error[30344]") >= 0, "message carries the error code") + tt |> equal(res.exit_code, 1) + } + t |> run("a failed benchmark function drops its arms and names itself; the others survive") @(tt : T?) { + var res <- fresh_result() + let lines <- [ + "benchmarks/core/x.das:5: values differ", + stats_line("failing", "fail_op", 100, 100l), + "[E] --- FAIL 'failing' in 'fail_op' (1.7s)", + stats_line("ok_after", "ok_op", 100, 100l), + "FAIL benchmarks/core/x.das (3.0s)"] + parse_bench_output(lines, report(false, ["failing", "ok_after"], ["failing"]), 1, res) + tt |> equal(res.status, FileStatus.failed) + tt |> success(find(res.message, "failing") >= 0, "message names the function") + tt |> equal(length(res.samples), 1) + tt |> equal(res.samples[0].id, "ok_after/ok_op") + } +} + +[test] +def test_parse_bench_output_verdicts(t : T?) { + t |> run("the FAIL log line alone marks the function failed when no report was written") @(tt : T?) { + var res <- fresh_result() + let lines <- [stats_line("failing", "fail_op", 100, 100l), "[E] --- FAIL 'failing' in 'fail_op' (1.7s)"] + parse_bench_output(lines, "", 1, res) + tt |> equal(res.status, FileStatus.failed) + tt |> equal(length(res.samples), 0) + } + t |> run("a measured file that dies at shutdown keeps its numbers and says so") @(tt : T?) { + var res <- fresh_result() + let lines <- [ + stats_line("b", "s", 10, 100l), + "SUCCESS! (48.9s)", + "FATAL: exiting with code 1 -- 1 smart pointer(s) still alive at shutdown"] + parse_bench_output(lines, report(true, ["b"], []), 1, res) + tt |> equal(res.status, FileStatus.ok, "the measurement stands: {res.message}") + tt |> equal(length(res.samples), 1) + tt |> success(find(res.message, "at shutdown") >= 0, "the message names when it happened") + tt |> success(find(res.message, "smart pointer") >= 0, "and carries the FATAL line") + } + t |> run("a nonzero exit with no finished report is a failure, samples or not") @(tt : T?) { + var res <- fresh_result() + let lines <- [stats_line("b", "s", 10, 100l)] + parse_bench_output(lines, "", 1, res) + tt |> equal(res.status, FileStatus.exit_nonzero) + tt |> success(find(res.message, "did not run to its end") >= 0, "the message says the suite was cut short: {res.message}") + } + t |> run("a limit verdict survives whatever the output says, keeping the finished arms") @(tt : T?) { + var res <- fresh_result(FileStatus.timeout) + res.message = "killed after 1 s (timeout_seconds)" + let lines <- [stats_line("b", "done", 10, 100l)] + parse_bench_output(lines, "", 9, res) + tt |> equal(res.status, FileStatus.timeout) + tt |> equal(res.message, "killed after 1 s (timeout_seconds)") + tt |> equal(length(res.samples), 1) + tt |> equal(res.exit_code, 9) + } + t |> run("a clean exit with no samples and no report is not ok") @(tt : T?) { + var res <- fresh_result() + let no_lines : array + parse_bench_output(no_lines, "", 0, res) + tt |> equal(res.status, FileStatus.exit_nonzero) + var res2 <- fresh_result() + parse_bench_output(no_lines, report(true, no_lines, no_lines), 0, res2) + tt |> equal(res2.status, FileStatus.failed) + tt |> success(find(res2.message, "no benchmark samples") >= 0, "message says what is missing") + } + t |> run("is_failure separates ok and skipped from the rest") @(tt : T?) { + tt |> success(!is_failure(FileStatus.ok), "ok") + tt |> success(!is_failure(FileStatus.skipped), "skipped") + tt |> success(is_failure(FileStatus.timeout), "timeout") + tt |> success(is_failure(FileStatus.exit_nonzero), "exit_nonzero") + } +} + +def private has_shell() : bool { + return fexist("/bin/sh") +} + +def private bench_file() : BenchFile { + return BenchFile(path = "x/y.das", id = "x/y", group = "x", timeout_seconds = 2, memory_limit_mb = 64) +} + +//! A stand-in for the daslang binary. `body` runs with `$REPORT` set to the path the runner +//! passed after `--json-file` and `$ALL` to the whole argv, so a fixture reads the arguments by +//! name rather than encoding their order. +def private fake_daslang(body : string) : string { + var err = "" + let path = create_temp_file("bench_stand_fake_", ".sh", err) + fwrite(path, "#!/bin/sh\nALL=\"$*\"\nREPORT=\"\"\nwhile [ $# -gt 0 ]; do\n [ \"$1\" = --json-file ] && REPORT=$2\n shift\ndone\n{body}\n") + var out = "" + run_and_capture(["/bin/chmod", "755", path], out) + return path +} + +let private report_text = "\{\"file\":\"x.das\",\"total\":0,\"passed\":0,\"failed\":0,\"errors\":0,\"skipped\":0,\"success\":true,\"time_usec\":1,\"tests\":[\{\"name\":\"b\",\"passed\":true,\"skipped\":false,\"time\":1,\"messages\":[],\"location\":\"\"\}]\}" + +[test] +def test_run_bench_file_limits(t : T?) { + if (!fexist("/bin/sh")) { + t |> skip("needs /bin/sh") + return + } + t |> run("a skipped file never spawns and carries its reason") @(tt : T?) { + var f = bench_file() + f.skip = "too slow" + let res <- run_bench_file("/bin/sh", ".", f, "interp", RunLimits()) + tt |> equal(res.status, FileStatus.skipped) + tt |> equal(res.message, "too slow") + } + t |> run("a child that outlives timeout_seconds is killed; the arms it printed survive") @(tt : T?) { + let fake = fake_daslang("echo '{stats_line("b", "done", 10, 100l)}'; exec sleep 30") + let res <- run_bench_file(fake, ".", bench_file(), "interp", RunLimits(timeout_seconds = 1)) + remove(fake) + tt |> equal(res.status, FileStatus.timeout) + tt |> success(find(res.message, "timeout_seconds") >= 0, "message names the knob") + tt |> success(res.seconds >= 1.0lf && res.seconds < 15.0lf, "killed near the limit, not at the child's own end") + tt |> equal(length(res.samples), 1) + } + t |> run("a child whose resident set passes memory_limit_mb is killed and reported") @(tt : T?) { + if (!fexist("/usr/bin/perl")) { + tt |> skip("needs perl to grow a child's resident set") + return + } + let fake = fake_daslang("exec /usr/bin/perl -e '$x = \"a\" x 200000000; sleep 30'") + let res <- run_bench_file(fake, ".", bench_file(), "interp", RunLimits(timeout_seconds = 20, memory_limit_mb = 64)) + remove(fake) + tt |> equal(res.status, FileStatus.memory) + tt |> success(find(res.message, "memory_limit_mb") >= 0, "message names the knob") + tt |> success(res.seconds < 15.0lf, "killed well before the timeout") + } + t |> run("a child that exits cleanly with samples is ok, from the repo root cwd") @(tt : T?) { + let fake = fake_daslang("echo '{stats_line("b", "done", 10, 100l)}'; echo '{report_text}' > \"$REPORT\"; exit 0") + let res <- run_bench_file(fake, ".", bench_file(), "interp", RunLimits(timeout_seconds = 5)) + remove(fake) + tt |> equal(res.status, FileStatus.ok) + tt |> equal(length(res.samples), 1) + tt |> equal(res.exit_code, 0) + } + t |> run("the jit lane puts -jit ahead of dastest, and the interp lane does not") @(tt : T?) { + let fake = fake_daslang("echo \"ARGV: $ALL\"; exit 1") + let jit <- run_bench_file(fake, ".", bench_file(), "jit", RunLimits(timeout_seconds = 10)) + let interp <- run_bench_file(fake, ".", bench_file(), "interp", RunLimits(timeout_seconds = 10, repeat = 1)) + remove(fake) + tt |> success(find(jit.log_tail, "ARGV: -jit ") >= 0, "the jit lane's first argument is -jit: {jit.log_tail}") + tt |> success(find(jit.log_tail, "dastest/dastest.das") >= 0, "and dastest follows it") + tt |> success(find(interp.log_tail, "-jit") < 0, "the interp lane names no -jit: {interp.log_tail}") + tt |> success(find(interp.log_tail, "--count 1") >= 0, "the repeat count reaches the child: {interp.log_tail}") + } +} + +[test] +def test_run_bench_file_lanes(t : T?) { + if (!has_shell()) { + t |> skip("needs /bin/sh") + return + } + t |> run("the aot lane carries both use-aot flags, or it would measure the interpreter") @(tt : T?) { + let fake = fake_daslang("echo \"ARGV: $ALL\"; exit 1") + let aot <- run_bench_file(fake, ".", bench_file(), "aot", RunLimits(timeout_seconds = 10)) + remove(fake) + // the host flag arms the stub lookup, the dastest flag arms fail_on_no_aot; with only the + // first, every function silently interprets and the lane reports interpreter numbers + tt |> success(find(aot.log_tail, "ARGV: -use-aot ") >= 0, "the host flag comes first: {aot.log_tail}") + tt |> success(find(aot.log_tail, "-- --use-aot ") >= 0, "and dastest's own flag follows the separator") + tt |> success(find(aot.log_tail, "-jit") < 0, "the aot lane is not the jit lane") + } + t |> run("a file skipped in one lane still measures in the others") @(tt : T?) { + var f = bench_file() + f.skip_lanes["aot"] = "the emitter cannot handle this one" + tt |> equal(skip_reason(f, "aot"), "the emitter cannot handle this one") + tt |> equal(skip_reason(f, "interp"), "", "the other lanes are untouched") + let fake = fake_daslang("echo '{stats_line("b", "done", 10, 100l)}'; echo '{report_text}' > \"$REPORT\"; exit 0") + let aot <- run_bench_file(fake, ".", f, "aot", RunLimits(timeout_seconds = 5)) + let interp <- run_bench_file(fake, ".", f, "interp", RunLimits(timeout_seconds = 5)) + remove(fake) + tt |> equal(aot.status, FileStatus.skipped) + tt |> equal(aot.message, "the emitter cannot handle this one") + tt |> equal(interp.status, FileStatus.ok, "and the interpreter still measured it") + f.skip = "broken everywhere" + tt |> equal(skip_reason(f, "interp"), "broken everywhere", "a whole-file skip answers for every lane") + } + t |> run("an unrunnable binary reports that nothing was measured, naming the cause") @(tt : T?) { + let res <- run_bench_file("/nonexistent/daslang", ".", bench_file(), "interp", RunLimits(timeout_seconds = 5)) + tt |> success(res.status == FileStatus.spawn_failed || res.status == FileStatus.exit_nonzero, "spawn failure or an immediate nonzero exit: {res.status}") + tt |> success(empty(res.samples), "nothing measured") + tt |> success(find(res.message, "before measuring anything") >= 0 || find(res.message, "cannot spawn") >= 0, + "the message says nothing ran rather than claiming the benchmarks passed: {res.message}") + } +} + +[test] +def test_process_rss_mb(t : T?) { + t |> run("a live child's resident set reads back as a positive number of megabytes") @(tt : T?) { + if (!fexist("/bin/sh")) { + tt |> skip("needs /bin/sh") + return + } + var mb = -1 + with_process(["/bin/sh", "-c", "sleep 5"]) $(var p) { + unsafe { + mb = process_rss_mb(process_pid(p)) + process_kill(p) + process_wait(p, 5.0) + } + } + tt |> success(mb > 0 && mb < 4096, "a shell's RSS is a small positive figure, got {mb} MB") + tt |> success(mb >= 1, "rounded up, so a process under a megabyte still reads as one") + } + t |> run("a pid nothing owns reads back as -1, which leaves the limit disarmed") @(tt : T?) { + tt |> equal(process_rss_mb(0), -1) + } +} + +[test] +def test_probe_aot(t : T?) { + if (!fexist("/bin/sh")) { + t |> skip("needs /bin/sh") + return + } + t |> run("a missing AOT binary names the target that builds it") @(tt : T?) { + let why = probe_aot("/nonexistent/test_aot") + tt |> success(find(why, "no AOT binary") >= 0, "it says what is absent: {why}") + tt |> success(find(why, "test_aot") >= 0, "and names the target that builds it") + } + t |> run("an AOT binary that runs the probe reports the lane available") @(tt : T?) { + let fake = fake_daslang("echo aot-probe-ok; exit 0") + let why = probe_aot(fake) + remove(fake) + tt |> equal(why, "", "an available lane has no reason") + } + t |> run("an AOT binary that prints its help instead of running is not available") @(tt : T?) { + // what the real binary does when handed a flag it does not take: usage text, exit 255 + let fake = fake_daslang("echo ' -log-compile-time log per-module breakdown'; exit 255") + let why = probe_aot(fake) + remove(fake) + tt |> success(find(why, "ran no program") >= 0, "the reason says no program ran: {why}") + tt |> success(find(why, "exit 255") >= 0, "and names the exit code") + } + t |> run("an AOT binary whose marker is missing is not available even on exit 0") @(tt : T?) { + let fake = fake_daslang("exit 0") + let why = probe_aot(fake) + remove(fake) + tt |> success(!empty(why), "silence is not availability: {why}") + } +} + +[test] +def test_probe_jit(t : T?) { + if (!fexist("/bin/sh")) { + t |> skip("needs /bin/sh") + return + } + t |> run("a binary that runs the probe reports the lane available") @(tt : T?) { + let fake = fake_daslang("echo jit-probe-ok; exit 0") + let why = probe_jit(fake) + remove(fake) + tt |> equal(why, "", "an available lane has no reason") + } + t |> run("a host whose only failure is its exit code still has a working lane") @(tt : T?) { + // what a RelWithDebInfo daslang does: the program runs, then the allocation tracker's + // exit-time report takes the process code to 1 + let fake = fake_daslang("echo jit-probe-ok; echo '=== daslang C++ heap leak report ==='; exit 1") + let why = probe_jit(fake) + remove(fake) + tt |> equal(why, "", "the marker printed, so the lane runs: {why}") + } + t |> run("a binary that fails the probe reports the reason it printed") @(tt : T?) { + let fake = fake_daslang("echo 'error[30812]: undefined function argument type NameLookup? const'; exit 1") + let why = probe_jit(fake) + remove(fake) + tt |> success(find(why, "exit 1") >= 0, "the reason names the exit code: {why}") + tt |> success(find(why, "error[30812]") >= 0, "and the compile error it printed, not some later line") + } + t |> run("a binary that exits 0 without the probe marker is not taken as available") @(tt : T?) { + let fake = fake_daslang("echo nothing to see; exit 0") + let why = probe_jit(fake) + remove(fake) + tt |> success(!empty(why), "silence is not availability: {why}") + tt |> success(find(why, "no jit-compiled run") >= 0, "and the reason says what was missing") + } +} diff --git a/utils/internal/bench-stand/test_bench_suite.das b/utils/internal/bench-stand/test_bench_suite.das new file mode 100644 index 0000000000..cbd7ae93f6 --- /dev/null +++ b/utils/internal/bench-stand/test_bench_suite.das @@ -0,0 +1,137 @@ +options gen2 + +require dastest/testing_boost public +require bench_suite +require daslib/fio +require strings + +def private with_temp_dir(prefix : string; blk : block<(dir : string) : void>) { + var err = "" + let dir = create_temp_directory(prefix, err) + verify(!empty(dir), "temp dir created") + invoke(blk, dir) + rmdir_rec(dir) +} + +def private touch(dir, rel : string) { + let p = path_join(dir, rel) + mkdir_rec(dir_name(p)) + fwrite(p, "options gen2\n") +} + +let private GOOD_CONFIG = "\{ \"root\": \"benchmarks\", \"exclude\": [\"**/tests/**\", \"**/_*.das\"], \"lanes\": [\"interp\"], \"repeat\": 2, \"timeout_seconds\": 30, \"memory_limit_mb\": 512, \"baseline_runs\": 5, \"files\": \{ \"core/slow.das\": \{ \"skip\": \"too slow\" \}, \"core/big.das\": \{ \"timeout_seconds\": 60, \"memory_limit_mb\": 2048 \} \} \}" + +[test] +def test_load_suite_config(t : T?) { + with_temp_dir("bench_stand_") $(dir) { + t |> run("a well-formed config parses with its overrides") @(tt : T?) { + let path = path_join(dir, "suite.json") + fwrite(path, GOOD_CONFIG) + var error = "" + let cfg <- load_suite_config(path, error) + tt |> equal(error, "") + tt |> equal(cfg.repeat, 2) + tt |> equal(cfg.timeout_seconds, 30) + tt |> equal(length(cfg.exclude), 2) + tt |> equal(length(cfg.lanes), 1) + cfg.files |> get("core/slow.das") $(ov) { + tt |> equal(ov.skip, "too slow") + } + cfg.files |> get("core/big.das") $(ov) { + tt |> equal(ov.timeout_seconds, 60) + } + tt |> equal(length(cfg.files), 2) + tt |> success(cfg.regression_threshold > 0.09lf && cfg.regression_threshold < 0.11lf, "threshold keeps its default") + } + t |> run("a missing config fails closed naming the path") @(tt : T?) { + var error = "" + load_suite_config(path_join(dir, "missing.json"), error) + tt |> success(find(error, "missing.json") >= 0, "error names the path") + } + t |> run("an unparseable config fails closed") @(tt : T?) { + let path = path_join(dir, "bad.json") + fwrite(path, "not json") + var error = "" + load_suite_config(path, error) + tt |> success(find(error, "parse") >= 0, "error says it did not parse") + } + t |> run("no lanes, a zero repeat, or an unknown lane are errors") @(tt : T?) { + let path = path_join(dir, "lanes.json") + fwrite(path, "\{ \"lanes\": [] \}") + var error = "" + load_suite_config(path, error) + tt |> success(find(error, "no lanes") >= 0, "empty lanes rejected") + fwrite(path, "\{ \"lanes\": [\"interp\"], \"repeat\": 0 \}") + load_suite_config(path, error) + tt |> success(find(error, "positive") >= 0, "zero repeat rejected") + fwrite(path, "\{ \"lanes\": [\"vulkan\"] \}") + load_suite_config(path, error) + tt |> success(find(error, "vulkan") >= 0, "unknown lane named") + fwrite(path, "\{ \"lanes\": [\"interp\", \"jit\", \"aot\"] \}") + load_suite_config(path, error) + tt |> equal(error, "", "the three real lanes are accepted") + fwrite(path, "\{ \"lanes\": [\"interp\"], \"baseline_runs\": 2 \}") + load_suite_config(path, error) + tt |> success(find(error, "baseline_runs") >= 0, "a baseline of two runs has no median worth gating on") + } + } +} + +def private load_cfg(dir : string) : SuiteConfig { + var error = "" + return <- load_suite_config(path_join(dir, "suite.json"), error) +} + +[test] +def test_discover_files(t : T?) { + with_temp_dir("bench_stand_") $(dir) { + fwrite(path_join(dir, "suite.json"), GOOD_CONFIG) + touch(dir, "benchmarks/core/hash/test02.das") + touch(dir, "benchmarks/core/hash/_common.das") + touch(dir, "benchmarks/core/slow.das") + touch(dir, "benchmarks/core/big.das") + touch(dir, "benchmarks/sql/tests/test_update.das") + touch(dir, "benchmarks/sort/sort.das") + touch(dir, "benchmarks/sort/README.md") + t |> run("helpers, test dirs and non-das files are excluded; the rest is sorted") @(tt : T?) { + let cfg <- load_cfg(dir) + var err = "" + let files <- discover_files(dir, cfg, err) + tt |> equal(err, "") + tt |> equal(length(files), 4) + tt |> equal(files[0].path, "core/big.das") + tt |> equal(files[1].path, "core/hash/test02.das") + tt |> equal(files[2].path, "core/slow.das") + tt |> equal(files[3].path, "sort/sort.das") + } + t |> run("group and id derive from the path") @(tt : T?) { + let cfg <- load_cfg(dir) + var err = "" + let files <- discover_files(dir, cfg, err) + tt |> equal(files[1].group, "core/hash") + tt |> equal(files[1].id, "core/hash/test02") + tt |> equal(files[3].group, "sort") + tt |> equal(group_of("top.das"), "root") + tt |> equal(stem_path("top.das"), "top") + } + t |> run("overrides apply and defaults fill the rest") @(tt : T?) { + let cfg <- load_cfg(dir) + var err = "" + let files <- discover_files(dir, cfg, err) + tt |> equal(files[2].skip, "too slow") + tt |> equal(files[2].timeout_seconds, 30) + tt |> equal(files[0].timeout_seconds, 60) + tt |> equal(files[0].memory_limit_mb, 2048) + tt |> equal(files[1].memory_limit_mb, 512) + tt |> equal(files[1].skip, "") + } + t |> run("an override naming a missing file is reported") @(tt : T?) { + let cfg <- load_cfg(dir) + remove(path_join(dir, "benchmarks/core/slow.das")) + var err = "" + let files <- discover_files(dir, cfg, err) + tt |> equal(length(files), 3) + tt |> success(find(err, "core/slow.das") >= 0, "error names the stale override") + } + } +}