From 5d07280a855aaf5bb3aedecc61bf0f6b7b2bf879 Mon Sep 17 00:00:00 2001 From: Nigro Simone Date: Thu, 24 Sep 2026 20:58:52 +0200 Subject: [PATCH 1/4] chore(benchmark): compare branches against the noise of the same code --- .gitignore | 3 + benchmark/bench-cmp-branch.js | 257 +++++++++++++++++++++++----------- benchmark/bench.js | 7 +- package.json | 3 - 4 files changed, 181 insertions(+), 89 deletions(-) diff --git a/.gitignore b/.gitignore index 0448cb49..727a12e6 100644 --- a/.gitignore +++ b/.gitignore @@ -152,3 +152,6 @@ yarn.lock .pi AGENTS.md CLAUDE.md + +# the other revision benchmark/bench-cmp-branch.js measures against +.bench-worktree diff --git a/benchmark/bench-cmp-branch.js b/benchmark/bench-cmp-branch.js index 0834ad15..1a08053e 100644 --- a/benchmark/bench-cmp-branch.js +++ b/benchmark/bench-cmp-branch.js @@ -1,116 +1,203 @@ 'use strict' -const { spawn } = require('child_process') +// Measures the working tree against another revision, with the noise of this machine beside it. +// +// Each benchmark runs in four workers at once, two on the other revision (A) and two on the working +// tree (B), and they take turns in batches of about 10ms, so a machine that warms up or throttles +// slows all four together. A against A and B against B is the same code against itself: how far it +// moves is the noise. A/B is followed by the band that noise gives to it, and a row is a result only +// when A/B is outside that band. +// +// npm run bench:cmp main against the working tree +// npm run bench:cmp -- --against v7.0.0 +// npm run bench:cmp -- --only "short string" --sessions 10 +// +// On a clean tree, --against HEAD puts the same code on both sides and every row should be noise. + +const { Worker, isMainThread, workerData, parentPort } = require('node:worker_threads') +const { execFileSync } = require('node:child_process') +const { once } = require('node:events') +const { parseArgs } = require('node:util') +const fs = require('node:fs') +const path = require('node:path') + +const ROOT = path.join(__dirname, '..') +// inside the repo, so the other revision resolves node_modules from here +const WORKTREE = path.join(ROOT, '.bench-worktree') +const WARMUP_MS = 250 +const BATCH_NS = 1e7 +// how many standard errors A/B has to be from zero: with 5 sessions, about 1 row in 400 of the +// same code against itself +const BAND = 4 + +// the four workers are a1 a2 b1 b2; turning through these orders keeps a drift of the machine out +// of A/B in every round, and out of A/A and B/B every four rounds +const ORDERS = [[0, 2, 3, 1], [2, 0, 1, 3], [1, 3, 2, 0], [3, 1, 0, 2]] -const cliSelect = require('cli-select') -const simpleGit = require('simple-git') - -const git = simpleGit(process.cwd()) - -const COMMAND = 'npm run bench' -const DEFAULT_BRANCH = 'main' -const PERCENT_THRESHOLD = 5 -const greyColor = '\x1b[30m' -const redColor = '\x1b[31m' const greenColor = '\x1b[32m' +const redColor = '\x1b[31m' const resetColor = '\x1b[0m' -async function selectBranchName (message, branches) { - console.log(message) - const result = await cliSelect({ - type: 'list', - name: 'branch', - values: branches +if (isMainThread) { + main().catch((error) => { + console.error(error) + removeWorktree() + process.exitCode = 1 }) - console.log(result.value) - return result.value +} else { + arm() } -async function executeCommandOnBranch (command, branch) { - console.log(`${greyColor}Checking out "${branch}"${resetColor}`) - await git.checkout(branch) +// One side: builds the serializer of a benchmark, then warms it up or runs a batch when asked +function arm () { + const build = require(workerData.lib) + const { schema, options, input, compile } = require('./bench.js')[workerData.index] + const stringify = compile ? null : build(schema, options) + // read at exit, so the calls are not optimized away + let sink = 0 + + function batch (calls) { + const start = process.hrtime.bigint() + if (compile) { + for (let i = 0; i < calls; i++) sink += build(schema, options)(input).length + } else { + for (let i = 0; i < calls; i++) sink += stringify(input).length + } + return Number(process.hrtime.bigint() - start) + } - console.log(`${greyColor}Execute "${command}"${resetColor}`) - const childProcess = spawn(command, { stdio: 'pipe', shell: true }) + // answers how many calls fill a batch + function warmup (ms) { + let calls = 1 + const end = Date.now() + ms + while (Date.now() < end) { + if (batch(calls) < BATCH_NS) calls *= 2 + } + return Math.max(1, Math.round(calls * BATCH_NS / batch(calls))) + } - let result = '' - childProcess.stdout.on('data', (data) => { - process.stdout.write(data.toString()) - result += data.toString() + parentPort.on('message', ({ ms, calls }) => { + parentPort.postMessage(ms ? warmup(ms) : Math.log(calls * 1e9 / batch(calls))) }) + process.on('exit', () => sink) +} - await new Promise(resolve => childProcess.on('close', resolve)) - - console.log() - - return parseBenchmarksStdout(result) +function median (values) { + const sorted = [...values].sort((a, b) => a - b) + return sorted[sorted.length >> 1] } -function parseBenchmarksStdout (text) { - const results = [] +async function ask (worker, message) { + const answer = once(worker, 'message') + worker.postMessage(message) + return (await answer)[0] +} - const lines = text.split('\n') - for (const line of lines) { - const match = /^(.+?)(\.*) x (.+) ops\/sec .*$/.exec(line) - if (match !== null) { - results.push({ - name: match[1], - alignedName: match[1] + match[2], - result: parseInt(match[3].split(',').join('')) - }) +// Four fresh workers taking turns batch by batch. Answers, in logs and as medians over the rounds, +// B against A, A2 against A1 and B2 against B1 +async function session (libs, index, rounds, number) { + // started and warmed up one at a time, as A B B A or B A A B, so no side is always first: + // warming up four together spread them further apart + const start = ORDERS[number % 2] + const workers = [] + for (const i of start) workers[i] = new Worker(__filename, { workerData: { lib: libs[i], index } }) + try { + const calls = [] + for (const i of start) calls[i] = await ask(workers[i], { ms: WARMUP_MS }) + + const ab = [] + const aa = [] + const bb = [] + for (let round = 0; round < rounds; round++) { + const rate = [] + for (const i of ORDERS[round % ORDERS.length]) rate[i] = await ask(workers[i], { calls: calls[i] }) + const [a1, a2, b1, b2] = rate + ab.push((b1 + b2 - a1 - a2) / 2) + aa.push(a2 - a1) + bb.push(b2 - b1) } + return { ab: median(ab), aa: median(aa), bb: median(bb) } + } finally { + await Promise.all(workers.map((worker) => worker.terminate())) } +} - return results +// A/B is the mean over the sessions, and its band comes from A/A and B/B: under the same code the +// four workers are alike, so A/B spreads as much as A/A and B/B together, halved +async function compare (libs, index, sessions, rounds) { + const results = [] + for (let i = 0; i < sessions; i++) results.push(await session(libs, index, rounds, i)) + + const spread = (key) => Math.sqrt(results.reduce((sum, result) => sum + result[key] ** 2, 0) / sessions) + const aa = spread('aa') + const bb = spread('bb') + const ab = results.reduce((sum, result) => sum + result.ab, 0) / sessions + const band = BAND * Math.sqrt((aa ** 2 + bb ** 2) / 4 / sessions) + const verdict = Math.abs(ab) <= band ? 'noise' : ab > 0 ? 'faster' : 'slower' + return { aa, bb, ab, band, verdict } } -function compareResults (featureBranch, mainBranch) { - for (const { name, alignedName, result: mainBranchResult } of mainBranch) { - const featureBranchBenchmark = featureBranch.find(result => result.name === name) - if (featureBranchBenchmark) { - const featureBranchResult = featureBranchBenchmark.result - const percent = (featureBranchResult - mainBranchResult) * 100 / mainBranchResult - const roundedPercent = Math.round(percent * 100) / 100 - - const percentString = roundedPercent > 0 ? `+${roundedPercent}%` : `${roundedPercent}%` - const message = alignedName + percentString.padStart(7, '.') - - if (roundedPercent > PERCENT_THRESHOLD) { - console.log(`${greenColor}${message}${resetColor}`) - } else if (roundedPercent < -PERCENT_THRESHOLD) { - console.log(`${redColor}${message}${resetColor}`) - } else { - console.log(message) - } +function git (...args) { + return execFileSync('git', args, { cwd: ROOT, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }) +} + +function removeWorktree () { + if (fs.existsSync(WORKTREE)) { + try { + git('worktree', 'remove', '--force', WORKTREE) + } catch { + fs.rmSync(WORKTREE, { recursive: true, force: true }) } } + git('worktree', 'prune') } -(async function () { - const branches = await git.branch() - const currentBranch = branches.branches[branches.current] +// a log of a ratio, as a percentage +const percent = (log) => (Math.expm1(log) * 100).toFixed(1) + '%' - let featureBranch = null - let mainBranch = null +async function main () { + const { values } = parseArgs({ + options: { + against: { type: 'string', default: 'main' }, + only: { type: 'string' }, + sessions: { type: 'string', default: '5' }, + rounds: { type: 'string', default: '20' } + } + }) + const sessions = Number(values.sessions) + const rounds = Number(values.rounds) - if (process.argv[2] === '--ci') { - featureBranch = currentBranch.name - mainBranch = DEFAULT_BRANCH - } else { - featureBranch = await selectBranchName('Select the branch you want to compare (feature branch):', branches.all) - mainBranch = await selectBranchName('Select the branch you want to compare with (main branch):', branches.all) - } + const benchmarks = require('./bench.js') + const selected = benchmarks + .map((benchmark, index) => ({ name: benchmark.name, index })) + .filter(({ name }) => !values.only || name.includes(values.only)) + const width = Math.max(...selected.map(({ name }) => name.length)) + 3 + + removeWorktree() + git('worktree', 'add', '--detach', WORKTREE, values.against) + const libs = [WORKTREE, WORKTREE, ROOT, ROOT] + console.log(`${values.against} (A) against the working tree (B), ${sessions} sessions of ${rounds} rounds\n`) + const counts = { faster: 0, slower: 0, noise: 0, failed: 0 } try { - const featureBranchResult = await executeCommandOnBranch(COMMAND, featureBranch) - const mainBranchResult = await executeCommandOnBranch(COMMAND, mainBranch) - compareResults(featureBranchResult, mainBranchResult) - } catch (error) { - console.error('Switch to origin branch due to an error', error.message) + for (const { name, index } of selected) { + const label = name.padEnd(width, '.') + try { + const { aa, bb, ab, band, verdict } = await compare(libs, index, sessions, rounds) + counts[verdict]++ + const sign = ab > 0 ? '+' : '' + const line = `${label} A/A ±${percent(aa)} B/B ±${percent(bb)} A/B ${sign}${percent(ab)} ±${percent(band)} ${verdict}` + const color = verdict === 'faster' ? greenColor : verdict === 'slower' ? redColor : '' + console.log(color ? `${color}${line}${resetColor}` : line) + } catch (error) { + counts.failed++ + console.log(`${label} failed: ${error.message}`) + } + } + } finally { + removeWorktree() } - await git.checkout(currentBranch.commit) - await git.checkout(currentBranch.name) - - console.log(`${greyColor}Back to ${currentBranch.name} ${currentBranch.commit}${resetColor}`) -})() + console.log(`\n${counts.faster} faster, ${counts.slower} slower, ${counts.noise} within the noise` + + (counts.failed ? `, ${counts.failed} failed` : '')) +} diff --git a/benchmark/bench.js b/benchmark/bench.js index 7a995a4e..a51688cb 100644 --- a/benchmark/bench.js +++ b/benchmark/bench.js @@ -474,4 +474,9 @@ async function runBenchmarks () { } } -runBenchmarks() +// bench-cmp-branch.js runs the same cases +module.exports = benchmarks + +if (require.main === module) { + runBenchmarks() +} diff --git a/package.json b/package.json index 859b702c..a7288050 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,6 @@ "scripts": { "bench": "node --expose-gc ./benchmark/bench.js", "bench:cmp": "node ./benchmark/bench-cmp-branch.js", - "bench:cmp:ci": "node ./benchmark/bench-cmp-branch.js --ci", "benchmark": "node --expose-gc ./benchmark/bench-cmp-lib.js", "lint": "eslint", "lint:fix": "eslint --fix", @@ -63,14 +62,12 @@ ], "devDependencies": { "c8": "^12.0.0", - "cli-select": "^1.1.2", "compile-json-stringify": "^0.1.2", "eslint": "^9.39.2", "fast-json-stringify": ".", "is-my-json-valid": "^2.20.6", "json-accelerator": "^0.1.7", "neostandard": "^0.13.0", - "simple-git": "^3.30.0", "tinybench": "^6.0.0", "tstyche": "^7.1.0", "typebox": "^1.0.81", From b084492a96df431c7e0b64899fc41f4ea2c0ba65 Mon Sep 17 00:00:00 2001 From: Nigro Simone Date: Thu, 24 Sep 2026 21:13:36 +0200 Subject: [PATCH 2/4] chore(benchmark): print the change net of the noise, and keep V8 from hoisting the call --- benchmark/bench-cmp-branch.js | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/benchmark/bench-cmp-branch.js b/benchmark/bench-cmp-branch.js index 1a08053e..ae831bdf 100644 --- a/benchmark/bench-cmp-branch.js +++ b/benchmark/bench-cmp-branch.js @@ -5,8 +5,8 @@ // Each benchmark runs in four workers at once, two on the other revision (A) and two on the working // tree (B), and they take turns in batches of about 10ms, so a machine that warms up or throttles // slows all four together. A against A and B against B is the same code against itself: how far it -// moves is the noise. A/B is followed by the band that noise gives to it, and a row is a result only -// when A/B is outside that band. +// moves is the noise. A/B is followed by the band that noise gives to it: outside the band the row is +// faster or slower, by at least what is left of A/B once the band is taken off. // // npm run bench:cmp main against the working tree // npm run bench:cmp -- --against v7.0.0 @@ -53,15 +53,18 @@ function arm () { const build = require(workerData.lib) const { schema, options, input, compile } = require('./bench.js')[workerData.index] const stringify = compile ? null : build(schema, options) - // read at exit, so the calls are not optimized away - let sink = 0 + // The same input at an index that changes, so V8 cannot move the call out of the loop and write + // the string once, and every result kept, so it cannot drop the call either. With a constant input + // it did one or the other, and two workers on the same code came out up to 3x apart. + const inputs = [input, input] + const results = new Array(16) function batch (calls) { const start = process.hrtime.bigint() if (compile) { - for (let i = 0; i < calls; i++) sink += build(schema, options)(input).length + for (let i = 0; i < calls; i++) results[i & 15] = build(schema, options)(inputs[i & 1]) } else { - for (let i = 0; i < calls; i++) sink += stringify(input).length + for (let i = 0; i < calls; i++) results[i & 15] = stringify(inputs[i & 1]) } return Number(process.hrtime.bigint() - start) } @@ -79,7 +82,6 @@ function arm () { parentPort.on('message', ({ ms, calls }) => { parentPort.postMessage(ms ? warmup(ms) : Math.log(calls * 1e9 / batch(calls))) }) - process.on('exit', () => sink) } function median (values) { @@ -186,7 +188,10 @@ async function main () { const { aa, bb, ab, band, verdict } = await compare(libs, index, sessions, rounds) counts[verdict]++ const sign = ab > 0 ? '+' : '' - const line = `${label} A/A ±${percent(aa)} B/B ±${percent(bb)} A/B ${sign}${percent(ab)} ±${percent(band)} ${verdict}` + // A/B with the band taken off: the part of it the noise cannot explain + const net = Math.abs(Math.expm1(ab - Math.sign(ab) * band)) * 100 + const result = verdict === 'noise' ? verdict : `${verdict} by at least ${net.toFixed(1)}%` + const line = `${label} A/A ±${percent(aa)} B/B ±${percent(bb)} A/B ${sign}${percent(ab)} ±${percent(band)} ${result}` const color = verdict === 'faster' ? greenColor : verdict === 'slower' ? redColor : '' console.log(color ? `${color}${line}${resetColor}` : line) } catch (error) { From af03693e213834ec1ec8180c54855c4122e42ccb Mon Sep 17 00:00:00 2001 From: Nigro Simone Date: Thu, 24 Sep 2026 21:57:35 +0200 Subject: [PATCH 3/4] chore(benchmark): keep the heap from favouring one branch --- benchmark/bench-cmp-branch.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/benchmark/bench-cmp-branch.js b/benchmark/bench-cmp-branch.js index ae831bdf..d24b3dae 100644 --- a/benchmark/bench-cmp-branch.js +++ b/benchmark/bench-cmp-branch.js @@ -20,6 +20,7 @@ const { once } = require('node:events') const { parseArgs } = require('node:util') const fs = require('node:fs') const path = require('node:path') +const v8 = require('node:v8') const ROOT = path.join(__dirname, '..') // inside the repo, so the other revision resolves node_modules from here @@ -53,18 +54,17 @@ function arm () { const build = require(workerData.lib) const { schema, options, input, compile } = require('./bench.js')[workerData.index] const stringify = compile ? null : build(schema, options) - // The same input at an index that changes, so V8 cannot move the call out of the loop and write - // the string once, and every result kept, so it cannot drop the call either. With a constant input - // it did one or the other, and two workers on the same code came out up to 3x apart. + // the input at an index that changes and the last two results kept, so V8 can neither write the + // string once outside the loop nor drop the call; more results kept put the GC in the measure const inputs = [input, input] - const results = new Array(16) + const results = [null, null] function batch (calls) { const start = process.hrtime.bigint() if (compile) { - for (let i = 0; i < calls; i++) results[i & 15] = build(schema, options)(inputs[i & 1]) + for (let i = 0; i < calls; i++) results[i & 1] = build(schema, options)(inputs[i & 1]) } else { - for (let i = 0; i < calls; i++) results[i & 15] = stringify(inputs[i & 1]) + for (let i = 0; i < calls; i++) results[i & 1] = stringify(inputs[i & 1]) } return Number(process.hrtime.bigint() - start) } @@ -166,6 +166,10 @@ async function main () { rounds: { type: 'string', default: '20' } } }) + // every worker with the young generation at its largest, or it grows with what the build allocated + // and a branch whose build allocates more runs the same code with fewer GCs + v8.setFlagsFromString('--min-semi-space-size=16') + const sessions = Number(values.sessions) const rounds = Number(values.rounds) From 050ad0aafd07de40e70e5dd47ce0897e3a09e313 Mon Sep 17 00:00:00 2001 From: Nigro Simone Date: Fri, 25 Sep 2026 05:04:37 +0200 Subject: [PATCH 4/4] chore(benchmark): use the long strings in the long strings row --- benchmark/bench.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/bench.js b/benchmark/bench.js index a51688cb..4195d41e 100644 --- a/benchmark/bench.js +++ b/benchmark/bench.js @@ -160,7 +160,7 @@ const benchmarks = [ type: 'array', items: { type: 'string' } }, - input: shortArrayOfShortStrings + input: shortArrayOfLongStrings }, { name: 'short array of objects with properties of different types',