chore(benchmark): compare branches against the noise of the same code

.gitignore

@@ -152,3 +152,6 @@ yarn.lock
 .pi
 AGENTS.md
 CLAUDE.md
+
+# the other revision benchmark/bench-cmp-branch.js measures against
+.bench-worktree

benchmark/bench-cmp-branch.js

@@ -1,116 +1,212 @@
 'use strict'
 
-const { spawn } = require('child_process')
+// Measures the working tree against another revision, with the noise of this machine beside it.
+//
+// Each benchmark runs in four workers at once, two on the other revision (A) and two on the working
+// tree (B), and they take turns in batches of about 10ms, so a machine that warms up or throttles
+// slows all four together. A against A and B against B is the same code against itself: how far it
+// moves is the noise. A/B is followed by the band that noise gives to it: outside the band the row is
+// faster or slower, by at least what is left of A/B once the band is taken off.
+//
+//   npm run bench:cmp                                     main against the working tree
+//   npm run bench:cmp -- --against v7.0.0
+//   npm run bench:cmp -- --only "short string" --sessions 10
+//
+// On a clean tree, --against HEAD puts the same code on both sides and every row should be noise.
+
+const { Worker, isMainThread, workerData, parentPort } = require('node:worker_threads')
+const { execFileSync } = require('node:child_process')
+const { once } = require('node:events')
+const { parseArgs } = require('node:util')
+const fs = require('node:fs')
+const path = require('node:path')
+const v8 = require('node:v8')
+
+const ROOT = path.join(__dirname, '..')
+// inside the repo, so the other revision resolves node_modules from here
+const WORKTREE = path.join(ROOT, '.bench-worktree')
+const WARMUP_MS = 250
+const BATCH_NS = 1e7
+// how many standard errors A/B has to be from zero: with 5 sessions, about 1 row in 400 of the
+// same code against itself
+const BAND = 4
+
+// the four workers are a1 a2 b1 b2; turning through these orders keeps a drift of the machine out
+// of A/B in every round, and out of A/A and B/B every four rounds
+const ORDERS = [[0, 2, 3, 1], [2, 0, 1, 3], [1, 3, 2, 0], [3, 1, 0, 2]]
 
-const cliSelect = require('cli-select')
-const simpleGit = require('simple-git')
-
-const git = simpleGit(process.cwd())
-
-const COMMAND = 'npm run bench'
-const DEFAULT_BRANCH = 'main'
-const PERCENT_THRESHOLD = 5
-const greyColor = '\x1b[30m'
-const redColor = '\x1b[31m'
 const greenColor = '\x1b[32m'
+const redColor = '\x1b[31m'
 const resetColor = '\x1b[0m'
 
-async function selectBranchName (message, branches) {
-  console.log(message)
-  const result = await cliSelect({
-    type: 'list',
-    name: 'branch',
-    values: branches
+if (isMainThread) {
+  main().catch((error) => {
+    console.error(error)
+    removeWorktree()
+    process.exitCode = 1
   })
-  console.log(result.value)
-  return result.value
+} else {
+  arm()
 }
 
-async function executeCommandOnBranch (command, branch) {
-  console.log(`${greyColor}Checking out "${branch}"${resetColor}`)
-  await git.checkout(branch)
+// One side: builds the serializer of a benchmark, then warms it up or runs a batch when asked
+function arm () {
+  const build = require(workerData.lib)
+  const { schema, options, input, compile } = require('./bench.js')[workerData.index]
+  const stringify = compile ? null : build(schema, options)
+  // the input at an index that changes and the last two results kept, so V8 can neither write the
+  // string once outside the loop nor drop the call; more results kept put the GC in the measure
+  const inputs = [input, input]
+  const results = [null, null]
+
+  function batch (calls) {
+    const start = process.hrtime.bigint()
+    if (compile) {
+      for (let i = 0; i < calls; i++) results[i & 1] = build(schema, options)(inputs[i & 1])
+    } else {
+      for (let i = 0; i < calls; i++) results[i & 1] = stringify(inputs[i & 1])
+    }
+    return Number(process.hrtime.bigint() - start)
+  }
 
-  console.log(`${greyColor}Execute "${command}"${resetColor}`)
-  const childProcess = spawn(command, { stdio: 'pipe', shell: true })
+  // answers how many calls fill a batch
+  function warmup (ms) {
+    let calls = 1
+    const end = Date.now() + ms
+    while (Date.now() < end) {
+      if (batch(calls) < BATCH_NS) calls *= 2
+    }
+    return Math.max(1, Math.round(calls * BATCH_NS / batch(calls)))
+  }
 
-  let result = ''
-  childProcess.stdout.on('data', (data) => {
-    process.stdout.write(data.toString())
-    result += data.toString()
+  parentPort.on('message', ({ ms, calls }) => {
+    parentPort.postMessage(ms ? warmup(ms) : Math.log(calls * 1e9 / batch(calls)))
   })
+}
 
-  await new Promise(resolve => childProcess.on('close', resolve))
-
-  console.log()
-
-  return parseBenchmarksStdout(result)
+function median (values) {
+  const sorted = [...values].sort((a, b) => a - b)
+  return sorted[sorted.length >> 1]
 }
 
-function parseBenchmarksStdout (text) {
-  const results = []
+async function ask (worker, message) {
+  const answer = once(worker, 'message')
+  worker.postMessage(message)
+  return (await answer)[0]
+}
 
-  const lines = text.split('\n')
-  for (const line of lines) {
-    const match = /^(.+?)(\.*) x (.+) ops\/sec .*$/.exec(line)
-    if (match !== null) {
-      results.push({
-        name: match[1],
-        alignedName: match[1] + match[2],
-        result: parseInt(match[3].split(',').join(''))
-      })
+// Four fresh workers taking turns batch by batch. Answers, in logs and as medians over the rounds,
+// B against A, A2 against A1 and B2 against B1
+async function session (libs, index, rounds, number) {
+  // started and warmed up one at a time, as A B B A or B A A B, so no side is always first:
+  // warming up four together spread them further apart
+  const start = ORDERS[number % 2]
+  const workers = []
+  for (const i of start) workers[i] = new Worker(__filename, { workerData: { lib: libs[i], index } })
+  try {
+    const calls = []
+    for (const i of start) calls[i] = await ask(workers[i], { ms: WARMUP_MS })
+
+    const ab = []
+    const aa = []
+    const bb = []
+    for (let round = 0; round < rounds; round++) {
+      const rate = []
+      for (const i of ORDERS[round % ORDERS.length]) rate[i] = await ask(workers[i], { calls: calls[i] })
+      const [a1, a2, b1, b2] = rate
+      ab.push((b1 + b2 - a1 - a2) / 2)
+      aa.push(a2 - a1)
+      bb.push(b2 - b1)
     }
+    return { ab: median(ab), aa: median(aa), bb: median(bb) }
+  } finally {
+    await Promise.all(workers.map((worker) => worker.terminate()))
   }
+}
 
-  return results
+// A/B is the mean over the sessions, and its band comes from A/A and B/B: under the same code the
+// four workers are alike, so A/B spreads as much as A/A and B/B together, halved
+async function compare (libs, index, sessions, rounds) {
+  const results = []
+  for (let i = 0; i < sessions; i++) results.push(await session(libs, index, rounds, i))
+
+  const spread = (key) => Math.sqrt(results.reduce((sum, result) => sum + result[key] ** 2, 0) / sessions)
+  const aa = spread('aa')
+  const bb = spread('bb')
+  const ab = results.reduce((sum, result) => sum + result.ab, 0) / sessions
+  const band = BAND * Math.sqrt((aa ** 2 + bb ** 2) / 4 / sessions)
+  const verdict = Math.abs(ab) <= band ? 'noise' : ab > 0 ? 'faster' : 'slower'
+  return { aa, bb, ab, band, verdict }
 }
 
-function compareResults (featureBranch, mainBranch) {
-  for (const { name, alignedName, result: mainBranchResult } of mainBranch) {
-    const featureBranchBenchmark = featureBranch.find(result => result.name === name)
-    if (featureBranchBenchmark) {
-      const featureBranchResult = featureBranchBenchmark.result
-      const percent = (featureBranchResult - mainBranchResult) * 100 / mainBranchResult
-      const roundedPercent = Math.round(percent * 100) / 100
-
-      const percentString = roundedPercent > 0 ? `+${roundedPercent}%` : `${roundedPercent}%`
-      const message = alignedName + percentString.padStart(7, '.')
-
-      if (roundedPercent > PERCENT_THRESHOLD) {
-        console.log(`${greenColor}${message}${resetColor}`)
-      } else if (roundedPercent < -PERCENT_THRESHOLD) {
-        console.log(`${redColor}${message}${resetColor}`)
-      } else {
-        console.log(message)
-      }
+function git (...args) {
+  return execFileSync('git', args, { cwd: ROOT, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] })
+}
+
+function removeWorktree () {
+  if (fs.existsSync(WORKTREE)) {
+    try {
+      git('worktree', 'remove', '--force', WORKTREE)
+    } catch {
+      fs.rmSync(WORKTREE, { recursive: true, force: true })
     }
   }
+  git('worktree', 'prune')
 }
 
-(async function () {
-  const branches = await git.branch()
-  const currentBranch = branches.branches[branches.current]
+// a log of a ratio, as a percentage
+const percent = (log) => (Math.expm1(log) * 100).toFixed(1) + '%'
+
+async function main () {
+  const { values } = parseArgs({
+    options: {
+      against: { type: 'string', default: 'main' },
+      only: { type: 'string' },
+      sessions: { type: 'string', default: '5' },
+      rounds: { type: 'string', default: '20' }
+    }
+  })
+  // every worker with the young generation at its largest, or it grows with what the build allocated
+  // and a branch whose build allocates more runs the same code with fewer GCs
+  v8.setFlagsFromString('--min-semi-space-size=16')
+
+  const sessions = Number(values.sessions)
+  const rounds = Number(values.rounds)
 
-  let featureBranch = null
-  let mainBranch = null
+  const benchmarks = require('./bench.js')
+  const selected = benchmarks
+    .map((benchmark, index) => ({ name: benchmark.name, index }))
+    .filter(({ name }) => !values.only || name.includes(values.only))
+  const width = Math.max(...selected.map(({ name }) => name.length)) + 3
 
-  if (process.argv[2] === '--ci') {
-    featureBranch = currentBranch.name
-    mainBranch = DEFAULT_BRANCH
-  } else {
-    featureBranch = await selectBranchName('Select the branch you want to compare (feature branch):', branches.all)
-    mainBranch = await selectBranchName('Select the branch you want to compare with (main branch):', branches.all)
-  }
+  removeWorktree()
+  git('worktree', 'add', '--detach', WORKTREE, values.against)
+  const libs = [WORKTREE, WORKTREE, ROOT, ROOT]
 
+  console.log(`${values.against} (A) against the working tree (B), ${sessions} sessions of ${rounds} rounds\n`)
+  const counts = { faster: 0, slower: 0, noise: 0, failed: 0 }
   try {
-    const featureBranchResult = await executeCommandOnBranch(COMMAND, featureBranch)
-    const mainBranchResult = await executeCommandOnBranch(COMMAND, mainBranch)
-    compareResults(featureBranchResult, mainBranchResult)
-  } catch (error) {
-    console.error('Switch to origin branch due to an error', error.message)
+    for (const { name, index } of selected) {
+      const label = name.padEnd(width, '.')
+      try {
+        const { aa, bb, ab, band, verdict } = await compare(libs, index, sessions, rounds)
+        counts[verdict]++
+        const sign = ab > 0 ? '+' : ''
+        // A/B with the band taken off: the part of it the noise cannot explain
+        const net = Math.abs(Math.expm1(ab - Math.sign(ab) * band)) * 100
+        const result = verdict === 'noise' ? verdict : `${verdict} by at least ${net.toFixed(1)}%`
+        const line = `${label} A/A ±${percent(aa)}  B/B ±${percent(bb)}  A/B ${sign}${percent(ab)} ±${percent(band)}  ${result}`
+        const color = verdict === 'faster' ? greenColor : verdict === 'slower' ? redColor : ''
+        console.log(color ? `${color}${line}${resetColor}` : line)
+      } catch (error) {
+        counts.failed++
+        console.log(`${label} failed: ${error.message}`)
+      }
+    }
+  } finally {
+    removeWorktree()
   }
 
-  await git.checkout(currentBranch.commit)
-  await git.checkout(currentBranch.name)
-
-  console.log(`${greyColor}Back to ${currentBranch.name} ${currentBranch.commit}${resetColor}`)
-})()
+  console.log(`\n${counts.faster} faster, ${counts.slower} slower, ${counts.noise} within the noise` +
+    (counts.failed ? `, ${counts.failed} failed` : ''))
+}

benchmark/bench.js

@@ -160,7 +160,7 @@ const benchmarks = [
       type: 'array',
       items: { type: 'string' }
     },
-    input: shortArrayOfShortStrings
+    input: shortArrayOfLongStrings
   },
   {
     name: 'short array of objects with properties of different types',
@@ -474,4 +474,9 @@ async function runBenchmarks () {
   }
 }
 
-runBenchmarks()
+// bench-cmp-branch.js runs the same cases
+module.exports = benchmarks
+
+if (require.main === module) {
+  runBenchmarks()
+}

package.json

@@ -8,7 +8,6 @@
   "scripts": {
     "bench": "node --expose-gc ./benchmark/bench.js",
     "bench:cmp": "node ./benchmark/bench-cmp-branch.js",
-    "bench:cmp:ci": "node ./benchmark/bench-cmp-branch.js --ci",
     "benchmark": "node --expose-gc ./benchmark/bench-cmp-lib.js",
     "lint": "eslint",
     "lint:fix": "eslint --fix",
@@ -63,14 +62,12 @@
   ],
   "devDependencies": {
     "c8": "^12.0.0",
-    "cli-select": "^1.1.2",
     "compile-json-stringify": "^0.1.2",
     "eslint": "^9.39.2",
     "fast-json-stringify": ".",
     "is-my-json-valid": "^2.20.6",
     "json-accelerator": "^0.1.7",
     "neostandard": "^0.13.0",
-    "simple-git": "^3.30.0",
     "tinybench": "^6.0.0",
     "tstyche": "^7.1.0",
     "typebox": "^1.0.81",