From 4bff510e9bcda1fe357d16e077e78c2ceb2e5f9a Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:24:56 -0700 Subject: [PATCH 1/4] Support Azure Linux benchmark pools Allow the temporary runner path to select a named 1ES pool and image without changing the image used by deployment jobs. Provision the existing sandboxes with distro-appropriate packages and verify the official gVisor archive before installing its runtime files. --- build/benchmark.yml | 17 +++- build/templates/installSandbox.yml | 45 ++++++++-- scripts/src/__tests__/scenarioSetup.test.ts | 98 ++++++++++++++++++--- 3 files changed, 139 insertions(+), 21 deletions(-) diff --git a/build/benchmark.yml b/build/benchmark.yml index 9aad6259..caa71e63 100644 --- a/build/benchmark.yml +++ b/build/benchmark.yml @@ -71,6 +71,16 @@ parameters: type: boolean default: true + - name: HOSTED_POOL + displayName: Optional 1ES benchmark pool + type: string + default: '' + + - name: HOSTED_IMAGE + displayName: Image in the 1ES benchmark pool + type: string + default: azure-linux-3 + # PR trigger params - name: DISTINCT_ID displayName: Distinct ID for this run (PR runs only) @@ -363,7 +373,12 @@ jobs: condition: and(succeeded(), not(eq(dependencies.Setup.outputs['setupPipeline.MATRIX_${{ replace(agent, '-', '_') }}'], '{}'))) pool: ${{ if parameters.USE_HOSTED_RUNNERS }}: - vmImage: ubuntu-latest + ${{ if eq(parameters.HOSTED_POOL, '') }}: + vmImage: ubuntu-latest + ${{ else }}: + name: ${{ parameters.HOSTED_POOL }} + demands: + - ImageOverride -equals ${{ parameters.HOSTED_IMAGE }} ${{ else }}: name: ts-perf-ddfun ${{ if not(eq(agent, 'any')) }}: diff --git a/build/templates/installSandbox.yml b/build/templates/installSandbox.yml index 92b29d46..acb40400 100644 --- a/build/templates/installSandbox.yml +++ b/build/templates/installSandbox.yml @@ -6,8 +6,17 @@ parameters: steps: - bash: | set -exo pipefail - sudo apt-get update - sudo apt-get install -y bubblewrap + if ! command -v bwrap >/dev/null; then + if command -v apt-get >/dev/null; then + sudo apt-get update + sudo apt-get install -y bubblewrap + elif command -v tdnf >/dev/null; then + sudo tdnf install -y bubblewrap + else + echo "Unsupported package manager for bubblewrap installation" >&2 + exit 1 + fi + fi # Ubuntu 24.04 blocks unprivileged user namespaces unless an AppArmor profile # allows them. Allow them for bubblewrap only, like Ubuntu's browser profiles. @@ -33,13 +42,33 @@ steps: # Only dependency installation uses gVisor; measured compilers use bubblewrap. - bash: | set -exo pipefail - sudo apt-get install -y apt-transport-https ca-certificates curl gnupg - curl -fsSL https://gvisor.dev/archive.key | sudo gpg --dearmor -o /usr/share/keyrings/gvisor-archive-keyring.gpg - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/gvisor-archive-keyring.gpg] https://storage.googleapis.com/gvisor/releases release main" | sudo tee /etc/apt/sources.list.d/gvisor.list > /dev/null - sudo apt-get update - sudo apt-get install -y runsc - sudo runsc install + if command -v apt-get >/dev/null; then + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg acl + curl -fsSL https://gvisor.dev/archive.key | sudo gpg --dearmor -o /usr/share/keyrings/gvisor-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/gvisor-archive-keyring.gpg] https://storage.googleapis.com/gvisor/releases release main" | sudo tee /etc/apt/sources.list.d/gvisor.list > /dev/null + sudo apt-get update + sudo apt-get install -y runsc + RUNSC=/usr/bin/runsc + elif command -v tdnf >/dev/null; then + sudo tdnf install -y ca-certificates curl moby-engine moby-cli util-linux acl tar bzip2 + INSTALL_DIR=$(mktemp -d "${AGENT_TEMPDIRECTORY:?}/tsperf-runsc.XXXXXX") + trap 'rm -f "$INSTALL_DIR/gvisor.tar.bz2" "$INSTALL_DIR/gvisor.tar.bz2.sha512"; rmdir "$INSTALL_DIR"' EXIT + URL="https://storage.googleapis.com/gvisor/releases/release/latest/$(uname -m)" + curl -fsSL "$URL/gvisor.tar.bz2" -o "$INSTALL_DIR/gvisor.tar.bz2" + curl -fsSL "$URL/gvisor.tar.bz2.sha512" -o "$INSTALL_DIR/gvisor.tar.bz2.sha512" + (cd "$INSTALL_DIR" && sha512sum --check gvisor.tar.bz2.sha512) + sudo tar -xjf "$INSTALL_DIR/gvisor.tar.bz2" -C /usr/local/bin --no-same-owner --no-same-permissions + RUNSC=/usr/local/bin/runsc + else + echo "Unsupported package manager for scenario sandbox installation" >&2 + exit 1 + fi + sudo "$RUNSC" install sudo systemctl restart docker + # New Docker installations do not grant the running agent access to its socket. + if [ ! -w /var/run/docker.sock ]; then + sudo setfacl -m "u:$(id -u):rw" /var/run/docker.sock + fi docker info target: commands: restricted diff --git a/scripts/src/__tests__/scenarioSetup.test.ts b/scripts/src/__tests__/scenarioSetup.test.ts index 00930fc4..6a88f715 100644 --- a/scripts/src/__tests__/scenarioSetup.test.ts +++ b/scripts/src/__tests__/scenarioSetup.test.ts @@ -20,23 +20,24 @@ const sandboxTemplate = parse( }; const tempDirs: string[] = []; -function hostedCpuScript() { - const hostedSteps = sandboxTemplate.steps.find(step => "${{ if parameters.HOSTED_BENCHMARK }}" in step); - if (!hostedSteps || !("${{ if parameters.HOSTED_BENCHMARK }}" in hostedSteps)) { - throw new Error("Missing hosted sandbox setup"); - } - const cpuStep = hostedSteps["${{ if parameters.HOSTED_BENCHMARK }}"] - .find(step => step.displayName === "Configure hosted benchmark CPUs"); - if (!cpuStep) { - throw new Error("Missing hosted CPU configuration"); +function sandboxScript(displayName: string) { + const steps = sandboxTemplate.steps.flatMap(step => + "bash" in step ? [step] : step["${{ if parameters.HOSTED_BENCHMARK }}"] + ); + const step = steps.find(step => step.displayName === displayName); + if (!step) { + throw new Error(`Missing sandbox step: ${displayName}`); } - return cpuStep.bash; + return step.bash; } test("hosted CPU configuration selects CPUs the benchmark process can actually use", () => { const allowed = fs.readFileSync("/proc/self/status", "utf8").match(/^Cpus_allowed_list:\s*(.+)$/m)?.[1]; expect(allowed).toBeTruthy(); - const result = spawnSync("bash", ["-c", hostedCpuScript()], { encoding: "utf8", timeout: 10_000 }); + const result = spawnSync("bash", ["-c", sandboxScript("Configure hosted benchmark CPUs")], { + encoding: "utf8", + timeout: 10_000, + }); expect(result.status).toBe(0); expect(result.stdout).toContain( `##vso[task.setvariable variable=TSPERF_AGENT_BENCHMARK_CPUS;isReadOnly=true]${allowed}`, @@ -48,7 +49,7 @@ test("hosted CPU configuration does not publish unusable affinity", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "typescript-benchmarking-hosted-")); tempDirs.push(tempDir); fs.writeFileSync(path.join(tempDir, "taskset"), "#!/bin/sh\nexit 37\n", { mode: 0o755 }); - const result = spawnSync("bash", ["-c", hostedCpuScript()], { + const result = spawnSync("bash", ["-c", sandboxScript("Configure hosted benchmark CPUs")], { env: { ...process.env, PATH: `${tempDir}:${process.env.PATH}` }, encoding: "utf8", timeout: 10_000, @@ -57,6 +58,79 @@ test("hosted CPU configuration does not publish unusable affinity", () => { expect(result.stdout).not.toContain("task.setvariable"); }); +function runAzureLinuxSetup(env: Record = {}) { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "typescript-benchmarking-azurelinux-")); + tempDirs.push(tempDir); + const commands = path.join(tempDir, "commands"); + const script = ` +command() { + if [ "$1" = "-v" ]; then + case "$2" in + apt-get) return 1 ;; + tdnf) return 0 ;; + esac + fi + builtin command "$@" +} +sudo() { + printf '%s\\n' "$*" >> "$COMMANDS" + if [ "$1" = "tdnf" ] && [ "\${FAIL_PACKAGES:-}" = "true" ]; then + return 37 + fi +} +curl() { + if [[ "$2" == */gvisor.tar.bz2.sha512 ]]; then + (cd "\${4%/*}" && sha512sum gvisor.tar.bz2) > "$4" + if [ "\${CORRUPT_DOWNLOAD:-}" = "true" ]; then + printf 'modified' >> "\${4%/*}/gvisor.tar.bz2" + fi + else + printf '#!/bin/sh\\nexit 0\\n' > "$4" + fi +} +docker() { printf '%s\\n' "docker $*" >> "$COMMANDS"; } +${sandboxScript("Install scenario sandbox")} +`; + const result = spawnSync("bash", ["-c", script], { + env: { + ...process.env, + AGENT_TEMPDIRECTORY: tempDir, + COMMANDS: commands, + ...env, + }, + encoding: "utf8", + timeout: 10_000, + }); + return { result, commands: fs.readFileSync(commands, "utf8"), files: fs.readdirSync(tempDir) }; +} + +test("Azure Linux installs and verifies gVisor before configuring Docker", () => { + const { result, commands, files } = runAzureLinuxSetup(); + expect(result.status).toBe(0); + expect(commands).toContain("tdnf install -y ca-certificates curl moby-engine moby-cli util-linux acl tar bzip2"); + expect(commands).toContain("tar -xjf"); + expect(commands).toContain("/usr/local/bin/runsc install\nsystemctl restart docker\n"); + expect(commands).toContain("docker info"); + expect(files).toEqual(["commands"]); +}); + +test("Azure Linux rejects a corrupted gVisor download before installing or starting it", () => { + const { result, commands, files } = runAzureLinuxSetup({ CORRUPT_DOWNLOAD: "true" }); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("checksum did NOT match"); + expect(commands).not.toContain("tar -xjf"); + expect(commands).not.toContain("systemctl"); + expect(files).toEqual(["commands"]); +}); + +test("Azure Linux stops when sandbox prerequisites cannot be installed", () => { + const { result, commands, files } = runAzureLinuxSetup({ FAIL_PACKAGES: "true" }); + expect(result.status).toBe(37); + expect(commands).not.toContain("runsc install"); + expect(commands).not.toContain("systemctl"); + expect(files).toEqual(["commands"]); +}); + afterEach(() => { for (const tempDir of tempDirs.splice(0)) { fs.rmSync(tempDir, { recursive: true }); From 2444c473f870cc0a628e9ee1383c42c66d20b225 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:52:21 -0700 Subject: [PATCH 2/4] Install Buildx for Azure Linux scenario images The legacy Docker builder drops heredoc contents from the proxy Dockerfile, leaving the dependency sandbox proxy unable to start. Install the Azure Linux Buildx package and require the plugin before configuring the scenario runtime. --- build/templates/installSandbox.yml | 3 ++- scripts/src/__tests__/scenarioSetup.test.ts | 21 ++++++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/build/templates/installSandbox.yml b/build/templates/installSandbox.yml index acb40400..7d3ba0db 100644 --- a/build/templates/installSandbox.yml +++ b/build/templates/installSandbox.yml @@ -50,7 +50,7 @@ steps: sudo apt-get install -y runsc RUNSC=/usr/bin/runsc elif command -v tdnf >/dev/null; then - sudo tdnf install -y ca-certificates curl moby-engine moby-cli util-linux acl tar bzip2 + sudo tdnf install -y ca-certificates curl moby-engine moby-cli moby-buildx util-linux acl tar bzip2 INSTALL_DIR=$(mktemp -d "${AGENT_TEMPDIRECTORY:?}/tsperf-runsc.XXXXXX") trap 'rm -f "$INSTALL_DIR/gvisor.tar.bz2" "$INSTALL_DIR/gvisor.tar.bz2.sha512"; rmdir "$INSTALL_DIR"' EXIT URL="https://storage.googleapis.com/gvisor/releases/release/latest/$(uname -m)" @@ -63,6 +63,7 @@ steps: echo "Unsupported package manager for scenario sandbox installation" >&2 exit 1 fi + docker buildx version sudo "$RUNSC" install sudo systemctl restart docker # New Docker installations do not grant the running agent access to its socket. diff --git a/scripts/src/__tests__/scenarioSetup.test.ts b/scripts/src/__tests__/scenarioSetup.test.ts index 6a88f715..faa51e25 100644 --- a/scripts/src/__tests__/scenarioSetup.test.ts +++ b/scripts/src/__tests__/scenarioSetup.test.ts @@ -88,7 +88,12 @@ curl() { printf '#!/bin/sh\\nexit 0\\n' > "$4" fi } -docker() { printf '%s\\n' "docker $*" >> "$COMMANDS"; } +docker() { + printf '%s\\n' "docker $*" >> "$COMMANDS" + if [ "$1" = "buildx" ] && [ "\${FAIL_BUILDX:-}" = "true" ]; then + return 37 + fi +} ${sandboxScript("Install scenario sandbox")} `; const result = spawnSync("bash", ["-c", script], { @@ -107,9 +112,11 @@ ${sandboxScript("Install scenario sandbox")} test("Azure Linux installs and verifies gVisor before configuring Docker", () => { const { result, commands, files } = runAzureLinuxSetup(); expect(result.status).toBe(0); - expect(commands).toContain("tdnf install -y ca-certificates curl moby-engine moby-cli util-linux acl tar bzip2"); + expect(commands).toContain( + "tdnf install -y ca-certificates curl moby-engine moby-cli moby-buildx util-linux acl tar bzip2", + ); expect(commands).toContain("tar -xjf"); - expect(commands).toContain("/usr/local/bin/runsc install\nsystemctl restart docker\n"); + expect(commands).toContain("docker buildx version\n/usr/local/bin/runsc install\nsystemctl restart docker\n"); expect(commands).toContain("docker info"); expect(files).toEqual(["commands"]); }); @@ -131,6 +138,14 @@ test("Azure Linux stops when sandbox prerequisites cannot be installed", () => { expect(files).toEqual(["commands"]); }); +test("Azure Linux stops before configuring Docker if Buildx is unavailable", () => { + const { result, commands, files } = runAzureLinuxSetup({ FAIL_BUILDX: "true" }); + expect(result.status).toBe(37); + expect(commands).not.toContain("runsc install"); + expect(commands).not.toContain("systemctl"); + expect(files).toEqual(["commands"]); +}); + afterEach(() => { for (const tempDir of tempDirs.splice(0)) { fs.rmSync(tempDir, { recursive: true }); From f4e29be188c3228f869f478d2f05516f8c8ad626 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:59:43 -0700 Subject: [PATCH 3/4] Wait for idle hosted benchmark machines Scenario setup can leave CPU activity and resource pressure behind. Require a bounded quiet interval before each side of hosted comparisons, and report the sampled conditions rather than filtering measured results. Keep dedicated-agent tuning unchanged. --- build/benchmark.yml | 15 +++ .../__tests__/waitForBenchmarkIdle.test.ts | 109 +++++++++++++++ scripts/src/waitForBenchmarkIdle.ts | 124 ++++++++++++++++++ 3 files changed, 248 insertions(+) create mode 100644 scripts/src/__tests__/waitForBenchmarkIdle.test.ts create mode 100644 scripts/src/waitForBenchmarkIdle.ts diff --git a/build/benchmark.yml b/build/benchmark.yml index caa71e63..3b8ea432 100644 --- a/build/benchmark.yml +++ b/build/benchmark.yml @@ -549,6 +549,14 @@ jobs: - BWRAP_ARGS displayName: Create benchmark sandbox + - ${{ if parameters.USE_HOSTED_RUNNERS }}: + - bash: node $(BENCH_SCRIPTS)/waitForBenchmarkIdle.js + target: + commands: restricted + settableVariables: none + displayName: Wait for idle before new benchmark + condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true')) + - bash: | set -exo pipefail bash $(Pipeline.Workspace)/typescript-benchmarking/scripts/runBenchmarkSandbox.sh \ @@ -586,6 +594,13 @@ jobs: displayName: Publish scenario ref for baseline benchmark condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true')) + - ${{ if parameters.USE_HOSTED_RUNNERS }}: + - bash: node $(BENCH_SCRIPTS)/waitForBenchmarkIdle.js + target: + commands: restricted + settableVariables: none + displayName: Wait for idle before baseline benchmark + - bash: | set -exo pipefail bash $(Pipeline.Workspace)/typescript-benchmarking/scripts/runBenchmarkSandbox.sh \ diff --git a/scripts/src/__tests__/waitForBenchmarkIdle.test.ts b/scripts/src/__tests__/waitForBenchmarkIdle.test.ts new file mode 100644 index 00000000..0c52b53a --- /dev/null +++ b/scripts/src/__tests__/waitForBenchmarkIdle.test.ts @@ -0,0 +1,109 @@ +import fs from "node:fs"; + +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +import { waitForBenchmarkIdle } from "../waitForBenchmarkIdle.js"; + +beforeEach(() => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] }); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); +}); + +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +function mockCounters(options: { + busy?: (second: number, cpu: number) => number; + pressure?: (second: number, resource: string) => number; + pressureError?: string; +} = {}) { + let second = -1; + const ticks = Array(8).fill(0); + const busy = Array(8).fill(0); + const pressure = new Map(); + return vi.spyOn(fs, "readFileSync").mockImplementation(file => { + if (file === "/proc/stat") { + second++; + return ticks.map((_, cpu) => { + ticks[cpu] += 100; + busy[cpu] += options.busy?.(second, cpu) ?? 3; + return `cpu${cpu} ${busy[cpu]} 0 0 ${ticks[cpu] - busy[cpu]} 0 0 0 0 ${busy[cpu]} 0`; + }).join("\n"); + } + const resource = String(file).split("/").at(-1)!; + if (options.pressureError) { + throw Object.assign(new Error("cannot read pressure"), { code: options.pressureError }); + } + const total = (pressure.get(resource) ?? 0) + (options.pressure?.(second, resource) ?? 0); + pressure.set(resource, total); + return `some avg10=99.99 avg60=99.99 avg300=99.99 total=${total}\nfull avg10=0 total=0\n`; + }); +} + +test("requires five quiet seconds and uses deltas rather than stale averages or double-counted guest time", async () => { + const read = mockCounters(); + const waiting = waitForBenchmarkIdle(); + await vi.advanceTimersByTimeAsync(4000); + expect(console.log).not.toHaveBeenCalledWith(expect.stringContaining("Machine is quiet")); + await vi.advanceTimersByTimeAsync(1000); + await waiting; + expect(console.log).toHaveBeenCalledWith(expect.stringContaining("Machine is quiet")); + expect(read).toHaveBeenCalledTimes(24); +}); + +test("restarts the quiet window when one CPU becomes busy", async () => { + mockCounters({ busy: (second, cpu) => second === 3 && cpu === 0 ? 30 : 0 }); + const waiting = waitForBenchmarkIdle(); + await vi.advanceTimersByTimeAsync(7000); + expect(console.log).not.toHaveBeenCalledWith(expect.stringContaining("Machine is quiet")); + await vi.advanceTimersByTimeAsync(1000); + await waiting; +}); + +test.each(["cpu", "io", "memory"])("waits for recent %s pressure to subside", async resource => { + mockCounters({ pressure: (second, name) => second <= 3 && name === resource ? 20_000 : 0 }); + const waiting = waitForBenchmarkIdle(); + await vi.advanceTimersByTimeAsync(8000); + await waiting; + expect(console.log).toHaveBeenCalledWith(expect.stringContaining(`${resource}=2.00%`)); + expect(console.log).toHaveBeenCalledWith(expect.stringContaining("Machine is quiet")); +}); + +test("fails within the time bound instead of starting measurements on a persistently busy machine", async () => { + mockCounters({ busy: () => 50 }); + const waiting = expect(waitForBenchmarkIdle()).rejects.toThrow("Machine did not become quiet within 60s"); + await vi.advanceTimersByTimeAsync(60_000); + await waiting; + expect(console.log).not.toHaveBeenCalledWith(expect.stringContaining("Machine is quiet")); + expect(vi.getTimerCount()).toBe(0); +}); + +test("explicitly warns once per unavailable pressure signal and still checks CPU counters", async () => { + mockCounters({ pressureError: "ENOENT" }); + const waiting = waitForBenchmarkIdle(); + await vi.advanceTimersByTimeAsync(5000); + await waiting; + expect(console.warn).toHaveBeenCalledTimes(3); + expect(console.warn).toHaveBeenCalledWith(expect.stringContaining("cpu pressure counters unavailable")); + expect(console.log).toHaveBeenCalledWith(expect.stringContaining("Machine is quiet")); +}); + +test("does not suppress permission errors reading telemetry", async () => { + mockCounters({ pressureError: "EACCES" }); + await expect(waitForBenchmarkIdle()).rejects.toMatchObject({ code: "EACCES" }); +}); + +test("rejects malformed pressure telemetry rather than treating it as idle", async () => { + vi.spyOn(fs, "readFileSync").mockImplementation(file => + file === "/proc/stat" ? "cpu0 0 0 0 100 0 0 0 0\n" : "some avg10=0 total=invalid\n" + ); + await expect(waitForBenchmarkIdle()).rejects.toThrow("Missing cpu pressure counter"); +}); + +test.each(["cpu0 invalid\n", "intr 1\n"])("rejects malformed or missing CPU counters: %s", async counters => { + vi.spyOn(fs, "readFileSync").mockReturnValue(counters); + await expect(waitForBenchmarkIdle()).rejects.toThrow(/CPU counters/); +}); diff --git a/scripts/src/waitForBenchmarkIdle.ts b/scripts/src/waitForBenchmarkIdle.ts new file mode 100644 index 00000000..37ae725a --- /dev/null +++ b/scripts/src/waitForBenchmarkIdle.ts @@ -0,0 +1,124 @@ +import assert from "node:assert"; +import fs from "node:fs"; + +import esMain from "es-main"; + +import { sleepSeconds } from "./utils.js"; + +const resources = ["cpu", "io", "memory"] as const; +type Resource = typeof resources[number]; + +interface CpuTimes { + name: string; + total: number; + idle: number; +} + +function readCpuTimes(): CpuTimes[] { + const cpus = fs.readFileSync("/proc/stat", "utf8").split("\n").filter(line => /^cpu\d+\s/.test(line)); + assert(cpus.length > 0, "Missing per-CPU counters in /proc/stat"); + return cpus.map(line => { + const [name, ...fields] = line.trim().split(/\s+/); + // Guest time is already included in user/nice; do not count it twice. + const counters = fields.slice(0, 8).map(Number); + assert( + counters.length === 8 && counters.every(value => Number.isSafeInteger(value) && value >= 0), + "Invalid CPU counters in /proc/stat", + ); + return { name, total: counters.reduce((a, b) => a + b, 0), idle: counters[3] }; + }); +} + +export async function waitForBenchmarkIdle() { + const unavailable = new Set(); + function readPressure(): Partial> { + const pressure: Partial> = {}; + for (const resource of resources) { + if (unavailable.has(resource)) continue; + let contents: string; + try { + contents = fs.readFileSync(`/proc/pressure/${resource}`, "utf8"); + } + catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + unavailable.add(resource); + console.warn( + `##vso[task.logissue type=warning]${resource} pressure counters unavailable; checking remaining idle signals.`, + ); + continue; + } + throw error; + } + const total = contents.match(/^some\s+.*\btotal=(\d+)\s*$/m)?.[1]; + assert(total !== undefined, `Missing ${resource} pressure counter`); + pressure[resource] = Number(total); + assert(Number.isSafeInteger(pressure[resource]), `Invalid ${resource} pressure counter`); + } + return pressure; + } + + function sample() { + return { cpus: readCpuTimes(), pressure: readPressure(), time: performance.now() }; + } + + console.log("Waiting up to 60s for 5 consecutive quiet seconds: CPU average <=5%, peak <=20%, pressure <=1%."); + let previous = sample(); + const start = previous.time; + let quietSeconds = 0; + let lastSummary = ""; + while (previous.time - start < 60_000) { + await sleepSeconds(1); + const current = sample(); + const elapsedMs = current.time - previous.time; + assert(elapsedMs > 0, "Expected elapsed time between idle samples"); + assert(current.cpus.length === previous.cpus.length, "Online CPUs changed during idle check"); + + let totalTicks = 0; + let busyTicks = 0; + let peakBusy = 0; + for (let index = 0; index < current.cpus.length; index++) { + const cpu = current.cpus[index]; + const before = previous.cpus[index]; + assert(cpu.name === before.name, "Online CPUs changed during idle check"); + const total = cpu.total - before.total; + const idle = cpu.idle - before.idle; + assert(total > 0 && idle >= 0 && idle <= total, "Invalid CPU counter delta during idle check"); + totalTicks += total; + busyTicks += total - idle; + peakBusy = Math.max(peakBusy, 100 * (total - idle) / total); + } + const averageBusy = 100 * busyTicks / totalTicks; + let quiet = averageBusy <= 5 && peakBusy <= 20; + const pressures: string[] = []; + for (const resource of resources) { + const total = current.pressure[resource]; + const before = previous.pressure[resource]; + if (total === undefined || before === undefined) continue; + const delta = total - before; + assert(delta >= 0, `${resource} pressure counter decreased during idle check`); + const percent = 100 * delta / (elapsedMs * 1000); + pressures.push(`${resource}=${percent.toFixed(2)}%`); + quiet &&= percent <= 1; + } + lastSummary = `CPU average=${averageBusy.toFixed(2)}%, peak=${peakBusy.toFixed(2)}%; pressure ${ + pressures.length ? pressures.join(", ") : "unavailable" + }`; + quietSeconds = quiet ? quietSeconds + elapsedMs / 1000 : 0; + console.log( + `${((current.time - start) / 1000).toFixed(1)}s: ${lastSummary}; quiet=${quietSeconds.toFixed(1)}s`, + ); + if (quietSeconds >= 5) { + console.log("Machine is quiet; starting benchmark without discarding any measured samples."); + return; + } + previous = current; + } + throw new Error(`Machine did not become quiet within 60s; benchmark not started. ${lastSummary}`); +} + +if (esMain(import.meta)) { + await waitForBenchmarkIdle().catch((error: unknown) => { + console.error(error); + process.exitCode = 1; + }); +} From 32db5662bfca86a73f12f9a33a3ff6931f3ec206 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:00:40 -0700 Subject: [PATCH 4/4] Use the Azure Linux benchmark pool by default Route ordinary benchmark requests to the purpose-built pool instead of shared Ubuntu runners, now that the Azure Linux image is available. --- build/benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/benchmark.yml b/build/benchmark.yml index 3b8ea432..9e33aac0 100644 --- a/build/benchmark.yml +++ b/build/benchmark.yml @@ -74,12 +74,12 @@ parameters: - name: HOSTED_POOL displayName: Optional 1ES benchmark pool type: string - default: '' + default: TypeScript-Benchmarking - name: HOSTED_IMAGE displayName: Image in the 1ES benchmark pool type: string - default: azure-linux-3 + default: azure-linux-3-benchmarks # PR trigger params - name: DISTINCT_ID