diff --git a/build/benchmark.yml b/build/benchmark.yml index 9aad6259..9e33aac0 100644 --- a/build/benchmark.yml +++ b/build/benchmark.yml @@ -71,6 +71,16 @@ parameters: type: boolean default: true + - name: HOSTED_POOL + displayName: Optional 1ES benchmark pool + type: string + default: TypeScript-Benchmarking + + - name: HOSTED_IMAGE + displayName: Image in the 1ES benchmark pool + type: string + default: azure-linux-3-benchmarks + # PR trigger params - name: DISTINCT_ID displayName: Distinct ID for this run (PR runs only) @@ -363,7 +373,12 @@ jobs: condition: and(succeeded(), not(eq(dependencies.Setup.outputs['setupPipeline.MATRIX_${{ replace(agent, '-', '_') }}'], '{}'))) pool: ${{ if parameters.USE_HOSTED_RUNNERS }}: - vmImage: ubuntu-latest + ${{ if eq(parameters.HOSTED_POOL, '') }}: + vmImage: ubuntu-latest + ${{ else }}: + name: ${{ parameters.HOSTED_POOL }} + demands: + - ImageOverride -equals ${{ parameters.HOSTED_IMAGE }} ${{ else }}: name: ts-perf-ddfun ${{ if not(eq(agent, 'any')) }}: @@ -534,6 +549,14 @@ jobs: - BWRAP_ARGS displayName: Create benchmark sandbox + - ${{ if parameters.USE_HOSTED_RUNNERS }}: + - bash: node $(BENCH_SCRIPTS)/waitForBenchmarkIdle.js + target: + commands: restricted + settableVariables: none + displayName: Wait for idle before new benchmark + condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true')) + - bash: | set -exo pipefail bash $(Pipeline.Workspace)/typescript-benchmarking/scripts/runBenchmarkSandbox.sh \ @@ -571,6 +594,13 @@ jobs: displayName: Publish scenario ref for baseline benchmark condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true')) + - ${{ if parameters.USE_HOSTED_RUNNERS }}: + - bash: node $(BENCH_SCRIPTS)/waitForBenchmarkIdle.js + target: + commands: restricted + settableVariables: none + displayName: Wait for idle before baseline benchmark + - bash: | set -exo pipefail bash $(Pipeline.Workspace)/typescript-benchmarking/scripts/runBenchmarkSandbox.sh \ diff --git a/build/templates/installSandbox.yml b/build/templates/installSandbox.yml index 92b29d46..7d3ba0db 100644 --- a/build/templates/installSandbox.yml +++ b/build/templates/installSandbox.yml @@ -6,8 +6,17 @@ parameters: steps: - bash: | set -exo pipefail - sudo apt-get update - sudo apt-get install -y bubblewrap + if ! command -v bwrap >/dev/null; then + if command -v apt-get >/dev/null; then + sudo apt-get update + sudo apt-get install -y bubblewrap + elif command -v tdnf >/dev/null; then + sudo tdnf install -y bubblewrap + else + echo "Unsupported package manager for bubblewrap installation" >&2 + exit 1 + fi + fi # Ubuntu 24.04 blocks unprivileged user namespaces unless an AppArmor profile # allows them. Allow them for bubblewrap only, like Ubuntu's browser profiles. @@ -33,13 +42,34 @@ steps: # Only dependency installation uses gVisor; measured compilers use bubblewrap. - bash: | set -exo pipefail - sudo apt-get install -y apt-transport-https ca-certificates curl gnupg - curl -fsSL https://gvisor.dev/archive.key | sudo gpg --dearmor -o /usr/share/keyrings/gvisor-archive-keyring.gpg - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/gvisor-archive-keyring.gpg] https://storage.googleapis.com/gvisor/releases release main" | sudo tee /etc/apt/sources.list.d/gvisor.list > /dev/null - sudo apt-get update - sudo apt-get install -y runsc - sudo runsc install + if command -v apt-get >/dev/null; then + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg acl + curl -fsSL https://gvisor.dev/archive.key | sudo gpg --dearmor -o /usr/share/keyrings/gvisor-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/gvisor-archive-keyring.gpg] https://storage.googleapis.com/gvisor/releases release main" | sudo tee /etc/apt/sources.list.d/gvisor.list > /dev/null + sudo apt-get update + sudo apt-get install -y runsc + RUNSC=/usr/bin/runsc + elif command -v tdnf >/dev/null; then + sudo tdnf install -y ca-certificates curl moby-engine moby-cli moby-buildx util-linux acl tar bzip2 + INSTALL_DIR=$(mktemp -d "${AGENT_TEMPDIRECTORY:?}/tsperf-runsc.XXXXXX") + trap 'rm -f "$INSTALL_DIR/gvisor.tar.bz2" "$INSTALL_DIR/gvisor.tar.bz2.sha512"; rmdir "$INSTALL_DIR"' EXIT + URL="https://storage.googleapis.com/gvisor/releases/release/latest/$(uname -m)" + curl -fsSL "$URL/gvisor.tar.bz2" -o "$INSTALL_DIR/gvisor.tar.bz2" + curl -fsSL "$URL/gvisor.tar.bz2.sha512" -o "$INSTALL_DIR/gvisor.tar.bz2.sha512" + (cd "$INSTALL_DIR" && sha512sum --check gvisor.tar.bz2.sha512) + sudo tar -xjf "$INSTALL_DIR/gvisor.tar.bz2" -C /usr/local/bin --no-same-owner --no-same-permissions + RUNSC=/usr/local/bin/runsc + else + echo "Unsupported package manager for scenario sandbox installation" >&2 + exit 1 + fi + docker buildx version + sudo "$RUNSC" install sudo systemctl restart docker + # New Docker installations do not grant the running agent access to its socket. + if [ ! -w /var/run/docker.sock ]; then + sudo setfacl -m "u:$(id -u):rw" /var/run/docker.sock + fi docker info target: commands: restricted diff --git a/scripts/src/__tests__/scenarioSetup.test.ts b/scripts/src/__tests__/scenarioSetup.test.ts index 00930fc4..faa51e25 100644 --- a/scripts/src/__tests__/scenarioSetup.test.ts +++ b/scripts/src/__tests__/scenarioSetup.test.ts @@ -20,23 +20,24 @@ const sandboxTemplate = parse( }; const tempDirs: string[] = []; -function hostedCpuScript() { - const hostedSteps = sandboxTemplate.steps.find(step => "${{ if parameters.HOSTED_BENCHMARK }}" in step); - if (!hostedSteps || !("${{ if parameters.HOSTED_BENCHMARK }}" in hostedSteps)) { - throw new Error("Missing hosted sandbox setup"); - } - const cpuStep = hostedSteps["${{ if parameters.HOSTED_BENCHMARK }}"] - .find(step => step.displayName === "Configure hosted benchmark CPUs"); - if (!cpuStep) { - throw new Error("Missing hosted CPU configuration"); +function sandboxScript(displayName: string) { + const steps = sandboxTemplate.steps.flatMap(step => + "bash" in step ? [step] : step["${{ if parameters.HOSTED_BENCHMARK }}"] + ); + const step = steps.find(step => step.displayName === displayName); + if (!step) { + throw new Error(`Missing sandbox step: ${displayName}`); } - return cpuStep.bash; + return step.bash; } test("hosted CPU configuration selects CPUs the benchmark process can actually use", () => { const allowed = fs.readFileSync("/proc/self/status", "utf8").match(/^Cpus_allowed_list:\s*(.+)$/m)?.[1]; expect(allowed).toBeTruthy(); - const result = spawnSync("bash", ["-c", hostedCpuScript()], { encoding: "utf8", timeout: 10_000 }); + const result = spawnSync("bash", ["-c", sandboxScript("Configure hosted benchmark CPUs")], { + encoding: "utf8", + timeout: 10_000, + }); expect(result.status).toBe(0); expect(result.stdout).toContain( `##vso[task.setvariable variable=TSPERF_AGENT_BENCHMARK_CPUS;isReadOnly=true]${allowed}`, @@ -48,7 +49,7 @@ test("hosted CPU configuration does not publish unusable affinity", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "typescript-benchmarking-hosted-")); tempDirs.push(tempDir); fs.writeFileSync(path.join(tempDir, "taskset"), "#!/bin/sh\nexit 37\n", { mode: 0o755 }); - const result = spawnSync("bash", ["-c", hostedCpuScript()], { + const result = spawnSync("bash", ["-c", sandboxScript("Configure hosted benchmark CPUs")], { env: { ...process.env, PATH: `${tempDir}:${process.env.PATH}` }, encoding: "utf8", timeout: 10_000, @@ -57,6 +58,94 @@ test("hosted CPU configuration does not publish unusable affinity", () => { expect(result.stdout).not.toContain("task.setvariable"); }); +function runAzureLinuxSetup(env: Record = {}) { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "typescript-benchmarking-azurelinux-")); + tempDirs.push(tempDir); + const commands = path.join(tempDir, "commands"); + const script = ` +command() { + if [ "$1" = "-v" ]; then + case "$2" in + apt-get) return 1 ;; + tdnf) return 0 ;; + esac + fi + builtin command "$@" +} +sudo() { + printf '%s\\n' "$*" >> "$COMMANDS" + if [ "$1" = "tdnf" ] && [ "\${FAIL_PACKAGES:-}" = "true" ]; then + return 37 + fi +} +curl() { + if [[ "$2" == */gvisor.tar.bz2.sha512 ]]; then + (cd "\${4%/*}" && sha512sum gvisor.tar.bz2) > "$4" + if [ "\${CORRUPT_DOWNLOAD:-}" = "true" ]; then + printf 'modified' >> "\${4%/*}/gvisor.tar.bz2" + fi + else + printf '#!/bin/sh\\nexit 0\\n' > "$4" + fi +} +docker() { + printf '%s\\n' "docker $*" >> "$COMMANDS" + if [ "$1" = "buildx" ] && [ "\${FAIL_BUILDX:-}" = "true" ]; then + return 37 + fi +} +${sandboxScript("Install scenario sandbox")} +`; + const result = spawnSync("bash", ["-c", script], { + env: { + ...process.env, + AGENT_TEMPDIRECTORY: tempDir, + COMMANDS: commands, + ...env, + }, + encoding: "utf8", + timeout: 10_000, + }); + return { result, commands: fs.readFileSync(commands, "utf8"), files: fs.readdirSync(tempDir) }; +} + +test("Azure Linux installs and verifies gVisor before configuring Docker", () => { + const { result, commands, files } = runAzureLinuxSetup(); + expect(result.status).toBe(0); + expect(commands).toContain( + "tdnf install -y ca-certificates curl moby-engine moby-cli moby-buildx util-linux acl tar bzip2", + ); + expect(commands).toContain("tar -xjf"); + expect(commands).toContain("docker buildx version\n/usr/local/bin/runsc install\nsystemctl restart docker\n"); + expect(commands).toContain("docker info"); + expect(files).toEqual(["commands"]); +}); + +test("Azure Linux rejects a corrupted gVisor download before installing or starting it", () => { + const { result, commands, files } = runAzureLinuxSetup({ CORRUPT_DOWNLOAD: "true" }); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("checksum did NOT match"); + expect(commands).not.toContain("tar -xjf"); + expect(commands).not.toContain("systemctl"); + expect(files).toEqual(["commands"]); +}); + +test("Azure Linux stops when sandbox prerequisites cannot be installed", () => { + const { result, commands, files } = runAzureLinuxSetup({ FAIL_PACKAGES: "true" }); + expect(result.status).toBe(37); + expect(commands).not.toContain("runsc install"); + expect(commands).not.toContain("systemctl"); + expect(files).toEqual(["commands"]); +}); + +test("Azure Linux stops before configuring Docker if Buildx is unavailable", () => { + const { result, commands, files } = runAzureLinuxSetup({ FAIL_BUILDX: "true" }); + expect(result.status).toBe(37); + expect(commands).not.toContain("runsc install"); + expect(commands).not.toContain("systemctl"); + expect(files).toEqual(["commands"]); +}); + afterEach(() => { for (const tempDir of tempDirs.splice(0)) { fs.rmSync(tempDir, { recursive: true }); diff --git a/scripts/src/__tests__/waitForBenchmarkIdle.test.ts b/scripts/src/__tests__/waitForBenchmarkIdle.test.ts new file mode 100644 index 00000000..0c52b53a --- /dev/null +++ b/scripts/src/__tests__/waitForBenchmarkIdle.test.ts @@ -0,0 +1,109 @@ +import fs from "node:fs"; + +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +import { waitForBenchmarkIdle } from "../waitForBenchmarkIdle.js"; + +beforeEach(() => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] }); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); +}); + +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +function mockCounters(options: { + busy?: (second: number, cpu: number) => number; + pressure?: (second: number, resource: string) => number; + pressureError?: string; +} = {}) { + let second = -1; + const ticks = Array(8).fill(0); + const busy = Array(8).fill(0); + const pressure = new Map(); + return vi.spyOn(fs, "readFileSync").mockImplementation(file => { + if (file === "/proc/stat") { + second++; + return ticks.map((_, cpu) => { + ticks[cpu] += 100; + busy[cpu] += options.busy?.(second, cpu) ?? 3; + return `cpu${cpu} ${busy[cpu]} 0 0 ${ticks[cpu] - busy[cpu]} 0 0 0 0 ${busy[cpu]} 0`; + }).join("\n"); + } + const resource = String(file).split("/").at(-1)!; + if (options.pressureError) { + throw Object.assign(new Error("cannot read pressure"), { code: options.pressureError }); + } + const total = (pressure.get(resource) ?? 0) + (options.pressure?.(second, resource) ?? 0); + pressure.set(resource, total); + return `some avg10=99.99 avg60=99.99 avg300=99.99 total=${total}\nfull avg10=0 total=0\n`; + }); +} + +test("requires five quiet seconds and uses deltas rather than stale averages or double-counted guest time", async () => { + const read = mockCounters(); + const waiting = waitForBenchmarkIdle(); + await vi.advanceTimersByTimeAsync(4000); + expect(console.log).not.toHaveBeenCalledWith(expect.stringContaining("Machine is quiet")); + await vi.advanceTimersByTimeAsync(1000); + await waiting; + expect(console.log).toHaveBeenCalledWith(expect.stringContaining("Machine is quiet")); + expect(read).toHaveBeenCalledTimes(24); +}); + +test("restarts the quiet window when one CPU becomes busy", async () => { + mockCounters({ busy: (second, cpu) => second === 3 && cpu === 0 ? 30 : 0 }); + const waiting = waitForBenchmarkIdle(); + await vi.advanceTimersByTimeAsync(7000); + expect(console.log).not.toHaveBeenCalledWith(expect.stringContaining("Machine is quiet")); + await vi.advanceTimersByTimeAsync(1000); + await waiting; +}); + +test.each(["cpu", "io", "memory"])("waits for recent %s pressure to subside", async resource => { + mockCounters({ pressure: (second, name) => second <= 3 && name === resource ? 20_000 : 0 }); + const waiting = waitForBenchmarkIdle(); + await vi.advanceTimersByTimeAsync(8000); + await waiting; + expect(console.log).toHaveBeenCalledWith(expect.stringContaining(`${resource}=2.00%`)); + expect(console.log).toHaveBeenCalledWith(expect.stringContaining("Machine is quiet")); +}); + +test("fails within the time bound instead of starting measurements on a persistently busy machine", async () => { + mockCounters({ busy: () => 50 }); + const waiting = expect(waitForBenchmarkIdle()).rejects.toThrow("Machine did not become quiet within 60s"); + await vi.advanceTimersByTimeAsync(60_000); + await waiting; + expect(console.log).not.toHaveBeenCalledWith(expect.stringContaining("Machine is quiet")); + expect(vi.getTimerCount()).toBe(0); +}); + +test("explicitly warns once per unavailable pressure signal and still checks CPU counters", async () => { + mockCounters({ pressureError: "ENOENT" }); + const waiting = waitForBenchmarkIdle(); + await vi.advanceTimersByTimeAsync(5000); + await waiting; + expect(console.warn).toHaveBeenCalledTimes(3); + expect(console.warn).toHaveBeenCalledWith(expect.stringContaining("cpu pressure counters unavailable")); + expect(console.log).toHaveBeenCalledWith(expect.stringContaining("Machine is quiet")); +}); + +test("does not suppress permission errors reading telemetry", async () => { + mockCounters({ pressureError: "EACCES" }); + await expect(waitForBenchmarkIdle()).rejects.toMatchObject({ code: "EACCES" }); +}); + +test("rejects malformed pressure telemetry rather than treating it as idle", async () => { + vi.spyOn(fs, "readFileSync").mockImplementation(file => + file === "/proc/stat" ? "cpu0 0 0 0 100 0 0 0 0\n" : "some avg10=0 total=invalid\n" + ); + await expect(waitForBenchmarkIdle()).rejects.toThrow("Missing cpu pressure counter"); +}); + +test.each(["cpu0 invalid\n", "intr 1\n"])("rejects malformed or missing CPU counters: %s", async counters => { + vi.spyOn(fs, "readFileSync").mockReturnValue(counters); + await expect(waitForBenchmarkIdle()).rejects.toThrow(/CPU counters/); +}); diff --git a/scripts/src/waitForBenchmarkIdle.ts b/scripts/src/waitForBenchmarkIdle.ts new file mode 100644 index 00000000..37ae725a --- /dev/null +++ b/scripts/src/waitForBenchmarkIdle.ts @@ -0,0 +1,124 @@ +import assert from "node:assert"; +import fs from "node:fs"; + +import esMain from "es-main"; + +import { sleepSeconds } from "./utils.js"; + +const resources = ["cpu", "io", "memory"] as const; +type Resource = typeof resources[number]; + +interface CpuTimes { + name: string; + total: number; + idle: number; +} + +function readCpuTimes(): CpuTimes[] { + const cpus = fs.readFileSync("/proc/stat", "utf8").split("\n").filter(line => /^cpu\d+\s/.test(line)); + assert(cpus.length > 0, "Missing per-CPU counters in /proc/stat"); + return cpus.map(line => { + const [name, ...fields] = line.trim().split(/\s+/); + // Guest time is already included in user/nice; do not count it twice. + const counters = fields.slice(0, 8).map(Number); + assert( + counters.length === 8 && counters.every(value => Number.isSafeInteger(value) && value >= 0), + "Invalid CPU counters in /proc/stat", + ); + return { name, total: counters.reduce((a, b) => a + b, 0), idle: counters[3] }; + }); +} + +export async function waitForBenchmarkIdle() { + const unavailable = new Set(); + function readPressure(): Partial> { + const pressure: Partial> = {}; + for (const resource of resources) { + if (unavailable.has(resource)) continue; + let contents: string; + try { + contents = fs.readFileSync(`/proc/pressure/${resource}`, "utf8"); + } + catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + unavailable.add(resource); + console.warn( + `##vso[task.logissue type=warning]${resource} pressure counters unavailable; checking remaining idle signals.`, + ); + continue; + } + throw error; + } + const total = contents.match(/^some\s+.*\btotal=(\d+)\s*$/m)?.[1]; + assert(total !== undefined, `Missing ${resource} pressure counter`); + pressure[resource] = Number(total); + assert(Number.isSafeInteger(pressure[resource]), `Invalid ${resource} pressure counter`); + } + return pressure; + } + + function sample() { + return { cpus: readCpuTimes(), pressure: readPressure(), time: performance.now() }; + } + + console.log("Waiting up to 60s for 5 consecutive quiet seconds: CPU average <=5%, peak <=20%, pressure <=1%."); + let previous = sample(); + const start = previous.time; + let quietSeconds = 0; + let lastSummary = ""; + while (previous.time - start < 60_000) { + await sleepSeconds(1); + const current = sample(); + const elapsedMs = current.time - previous.time; + assert(elapsedMs > 0, "Expected elapsed time between idle samples"); + assert(current.cpus.length === previous.cpus.length, "Online CPUs changed during idle check"); + + let totalTicks = 0; + let busyTicks = 0; + let peakBusy = 0; + for (let index = 0; index < current.cpus.length; index++) { + const cpu = current.cpus[index]; + const before = previous.cpus[index]; + assert(cpu.name === before.name, "Online CPUs changed during idle check"); + const total = cpu.total - before.total; + const idle = cpu.idle - before.idle; + assert(total > 0 && idle >= 0 && idle <= total, "Invalid CPU counter delta during idle check"); + totalTicks += total; + busyTicks += total - idle; + peakBusy = Math.max(peakBusy, 100 * (total - idle) / total); + } + const averageBusy = 100 * busyTicks / totalTicks; + let quiet = averageBusy <= 5 && peakBusy <= 20; + const pressures: string[] = []; + for (const resource of resources) { + const total = current.pressure[resource]; + const before = previous.pressure[resource]; + if (total === undefined || before === undefined) continue; + const delta = total - before; + assert(delta >= 0, `${resource} pressure counter decreased during idle check`); + const percent = 100 * delta / (elapsedMs * 1000); + pressures.push(`${resource}=${percent.toFixed(2)}%`); + quiet &&= percent <= 1; + } + lastSummary = `CPU average=${averageBusy.toFixed(2)}%, peak=${peakBusy.toFixed(2)}%; pressure ${ + pressures.length ? pressures.join(", ") : "unavailable" + }`; + quietSeconds = quiet ? quietSeconds + elapsedMs / 1000 : 0; + console.log( + `${((current.time - start) / 1000).toFixed(1)}s: ${lastSummary}; quiet=${quietSeconds.toFixed(1)}s`, + ); + if (quietSeconds >= 5) { + console.log("Machine is quiet; starting benchmark without discarding any measured samples."); + return; + } + previous = current; + } + throw new Error(`Machine did not become quiet within 60s; benchmark not started. ${lastSummary}`); +} + +if (esMain(import.meta)) { + await waitForBenchmarkIdle().catch((error: unknown) => { + console.error(error); + process.exitCode = 1; + }); +}