Run hosted benchmarks on the Azure Linux pool

build/benchmark.yml

@@ -71,6 +71,16 @@ parameters:
     type: boolean
     default: true
 
+  - name: HOSTED_POOL
+    displayName: Optional 1ES benchmark pool
+    type: string
+    default: TypeScript-Benchmarking
+
+  - name: HOSTED_IMAGE
+    displayName: Image in the 1ES benchmark pool
+    type: string
+    default: azure-linux-3-benchmarks
+
   # PR trigger params
   - name: DISTINCT_ID
     displayName: Distinct ID for this run (PR runs only)
@@ -363,7 +373,12 @@ jobs:
         condition: and(succeeded(), not(eq(dependencies.Setup.outputs['setupPipeline.MATRIX_${{ replace(agent, '-', '_') }}'], '{}')))
         pool:
           ${{ if parameters.USE_HOSTED_RUNNERS }}:
-            vmImage: ubuntu-latest
+            ${{ if eq(parameters.HOSTED_POOL, '') }}:
+              vmImage: ubuntu-latest
+            ${{ else }}:
+              name: ${{ parameters.HOSTED_POOL }}
+              demands:
+                - ImageOverride -equals ${{ parameters.HOSTED_IMAGE }}
           ${{ else }}:
             name: ts-perf-ddfun
             ${{ if not(eq(agent, 'any')) }}:
@@ -534,6 +549,14 @@ jobs:
                 - BWRAP_ARGS
             displayName: Create benchmark sandbox
 
+          - ${{ if parameters.USE_HOSTED_RUNNERS }}:
+              - bash: node $(BENCH_SCRIPTS)/waitForBenchmarkIdle.js
+                target:
+                  commands: restricted
+                  settableVariables: none
+                displayName: Wait for idle before new benchmark
+                condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true'))
+
           - bash: |
               set -exo pipefail
               bash $(Pipeline.Workspace)/typescript-benchmarking/scripts/runBenchmarkSandbox.sh \
@@ -571,6 +594,13 @@ jobs:
             displayName: Publish scenario ref for baseline benchmark
             condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true'))
 
+          - ${{ if parameters.USE_HOSTED_RUNNERS }}:
+              - bash: node $(BENCH_SCRIPTS)/waitForBenchmarkIdle.js
+                target:
+                  commands: restricted
+                  settableVariables: none
+                displayName: Wait for idle before baseline benchmark
+
           - bash: |
               set -exo pipefail
               bash $(Pipeline.Workspace)/typescript-benchmarking/scripts/runBenchmarkSandbox.sh \

build/templates/installSandbox.yml

@@ -6,8 +6,17 @@ parameters:
 steps:
   - bash: |
       set -exo pipefail
-      sudo apt-get update
-      sudo apt-get install -y bubblewrap
+      if ! command -v bwrap >/dev/null; then
+        if command -v apt-get >/dev/null; then
+          sudo apt-get update
+          sudo apt-get install -y bubblewrap
+        elif command -v tdnf >/dev/null; then
+          sudo tdnf install -y bubblewrap
+        else
+          echo "Unsupported package manager for bubblewrap installation" >&2
+          exit 1
+        fi
+      fi
 
       # Ubuntu 24.04 blocks unprivileged user namespaces unless an AppArmor profile
       # allows them. Allow them for bubblewrap only, like Ubuntu's browser profiles.
@@ -33,13 +42,34 @@ steps:
       # Only dependency installation uses gVisor; measured compilers use bubblewrap.
       - bash: |
           set -exo pipefail
-          sudo apt-get install -y apt-transport-https ca-certificates curl gnupg
-          curl -fsSL https://gvisor.dev/archive.key | sudo gpg --dearmor -o /usr/share/keyrings/gvisor-archive-keyring.gpg
-          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/gvisor-archive-keyring.gpg] https://storage.googleapis.com/gvisor/releases release main" | sudo tee /etc/apt/sources.list.d/gvisor.list > /dev/null
-          sudo apt-get update
-          sudo apt-get install -y runsc
-          sudo runsc install
+          if command -v apt-get >/dev/null; then
+            sudo apt-get install -y apt-transport-https ca-certificates curl gnupg acl
+            curl -fsSL https://gvisor.dev/archive.key | sudo gpg --dearmor -o /usr/share/keyrings/gvisor-archive-keyring.gpg
+            echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/gvisor-archive-keyring.gpg] https://storage.googleapis.com/gvisor/releases release main" | sudo tee /etc/apt/sources.list.d/gvisor.list > /dev/null
+            sudo apt-get update
+            sudo apt-get install -y runsc
+            RUNSC=/usr/bin/runsc
+          elif command -v tdnf >/dev/null; then
+            sudo tdnf install -y ca-certificates curl moby-engine moby-cli moby-buildx util-linux acl tar bzip2
+            INSTALL_DIR=$(mktemp -d "${AGENT_TEMPDIRECTORY:?}/tsperf-runsc.XXXXXX")
+            trap 'rm -f "$INSTALL_DIR/gvisor.tar.bz2" "$INSTALL_DIR/gvisor.tar.bz2.sha512"; rmdir "$INSTALL_DIR"' EXIT
+            URL="https://storage.googleapis.com/gvisor/releases/release/latest/$(uname -m)"
+            curl -fsSL "$URL/gvisor.tar.bz2" -o "$INSTALL_DIR/gvisor.tar.bz2"
+            curl -fsSL "$URL/gvisor.tar.bz2.sha512" -o "$INSTALL_DIR/gvisor.tar.bz2.sha512"
+            (cd "$INSTALL_DIR" && sha512sum --check gvisor.tar.bz2.sha512)
+            sudo tar -xjf "$INSTALL_DIR/gvisor.tar.bz2" -C /usr/local/bin --no-same-owner --no-same-permissions
+            RUNSC=/usr/local/bin/runsc
+          else
+            echo "Unsupported package manager for scenario sandbox installation" >&2
+            exit 1
+          fi
+          docker buildx version
+          sudo "$RUNSC" install
           sudo systemctl restart docker
+          # New Docker installations do not grant the running agent access to its socket.
+          if [ ! -w /var/run/docker.sock ]; then
+            sudo setfacl -m "u:$(id -u):rw" /var/run/docker.sock
+          fi
           docker info
         target:
           commands: restricted

scripts/src/__tests__/scenarioSetup.test.ts

@@ -20,23 +20,24 @@ const sandboxTemplate = parse(
 };
 const tempDirs: string[] = [];
 
-function hostedCpuScript() {
-    const hostedSteps = sandboxTemplate.steps.find(step => "${{ if parameters.HOSTED_BENCHMARK }}" in step);
-    if (!hostedSteps || !("${{ if parameters.HOSTED_BENCHMARK }}" in hostedSteps)) {
-        throw new Error("Missing hosted sandbox setup");
-    }
-    const cpuStep = hostedSteps["${{ if parameters.HOSTED_BENCHMARK }}"]
-        .find(step => step.displayName === "Configure hosted benchmark CPUs");
-    if (!cpuStep) {
-        throw new Error("Missing hosted CPU configuration");
+function sandboxScript(displayName: string) {
+    const steps = sandboxTemplate.steps.flatMap(step =>
+        "bash" in step ? [step] : step["${{ if parameters.HOSTED_BENCHMARK }}"]
+    );
+    const step = steps.find(step => step.displayName === displayName);
+    if (!step) {
+        throw new Error(`Missing sandbox step: ${displayName}`);
     }
-    return cpuStep.bash;
+    return step.bash;
 }
 
 test("hosted CPU configuration selects CPUs the benchmark process can actually use", () => {
     const allowed = fs.readFileSync("/proc/self/status", "utf8").match(/^Cpus_allowed_list:\s*(.+)$/m)?.[1];
     expect(allowed).toBeTruthy();
-    const result = spawnSync("bash", ["-c", hostedCpuScript()], { encoding: "utf8", timeout: 10_000 });
+    const result = spawnSync("bash", ["-c", sandboxScript("Configure hosted benchmark CPUs")], {
+        encoding: "utf8",
+        timeout: 10_000,
+    });
     expect(result.status).toBe(0);
     expect(result.stdout).toContain(
         `##vso[task.setvariable variable=TSPERF_AGENT_BENCHMARK_CPUS;isReadOnly=true]${allowed}`,
@@ -48,7 +49,7 @@ test("hosted CPU configuration does not publish unusable affinity", () => {
     const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "typescript-benchmarking-hosted-"));
     tempDirs.push(tempDir);
     fs.writeFileSync(path.join(tempDir, "taskset"), "#!/bin/sh\nexit 37\n", { mode: 0o755 });
-    const result = spawnSync("bash", ["-c", hostedCpuScript()], {
+    const result = spawnSync("bash", ["-c", sandboxScript("Configure hosted benchmark CPUs")], {
         env: { ...process.env, PATH: `${tempDir}:${process.env.PATH}` },
         encoding: "utf8",
         timeout: 10_000,
@@ -57,6 +58,94 @@ test("hosted CPU configuration does not publish unusable affinity", () => {
     expect(result.stdout).not.toContain("task.setvariable");
 });
 
+function runAzureLinuxSetup(env: Record<string, string> = {}) {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "typescript-benchmarking-azurelinux-"));
+    tempDirs.push(tempDir);
+    const commands = path.join(tempDir, "commands");
+    const script = `
+command() {
+    if [ "$1" = "-v" ]; then
+        case "$2" in
+            apt-get) return 1 ;;
+            tdnf) return 0 ;;
+        esac
+    fi
+    builtin command "$@"
+}
+sudo() {
+    printf '%s\\n' "$*" >> "$COMMANDS"
+    if [ "$1" = "tdnf" ] && [ "\${FAIL_PACKAGES:-}" = "true" ]; then
+        return 37
+    fi
+}
+curl() {
+    if [[ "$2" == */gvisor.tar.bz2.sha512 ]]; then
+        (cd "\${4%/*}" && sha512sum gvisor.tar.bz2) > "$4"
+        if [ "\${CORRUPT_DOWNLOAD:-}" = "true" ]; then
+            printf 'modified' >> "\${4%/*}/gvisor.tar.bz2"
+        fi
+    else
+        printf '#!/bin/sh\\nexit 0\\n' > "$4"
+    fi
+}
+docker() {
+    printf '%s\\n' "docker $*" >> "$COMMANDS"
+    if [ "$1" = "buildx" ] && [ "\${FAIL_BUILDX:-}" = "true" ]; then
+        return 37
+    fi
+}
+${sandboxScript("Install scenario sandbox")}
+`;
+    const result = spawnSync("bash", ["-c", script], {
+        env: {
+            ...process.env,
+            AGENT_TEMPDIRECTORY: tempDir,
+            COMMANDS: commands,
+            ...env,
+        },
+        encoding: "utf8",
+        timeout: 10_000,
+    });
+    return { result, commands: fs.readFileSync(commands, "utf8"), files: fs.readdirSync(tempDir) };
+}
+
+test("Azure Linux installs and verifies gVisor before configuring Docker", () => {
+    const { result, commands, files } = runAzureLinuxSetup();
+    expect(result.status).toBe(0);
+    expect(commands).toContain(
+        "tdnf install -y ca-certificates curl moby-engine moby-cli moby-buildx util-linux acl tar bzip2",
+    );
+    expect(commands).toContain("tar -xjf");
+    expect(commands).toContain("docker buildx version\n/usr/local/bin/runsc install\nsystemctl restart docker\n");
+    expect(commands).toContain("docker info");
+    expect(files).toEqual(["commands"]);
+});
+
+test("Azure Linux rejects a corrupted gVisor download before installing or starting it", () => {
+    const { result, commands, files } = runAzureLinuxSetup({ CORRUPT_DOWNLOAD: "true" });
+    expect(result.status).not.toBe(0);
+    expect(result.stderr).toContain("checksum did NOT match");
+    expect(commands).not.toContain("tar -xjf");
+    expect(commands).not.toContain("systemctl");
+    expect(files).toEqual(["commands"]);
+});
+
+test("Azure Linux stops when sandbox prerequisites cannot be installed", () => {
+    const { result, commands, files } = runAzureLinuxSetup({ FAIL_PACKAGES: "true" });
+    expect(result.status).toBe(37);
+    expect(commands).not.toContain("runsc install");
+    expect(commands).not.toContain("systemctl");
+    expect(files).toEqual(["commands"]);
+});
+
+test("Azure Linux stops before configuring Docker if Buildx is unavailable", () => {
+    const { result, commands, files } = runAzureLinuxSetup({ FAIL_BUILDX: "true" });
+    expect(result.status).toBe(37);
+    expect(commands).not.toContain("runsc install");
+    expect(commands).not.toContain("systemctl");
+    expect(files).toEqual(["commands"]);
+});
+
 afterEach(() => {
     for (const tempDir of tempDirs.splice(0)) {
         fs.rmSync(tempDir, { recursive: true });

scripts/src/__tests__/waitForBenchmarkIdle.test.ts

@@ -0,0 +1,109 @@
+import fs from "node:fs";
+
+import { afterEach, beforeEach, expect, test, vi } from "vitest";
+
+import { waitForBenchmarkIdle } from "../waitForBenchmarkIdle.js";
+
+beforeEach(() => {
+    vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
+    vi.spyOn(console, "log").mockImplementation(() => {});
+    vi.spyOn(console, "warn").mockImplementation(() => {});
+});
+
+afterEach(() => {
+    vi.restoreAllMocks();
+    vi.useRealTimers();
+});
+
+function mockCounters(options: {
+    busy?: (second: number, cpu: number) => number;
+    pressure?: (second: number, resource: string) => number;
+    pressureError?: string;
+} = {}) {
+    let second = -1;
+    const ticks = Array<number>(8).fill(0);
+    const busy = Array<number>(8).fill(0);
+    const pressure = new Map<string, number>();
+    return vi.spyOn(fs, "readFileSync").mockImplementation(file => {
+        if (file === "/proc/stat") {
+            second++;
+            return ticks.map((_, cpu) => {
+                ticks[cpu] += 100;
+                busy[cpu] += options.busy?.(second, cpu) ?? 3;
+                return `cpu${cpu} ${busy[cpu]} 0 0 ${ticks[cpu] - busy[cpu]} 0 0 0 0 ${busy[cpu]} 0`;
+            }).join("\n");
+        }
+        const resource = String(file).split("/").at(-1)!;
+        if (options.pressureError) {
+            throw Object.assign(new Error("cannot read pressure"), { code: options.pressureError });
+        }
+        const total = (pressure.get(resource) ?? 0) + (options.pressure?.(second, resource) ?? 0);
+        pressure.set(resource, total);
+        return `some avg10=99.99 avg60=99.99 avg300=99.99 total=${total}\nfull avg10=0 total=0\n`;
+    });
+}
+
+test("requires five quiet seconds and uses deltas rather than stale averages or double-counted guest time", async () => {
+    const read = mockCounters();
+    const waiting = waitForBenchmarkIdle();
+    await vi.advanceTimersByTimeAsync(4000);
+    expect(console.log).not.toHaveBeenCalledWith(expect.stringContaining("Machine is quiet"));
+    await vi.advanceTimersByTimeAsync(1000);
+    await waiting;
+    expect(console.log).toHaveBeenCalledWith(expect.stringContaining("Machine is quiet"));
+    expect(read).toHaveBeenCalledTimes(24);
+});
+
+test("restarts the quiet window when one CPU becomes busy", async () => {
+    mockCounters({ busy: (second, cpu) => second === 3 && cpu === 0 ? 30 : 0 });
+    const waiting = waitForBenchmarkIdle();
+    await vi.advanceTimersByTimeAsync(7000);
+    expect(console.log).not.toHaveBeenCalledWith(expect.stringContaining("Machine is quiet"));
+    await vi.advanceTimersByTimeAsync(1000);
+    await waiting;
+});
+
+test.each(["cpu", "io", "memory"])("waits for recent %s pressure to subside", async resource => {
+    mockCounters({ pressure: (second, name) => second <= 3 && name === resource ? 20_000 : 0 });
+    const waiting = waitForBenchmarkIdle();
+    await vi.advanceTimersByTimeAsync(8000);
+    await waiting;
+    expect(console.log).toHaveBeenCalledWith(expect.stringContaining(`${resource}=2.00%`));
+    expect(console.log).toHaveBeenCalledWith(expect.stringContaining("Machine is quiet"));
+});
+
+test("fails within the time bound instead of starting measurements on a persistently busy machine", async () => {
+    mockCounters({ busy: () => 50 });
+    const waiting = expect(waitForBenchmarkIdle()).rejects.toThrow("Machine did not become quiet within 60s");
+    await vi.advanceTimersByTimeAsync(60_000);
+    await waiting;
+    expect(console.log).not.toHaveBeenCalledWith(expect.stringContaining("Machine is quiet"));
+    expect(vi.getTimerCount()).toBe(0);
+});
+
+test("explicitly warns once per unavailable pressure signal and still checks CPU counters", async () => {
+    mockCounters({ pressureError: "ENOENT" });
+    const waiting = waitForBenchmarkIdle();
+    await vi.advanceTimersByTimeAsync(5000);
+    await waiting;
+    expect(console.warn).toHaveBeenCalledTimes(3);
+    expect(console.warn).toHaveBeenCalledWith(expect.stringContaining("cpu pressure counters unavailable"));
+    expect(console.log).toHaveBeenCalledWith(expect.stringContaining("Machine is quiet"));
+});
+
+test("does not suppress permission errors reading telemetry", async () => {
+    mockCounters({ pressureError: "EACCES" });
+    await expect(waitForBenchmarkIdle()).rejects.toMatchObject({ code: "EACCES" });
+});
+
+test("rejects malformed pressure telemetry rather than treating it as idle", async () => {
+    vi.spyOn(fs, "readFileSync").mockImplementation(file =>
+        file === "/proc/stat" ? "cpu0 0 0 0 100 0 0 0 0\n" : "some avg10=0 total=invalid\n"
+    );
+    await expect(waitForBenchmarkIdle()).rejects.toThrow("Missing cpu pressure counter");
+});
+
+test.each(["cpu0 invalid\n", "intr 1\n"])("rejects malformed or missing CPU counters: %s", async counters => {
+    vi.spyOn(fs, "readFileSync").mockReturnValue(counters);
+    await expect(waitForBenchmarkIdle()).rejects.toThrow(/CPU counters/);
+});