From 5198f2d449d7382277d5d68beb9f5602af837894 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:06:51 -0700 Subject: [PATCH 1/4] Allow provisional benchmarks on hosted runners An outage of the dedicated runner fleet leaves compiler comparisons blocked. Allow temporary hosted measurements without mixing them into the historical series or losing the existing sandbox boundaries. --- README.md | 14 +++++ build/benchmark.yml | 69 +++++++++++---------- build/templates/installSandbox.yml | 60 ++++++++++++++++++ scripts/src/__tests__/scenarioSetup.test.ts | 46 ++++++++++++++ 4 files changed, 155 insertions(+), 34 deletions(-) create mode 100644 build/templates/installSandbox.yml diff --git a/README.md b/README.md index 1d748cda..bede5471 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,20 @@ API benchmarks use Node hosts installed with `ts-perf host install`, not the amb To run the benchmark in CI, add it to `allScenarios` in `scripts/src/setupPipeline.ts` with `kind: "api"` and `runIn: RunType.None`, then request the `api` preset. +## Temporary hosted benchmark runners + +When the dedicated machines are unavailable, manually queue `build/benchmark.yml` +with `USE_HOSTED_RUNNERS: true`. The benchmark jobs use Microsoft-hosted +`ubuntu-24.04` VMs, with both revisions of each scenario measured on the same VM. +Presets and explicit commit comparisons work as usual. + +Hosted results are provisional: the VMs lack the dedicated machines' CPU isolation +and system tuning. Reports include a warning, and results are never uploaded to +the historical series, even for main/release triggers or `HISTORICAL_RUN`. +The compiler and dependency-installation sandboxes remain enabled. + +Leave `USE_HOSTED_RUNNERS` at its default `false` to use the dedicated machines. + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/build/benchmark.yml b/build/benchmark.yml index c0bb645b..1287c728 100644 --- a/build/benchmark.yml +++ b/build/benchmark.yml @@ -66,6 +66,11 @@ parameters: type: boolean default: false + - name: USE_HOSTED_RUNNERS + displayName: Use temporary hosted runners (no historical uploads; noisier results) + type: boolean + default: false + # PR trigger params - name: DISTINCT_ID displayName: Distinct ID for this run (PR runs only) @@ -128,7 +133,7 @@ variables: PRETTY_REF: $[ replace(replace(replace(replace(variables['REF'], '/merge', ''), 'refs/pull/', 'pr.'), 'refs/heads/', ''), '/', '_') ] IS_PR: $[ startsWith(variables['REF'], 'refs/pull/') ] # True if this run should demand a baseline machine. - USE_BASELINE_MACHINE: $[ or(eq(variables['Build.Reason'], 'ResourceTrigger'), ${{ parameters.HISTORICAL_RUN }}) ] + USE_BASELINE_MACHINE: $[ and(not(${{ parameters.USE_HOSTED_RUNNERS }}), or(eq(variables['Build.Reason'], 'ResourceTrigger'), ${{ parameters.HISTORICAL_RUN }})) ] # Only upload if the provided ref is explicitly main/release-* and we've run on the baseline machine. # This means that main pushes will automatically upload, but we can still queue builds off of main with a specific commit in the API. SHOULD_UPLOAD: $[ and(or(eq(variables['REF'], 'refs/heads/main'), startsWith(variables['REF'], 'refs/heads/release-')), eq(variables['USE_BASELINE_MACHINE'], 'true')) ] @@ -158,29 +163,7 @@ jobs: - template: templates/setup.yml - template: templates/cloneAndBuildBenchmarkRepo.yml # Sets $(BENCH_SCRIPTS), $(TSPERF_EXE) - - bash: | - set -exo pipefail - sudo apt-get update - sudo apt-get install -y bubblewrap - - # Ubuntu 24.04 blocks unprivileged user namespaces unless an AppArmor profile - # allows them. Allow them for bubblewrap only, like Ubuntu's browser profiles. - if [ "$(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns 2>/dev/null)" = "1" ]; then - sudo tee /etc/apparmor.d/bwrap > /dev/null << 'EOF' - abi , - include - - profile bwrap /usr/bin/bwrap flags=(unconfined) { - userns, - } - EOF - sudo apparmor_parser -r /etc/apparmor.d/bwrap - fi - bwrap --ro-bind / / --unshare-user --cap-drop ALL true - target: - commands: restricted - settableVariables: none - displayName: Install compiler build sandbox + - template: templates/installSandbox.yml - ${{ if eq(parameters.TS_GO, true) }}: - checkout: typescript-go @@ -379,9 +362,12 @@ jobs: dependsOn: Setup condition: and(succeeded(), not(eq(dependencies.Setup.outputs['setupPipeline.MATRIX_${{ replace(agent, '-', '_') }}'], '{}'))) pool: - name: ts-perf-ddfun - ${{ if not(eq(agent, 'any')) }}: - demands: Agent.Name -equals ${{ agent }} + ${{ if parameters.USE_HOSTED_RUNNERS }}: + vmImage: ubuntu-24.04 + ${{ else }}: + name: ts-perf-ddfun + ${{ if not(eq(agent, 'any')) }}: + demands: Agent.Name -equals ${{ agent }} workspace: clean: all # Always start with a clean slate. @@ -404,6 +390,11 @@ jobs: - template: templates/setup.yml - template: templates/cloneAndBuildBenchmarkRepo.yml # Sets $(BENCH_SCRIPTS), $(TSPERF_EXE) + - ${{ if parameters.USE_HOSTED_RUNNERS }}: + - template: templates/installSandbox.yml + parameters: + HOSTED_BENCHMARK: true + - download: current artifact: BuiltTypeScript displayName: Download built TypeScript @@ -417,13 +408,14 @@ jobs: displayName: Make native TypeScript executable # Installed by ansible/tasks/pyperf.yml. - - bash: | - set -exo pipefail - sudo -n /usr/local/sbin/tsperf-tune-system - target: - commands: restricted - settableVariables: none - displayName: Tune system + - ${{ if not(parameters.USE_HOSTED_RUNNERS) }}: + - bash: | + set -exo pipefail + sudo -n /usr/local/sbin/tsperf-tune-system + target: + commands: restricted + settableVariables: none + displayName: Tune system # Install hosts before entering the network-isolated benchmark sandbox. # Read-only mounts protect these binaries from writes by the compiler. @@ -707,6 +699,15 @@ jobs: settableVariables: none displayName: Create merged output directory + - ${{ if parameters.USE_HOSTED_RUNNERS }}: + - bash: | + set -eo pipefail + echo '

Provisional hosted-runner results: these VMs are not the dedicated benchmark machines. Results may be noisier and are not comparable to the historical series. For comparisons, both revisions of each scenario ran on the same VM.

' > "$(Pipeline.Workspace)/comment.html" + target: + commands: restricted + settableVariables: none + displayName: Label provisional hosted-runner results + - bash: | set -eo pipefail diff --git a/build/templates/installSandbox.yml b/build/templates/installSandbox.yml new file mode 100644 index 00000000..92b29d46 --- /dev/null +++ b/build/templates/installSandbox.yml @@ -0,0 +1,60 @@ +parameters: + - name: HOSTED_BENCHMARK + type: boolean + default: false + +steps: + - bash: | + set -exo pipefail + sudo apt-get update + sudo apt-get install -y bubblewrap + + # Ubuntu 24.04 blocks unprivileged user namespaces unless an AppArmor profile + # allows them. Allow them for bubblewrap only, like Ubuntu's browser profiles. + if [ "$(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns 2>/dev/null)" = "1" ]; then + sudo tee /etc/apparmor.d/bwrap > /dev/null << 'EOF' + abi , + include + + profile bwrap /usr/bin/bwrap flags=(unconfined) { + userns, + } + EOF + sudo apparmor_parser -r /etc/apparmor.d/bwrap + fi + bwrap --ro-bind / / --unshare-user --cap-drop ALL true + target: + commands: restricted + settableVariables: none + displayName: Install compiler sandbox + + - ${{ if parameters.HOSTED_BENCHMARK }}: + # Match the scenario installer sandbox used by the GitHub-hosted scenario tests. + # Only dependency installation uses gVisor; measured compilers use bubblewrap. + - bash: | + set -exo pipefail + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg + curl -fsSL https://gvisor.dev/archive.key | sudo gpg --dearmor -o /usr/share/keyrings/gvisor-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/gvisor-archive-keyring.gpg] https://storage.googleapis.com/gvisor/releases release main" | sudo tee /etc/apt/sources.list.d/gvisor.list > /dev/null + sudo apt-get update + sudo apt-get install -y runsc + sudo runsc install + sudo systemctl restart docker + docker info + target: + commands: restricted + settableVariables: none + displayName: Install scenario sandbox + + - bash: | + set -eo pipefail + CPUS=$(awk '/^Cpus_allowed_list:/ {print $2}' /proc/self/status) + test -n "$CPUS" + taskset --cpu-list "$CPUS" true + echo "##vso[task.setvariable variable=TSPERF_AGENT_BENCHMARK_CPUS;isReadOnly=true]$CPUS" + echo "##vso[task.logissue type=warning]Using a hosted VM without dedicated CPU isolation or system tuning; results are provisional." + target: + commands: restricted + settableVariables: + - TSPERF_AGENT_BENCHMARK_CPUS + displayName: Configure hosted benchmark CPUs diff --git a/scripts/src/__tests__/scenarioSetup.test.ts b/scripts/src/__tests__/scenarioSetup.test.ts index 126de7af..00930fc4 100644 --- a/scripts/src/__tests__/scenarioSetup.test.ts +++ b/scripts/src/__tests__/scenarioSetup.test.ts @@ -5,12 +5,58 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { afterEach, expect, test } from "vitest"; +import { parse } from "yaml"; const commonScript = fileURLToPath(new URL("../../../cases/common.sh", import.meta.url)); const setupScript = fileURLToPath(new URL("../../setupScenario.sh", import.meta.url)); const benchmarkScript = fileURLToPath(new URL("../../runBenchmarkSandbox.sh", import.meta.url)); +const sandboxTemplate = parse( + fs.readFileSync(fileURLToPath(new URL("../../../build/templates/installSandbox.yml", import.meta.url)), "utf8"), +) as { + steps: ( + | { bash: string; displayName: string; } + | { "${{ if parameters.HOSTED_BENCHMARK }}": { bash: string; displayName: string; }[]; } + )[]; +}; const tempDirs: string[] = []; +function hostedCpuScript() { + const hostedSteps = sandboxTemplate.steps.find(step => "${{ if parameters.HOSTED_BENCHMARK }}" in step); + if (!hostedSteps || !("${{ if parameters.HOSTED_BENCHMARK }}" in hostedSteps)) { + throw new Error("Missing hosted sandbox setup"); + } + const cpuStep = hostedSteps["${{ if parameters.HOSTED_BENCHMARK }}"] + .find(step => step.displayName === "Configure hosted benchmark CPUs"); + if (!cpuStep) { + throw new Error("Missing hosted CPU configuration"); + } + return cpuStep.bash; +} + +test("hosted CPU configuration selects CPUs the benchmark process can actually use", () => { + const allowed = fs.readFileSync("/proc/self/status", "utf8").match(/^Cpus_allowed_list:\s*(.+)$/m)?.[1]; + expect(allowed).toBeTruthy(); + const result = spawnSync("bash", ["-c", hostedCpuScript()], { encoding: "utf8", timeout: 10_000 }); + expect(result.status).toBe(0); + expect(result.stdout).toContain( + `##vso[task.setvariable variable=TSPERF_AGENT_BENCHMARK_CPUS;isReadOnly=true]${allowed}`, + ); + expect(result.stdout).toContain("results are provisional"); +}); + +test("hosted CPU configuration does not publish unusable affinity", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "typescript-benchmarking-hosted-")); + tempDirs.push(tempDir); + fs.writeFileSync(path.join(tempDir, "taskset"), "#!/bin/sh\nexit 37\n", { mode: 0o755 }); + const result = spawnSync("bash", ["-c", hostedCpuScript()], { + env: { ...process.env, PATH: `${tempDir}:${process.env.PATH}` }, + encoding: "utf8", + timeout: 10_000, + }); + expect(result.status).toBe(37); + expect(result.stdout).not.toContain("task.setvariable"); +}); + afterEach(() => { for (const tempDir of tempDirs.splice(0)) { fs.rmSync(tempDir, { recursive: true }); From 0fb6a5235d06db34d1786b5f0ee995896e36344e Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:10:11 -0700 Subject: [PATCH 2/4] Default to hosted benchmark runners during the outage The dedicated runner fleet is unavailable, so leaving it as the default keeps automatically triggered benchmarks blocked until it recovers. --- README.md | 6 +++--- build/benchmark.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index bede5471..d3d26c98 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,8 @@ To run the benchmark in CI, add it to `allScenarios` in `scripts/src/setupPipeli ## Temporary hosted benchmark runners -When the dedicated machines are unavailable, manually queue `build/benchmark.yml` -with `USE_HOSTED_RUNNERS: true`. The benchmark jobs use Microsoft-hosted +While the dedicated machines are unavailable, `build/benchmark.yml` defaults to +`USE_HOSTED_RUNNERS: true`. The benchmark jobs use Microsoft-hosted `ubuntu-24.04` VMs, with both revisions of each scenario measured on the same VM. Presets and explicit commit comparisons work as usual. @@ -46,7 +46,7 @@ and system tuning. Reports include a warning, and results are never uploaded to the historical series, even for main/release triggers or `HISTORICAL_RUN`. The compiler and dependency-installation sandboxes remain enabled. -Leave `USE_HOSTED_RUNNERS` at its default `false` to use the dedicated machines. +Set `USE_HOSTED_RUNNERS: false` to use the dedicated machines. ## Contributing diff --git a/build/benchmark.yml b/build/benchmark.yml index 1287c728..af96e4c3 100644 --- a/build/benchmark.yml +++ b/build/benchmark.yml @@ -69,7 +69,7 @@ parameters: - name: USE_HOSTED_RUNNERS displayName: Use temporary hosted runners (no historical uploads; noisier results) type: boolean - default: false + default: true # PR trigger params - name: DISTINCT_ID From 2ce93555c1305af787df5d719f68d89a7a629311 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:10:34 -0700 Subject: [PATCH 3/4] Remove documentation changes from the hosted runner fallback --- README.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/README.md b/README.md index d3d26c98..1d748cda 100644 --- a/README.md +++ b/README.md @@ -34,20 +34,6 @@ API benchmarks use Node hosts installed with `ts-perf host install`, not the amb To run the benchmark in CI, add it to `allScenarios` in `scripts/src/setupPipeline.ts` with `kind: "api"` and `runIn: RunType.None`, then request the `api` preset. -## Temporary hosted benchmark runners - -While the dedicated machines are unavailable, `build/benchmark.yml` defaults to -`USE_HOSTED_RUNNERS: true`. The benchmark jobs use Microsoft-hosted -`ubuntu-24.04` VMs, with both revisions of each scenario measured on the same VM. -Presets and explicit commit comparisons work as usual. - -Hosted results are provisional: the VMs lack the dedicated machines' CPU isolation -and system tuning. Reports include a warning, and results are never uploaded to -the historical series, even for main/release triggers or `HISTORICAL_RUN`. -The compiler and dependency-installation sandboxes remain enabled. - -Set `USE_HOSTED_RUNNERS: false` to use the dedicated machines. - ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a From de39f4ec90ee2e97822e7c55608a3d039f2f6273 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:31:29 -0700 Subject: [PATCH 4/4] Use the latest Ubuntu image for hosted benchmarks --- build/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/benchmark.yml b/build/benchmark.yml index af96e4c3..9aad6259 100644 --- a/build/benchmark.yml +++ b/build/benchmark.yml @@ -363,7 +363,7 @@ jobs: condition: and(succeeded(), not(eq(dependencies.Setup.outputs['setupPipeline.MATRIX_${{ replace(agent, '-', '_') }}'], '{}'))) pool: ${{ if parameters.USE_HOSTED_RUNNERS }}: - vmImage: ubuntu-24.04 + vmImage: ubuntu-latest ${{ else }}: name: ts-perf-ddfun ${{ if not(eq(agent, 'any')) }}: