diff --git a/build/benchmark.yml b/build/benchmark.yml index c0bb645b..9aad6259 100644 --- a/build/benchmark.yml +++ b/build/benchmark.yml @@ -66,6 +66,11 @@ parameters: type: boolean default: false + - name: USE_HOSTED_RUNNERS + displayName: Use temporary hosted runners (no historical uploads; noisier results) + type: boolean + default: true + # PR trigger params - name: DISTINCT_ID displayName: Distinct ID for this run (PR runs only) @@ -128,7 +133,7 @@ variables: PRETTY_REF: $[ replace(replace(replace(replace(variables['REF'], '/merge', ''), 'refs/pull/', 'pr.'), 'refs/heads/', ''), '/', '_') ] IS_PR: $[ startsWith(variables['REF'], 'refs/pull/') ] # True if this run should demand a baseline machine. - USE_BASELINE_MACHINE: $[ or(eq(variables['Build.Reason'], 'ResourceTrigger'), ${{ parameters.HISTORICAL_RUN }}) ] + USE_BASELINE_MACHINE: $[ and(not(${{ parameters.USE_HOSTED_RUNNERS }}), or(eq(variables['Build.Reason'], 'ResourceTrigger'), ${{ parameters.HISTORICAL_RUN }})) ] # Only upload if the provided ref is explicitly main/release-* and we've run on the baseline machine. # This means that main pushes will automatically upload, but we can still queue builds off of main with a specific commit in the API. SHOULD_UPLOAD: $[ and(or(eq(variables['REF'], 'refs/heads/main'), startsWith(variables['REF'], 'refs/heads/release-')), eq(variables['USE_BASELINE_MACHINE'], 'true')) ] @@ -158,29 +163,7 @@ jobs: - template: templates/setup.yml - template: templates/cloneAndBuildBenchmarkRepo.yml # Sets $(BENCH_SCRIPTS), $(TSPERF_EXE) - - bash: | - set -exo pipefail - sudo apt-get update - sudo apt-get install -y bubblewrap - - # Ubuntu 24.04 blocks unprivileged user namespaces unless an AppArmor profile - # allows them. Allow them for bubblewrap only, like Ubuntu's browser profiles. - if [ "$(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns 2>/dev/null)" = "1" ]; then - sudo tee /etc/apparmor.d/bwrap > /dev/null << 'EOF' - abi , - include - - profile bwrap /usr/bin/bwrap flags=(unconfined) { - userns, - } - EOF - sudo apparmor_parser -r /etc/apparmor.d/bwrap - fi - bwrap --ro-bind / / --unshare-user --cap-drop ALL true - target: - commands: restricted - settableVariables: none - displayName: Install compiler build sandbox + - template: templates/installSandbox.yml - ${{ if eq(parameters.TS_GO, true) }}: - checkout: typescript-go @@ -379,9 +362,12 @@ jobs: dependsOn: Setup condition: and(succeeded(), not(eq(dependencies.Setup.outputs['setupPipeline.MATRIX_${{ replace(agent, '-', '_') }}'], '{}'))) pool: - name: ts-perf-ddfun - ${{ if not(eq(agent, 'any')) }}: - demands: Agent.Name -equals ${{ agent }} + ${{ if parameters.USE_HOSTED_RUNNERS }}: + vmImage: ubuntu-latest + ${{ else }}: + name: ts-perf-ddfun + ${{ if not(eq(agent, 'any')) }}: + demands: Agent.Name -equals ${{ agent }} workspace: clean: all # Always start with a clean slate. @@ -404,6 +390,11 @@ jobs: - template: templates/setup.yml - template: templates/cloneAndBuildBenchmarkRepo.yml # Sets $(BENCH_SCRIPTS), $(TSPERF_EXE) + - ${{ if parameters.USE_HOSTED_RUNNERS }}: + - template: templates/installSandbox.yml + parameters: + HOSTED_BENCHMARK: true + - download: current artifact: BuiltTypeScript displayName: Download built TypeScript @@ -417,13 +408,14 @@ jobs: displayName: Make native TypeScript executable # Installed by ansible/tasks/pyperf.yml. - - bash: | - set -exo pipefail - sudo -n /usr/local/sbin/tsperf-tune-system - target: - commands: restricted - settableVariables: none - displayName: Tune system + - ${{ if not(parameters.USE_HOSTED_RUNNERS) }}: + - bash: | + set -exo pipefail + sudo -n /usr/local/sbin/tsperf-tune-system + target: + commands: restricted + settableVariables: none + displayName: Tune system # Install hosts before entering the network-isolated benchmark sandbox. # Read-only mounts protect these binaries from writes by the compiler. @@ -707,6 +699,15 @@ jobs: settableVariables: none displayName: Create merged output directory + - ${{ if parameters.USE_HOSTED_RUNNERS }}: + - bash: | + set -eo pipefail + echo '

Provisional hosted-runner results: these VMs are not the dedicated benchmark machines. Results may be noisier and are not comparable to the historical series. For comparisons, both revisions of each scenario ran on the same VM.

' > "$(Pipeline.Workspace)/comment.html" + target: + commands: restricted + settableVariables: none + displayName: Label provisional hosted-runner results + - bash: | set -eo pipefail diff --git a/build/templates/installSandbox.yml b/build/templates/installSandbox.yml new file mode 100644 index 00000000..92b29d46 --- /dev/null +++ b/build/templates/installSandbox.yml @@ -0,0 +1,60 @@ +parameters: + - name: HOSTED_BENCHMARK + type: boolean + default: false + +steps: + - bash: | + set -exo pipefail + sudo apt-get update + sudo apt-get install -y bubblewrap + + # Ubuntu 24.04 blocks unprivileged user namespaces unless an AppArmor profile + # allows them. Allow them for bubblewrap only, like Ubuntu's browser profiles. + if [ "$(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns 2>/dev/null)" = "1" ]; then + sudo tee /etc/apparmor.d/bwrap > /dev/null << 'EOF' + abi , + include + + profile bwrap /usr/bin/bwrap flags=(unconfined) { + userns, + } + EOF + sudo apparmor_parser -r /etc/apparmor.d/bwrap + fi + bwrap --ro-bind / / --unshare-user --cap-drop ALL true + target: + commands: restricted + settableVariables: none + displayName: Install compiler sandbox + + - ${{ if parameters.HOSTED_BENCHMARK }}: + # Match the scenario installer sandbox used by the GitHub-hosted scenario tests. + # Only dependency installation uses gVisor; measured compilers use bubblewrap. + - bash: | + set -exo pipefail + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg + curl -fsSL https://gvisor.dev/archive.key | sudo gpg --dearmor -o /usr/share/keyrings/gvisor-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/gvisor-archive-keyring.gpg] https://storage.googleapis.com/gvisor/releases release main" | sudo tee /etc/apt/sources.list.d/gvisor.list > /dev/null + sudo apt-get update + sudo apt-get install -y runsc + sudo runsc install + sudo systemctl restart docker + docker info + target: + commands: restricted + settableVariables: none + displayName: Install scenario sandbox + + - bash: | + set -eo pipefail + CPUS=$(awk '/^Cpus_allowed_list:/ {print $2}' /proc/self/status) + test -n "$CPUS" + taskset --cpu-list "$CPUS" true + echo "##vso[task.setvariable variable=TSPERF_AGENT_BENCHMARK_CPUS;isReadOnly=true]$CPUS" + echo "##vso[task.logissue type=warning]Using a hosted VM without dedicated CPU isolation or system tuning; results are provisional." + target: + commands: restricted + settableVariables: + - TSPERF_AGENT_BENCHMARK_CPUS + displayName: Configure hosted benchmark CPUs diff --git a/scripts/src/__tests__/scenarioSetup.test.ts b/scripts/src/__tests__/scenarioSetup.test.ts index 126de7af..00930fc4 100644 --- a/scripts/src/__tests__/scenarioSetup.test.ts +++ b/scripts/src/__tests__/scenarioSetup.test.ts @@ -5,12 +5,58 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { afterEach, expect, test } from "vitest"; +import { parse } from "yaml"; const commonScript = fileURLToPath(new URL("../../../cases/common.sh", import.meta.url)); const setupScript = fileURLToPath(new URL("../../setupScenario.sh", import.meta.url)); const benchmarkScript = fileURLToPath(new URL("../../runBenchmarkSandbox.sh", import.meta.url)); +const sandboxTemplate = parse( + fs.readFileSync(fileURLToPath(new URL("../../../build/templates/installSandbox.yml", import.meta.url)), "utf8"), +) as { + steps: ( + | { bash: string; displayName: string; } + | { "${{ if parameters.HOSTED_BENCHMARK }}": { bash: string; displayName: string; }[]; } + )[]; +}; const tempDirs: string[] = []; +function hostedCpuScript() { + const hostedSteps = sandboxTemplate.steps.find(step => "${{ if parameters.HOSTED_BENCHMARK }}" in step); + if (!hostedSteps || !("${{ if parameters.HOSTED_BENCHMARK }}" in hostedSteps)) { + throw new Error("Missing hosted sandbox setup"); + } + const cpuStep = hostedSteps["${{ if parameters.HOSTED_BENCHMARK }}"] + .find(step => step.displayName === "Configure hosted benchmark CPUs"); + if (!cpuStep) { + throw new Error("Missing hosted CPU configuration"); + } + return cpuStep.bash; +} + +test("hosted CPU configuration selects CPUs the benchmark process can actually use", () => { + const allowed = fs.readFileSync("/proc/self/status", "utf8").match(/^Cpus_allowed_list:\s*(.+)$/m)?.[1]; + expect(allowed).toBeTruthy(); + const result = spawnSync("bash", ["-c", hostedCpuScript()], { encoding: "utf8", timeout: 10_000 }); + expect(result.status).toBe(0); + expect(result.stdout).toContain( + `##vso[task.setvariable variable=TSPERF_AGENT_BENCHMARK_CPUS;isReadOnly=true]${allowed}`, + ); + expect(result.stdout).toContain("results are provisional"); +}); + +test("hosted CPU configuration does not publish unusable affinity", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "typescript-benchmarking-hosted-")); + tempDirs.push(tempDir); + fs.writeFileSync(path.join(tempDir, "taskset"), "#!/bin/sh\nexit 37\n", { mode: 0o755 }); + const result = spawnSync("bash", ["-c", hostedCpuScript()], { + env: { ...process.env, PATH: `${tempDir}:${process.env.PATH}` }, + encoding: "utf8", + timeout: 10_000, + }); + expect(result.status).toBe(37); + expect(result.stdout).not.toContain("task.setvariable"); +}); + afterEach(() => { for (const tempDir of tempDirs.splice(0)) { fs.rmSync(tempDir, { recursive: true });