Interleave baseline and new benchmark rounds

build/benchmark.yml

@@ -81,6 +81,11 @@ parameters:
     type: string
     default: azure-linux-3-benchmarks
 
+  - name: INTERLEAVE_BENCHMARKS
+    displayName: Alternate baseline and new benchmark rounds
+    type: boolean
+    default: true
+
   # PR trigger params
   - name: DISTINCT_ID
     displayName: Distinct ID for this run (PR runs only)
@@ -464,6 +469,7 @@ jobs:
               TYPESCRIPT_COMMIT_NEW: $(TYPESCRIPT_COMMIT_NEW)
               TYPESCRIPT_COMMIT_BASELINE: $(TYPESCRIPT_COMMIT_BASELINE)
               TSPERF_VARIABLES_FILE: $(Pipeline.Workspace)/first-scenario-variables.json
+              TSPERF_SCENARIO_COMMIT_DEPENDENT_FILE: $(Pipeline.Workspace)/scenario-commit-dependent
 
           - bash: |
               set -eo pipefail
@@ -549,57 +555,167 @@ jobs:
                 - BWRAP_ARGS
             displayName: Create benchmark sandbox
 
-          - ${{ if parameters.USE_HOSTED_RUNNERS }}:
-              - bash: node $(BENCH_SCRIPTS)/waitForBenchmarkIdle.js
+          # Sequential phases on shared cloud VMs can shift every sample when the VM's speed
+          # changes between phases. Alternate shorter rounds so drift affects both sides.
+          # Pairs alternate order (new/baseline, then baseline/new) to cancel linear drift.
+          # Each round runs the configured warmups; merging restores the per-side sample count.
+          - ${{ if parameters.INTERLEAVE_BENCHMARKS }}:
+              - bash: |
+                  set -eo pipefail
+                  SCRIPTS_DIR=$(Pipeline.Workspace)/typescript-benchmarking/scripts
+                  ROUNDS_DIR=$(BENCH_ARTIFACTS_DIR)/rounds
+                  mkdir -p "$ROUNDS_DIR"
+
+                  TOTAL=$TSPERF_JOB_ITERATIONS
+                  if [[ ! "$TOTAL" =~ ^[1-9][0-9]*$ ]]; then
+                    echo "Unexpected iteration count" >&2
+                    exit 1
+                  fi
+                  PAIRS=$(( TOTAL < 4 ? TOTAL : 4 ))
+
+                  # Scenarios cloned at the compiler commit must be set up again per side.
+                  RESETUP=false
+                  if [ "$TYPESCRIPT_COMMIT_BASELINE" != "$TYPESCRIPT_COMMIT_NEW" ] \
+                    && [ -e "$TSPERF_SCENARIO_COMMIT_DEPENDENT_FILE" ]; then
+                    RESETUP=true
+                  fi
+                  # The first setup step used the new commit.
+                  CURRENT_SIDE=new
+
+                  setup_side() {
+                    if [ "$1" = new ]; then
+                      export TYPESCRIPT_COMMIT="$TYPESCRIPT_COMMIT_NEW"
+                    else
+                      export TYPESCRIPT_COMMIT="$TYPESCRIPT_COMMIT_BASELINE"
+                    fi
+                    for ATTEMPT in 1 2 3 4; do
+                      if bash "$SCRIPTS_DIR/setupScenario.sh"; then
+                        return 0
+                      fi
+                      echo "Scenario setup failed (attempt $ATTEMPT)" >&2
+                      sleep 10
+                    done
+                    return 1
+                  }
+
+                  # Rounds check for quiet four times as often as sequential runs. Brief
+                  # periodic I/O on otherwise idle VMs occasionally prevents a quiet window,
+                  # so retry before failing the job.
+                  wait_for_idle() {
+                    for ATTEMPT in 1 2 3; do
+                      if node $(BENCH_SCRIPTS)/waitForBenchmarkIdle.js; then
+                        return 0
+                      fi
+                      echo "Machine was not quiet (attempt $ATTEMPT)" >&2
+                    done
+                    return 1
+                  }
+
+                  NEW_FILES=()
+                  BASELINE_FILES=()
+                  for (( PAIR = 0; PAIR < PAIRS; PAIR++ )); do
+                    ITERATIONS=$(( TOTAL / PAIRS + (PAIR < TOTAL % PAIRS ? 1 : 0) ))
+                    if (( PAIR % 2 == 0 )); then
+                      SIDES="new baseline"
+                    else
+                      SIDES="baseline new"
+                    fi
+                    for SIDE in $SIDES; do
+                      echo "Round $PAIR: $SIDE, $ITERATIONS iterations"
+                      if [ "$RESETUP" = true ] && [ "$SIDE" != "$CURRENT_SIDE" ]; then
+                        setup_side "$SIDE"
+                        CURRENT_SIDE=$SIDE
+                      fi
+                      if [ "${WAIT_FOR_IDLE,,}" = true ]; then
+                        wait_for_idle
+                      fi
+                      OUT=$ROUNDS_DIR/${SIDE}_${PAIR}.benchmark
+                      TSPERF_JOB_ITERATIONS=$ITERATIONS bash "$SCRIPTS_DIR/runBenchmarkSandbox.sh" \
+                        $(BWRAP_ARGS) node $(BENCH_SCRIPTS)/runTsPerf.js benchmark-$(TSPERF_JOB_KIND) \
+                        --builtDir $(BUILT_TYPESCRIPT_DIR)/$SIDE \
+                        --save "$OUT"
+                      if [ "$SIDE" = new ]; then
+                        NEW_FILES+=("$OUT")
+                      else
+                        BASELINE_FILES+=("$OUT")
+                      fi
+                    done
+                  done
+
+                  # Round files are written by the sandbox, so merge them there too.
+                  bash "$SCRIPTS_DIR/runBenchmarkSandbox.sh" $(BWRAP_ARGS) \
+                    node $(TSPERF_EXE) merge --iterations "$TOTAL" \
+                    --output $(BENCH_ARTIFACTS_DIR)/new_$(TSPERF_JOB_NAME).$(TSPERF_JOB_KIND).benchmark \
+                    "${NEW_FILES[@]}"
+                  bash "$SCRIPTS_DIR/runBenchmarkSandbox.sh" $(BWRAP_ARGS) \
+                    node $(TSPERF_EXE) merge --iterations "$TOTAL" \
+                    --output $(BENCH_ARTIFACTS_DIR)/baseline_$(TSPERF_JOB_NAME).$(TSPERF_JOB_KIND).benchmark \
+                    "${BASELINE_FILES[@]}"
                 target:
                   commands: restricted
                   settableVariables: none
-                displayName: Wait for idle before new benchmark
+                displayName: Run interleaved $(TSPERF_JOB_KIND) benchmarks
                 condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true'))
+                env:
+                  TYPESCRIPT_COMMIT_NEW: $(TYPESCRIPT_COMMIT_NEW)
+                  TYPESCRIPT_COMMIT_BASELINE: $(TYPESCRIPT_COMMIT_BASELINE)
+                  TSPERF_VARIABLES_FILE: $(Pipeline.Workspace)/interleaved-scenario-variables.json
+                  TSPERF_SCENARIO_COMMIT_DEPENDENT_FILE: $(Pipeline.Workspace)/scenario-commit-dependent
+                  WAIT_FOR_IDLE: ${{ parameters.USE_HOSTED_RUNNERS }}
+          - ${{ else }}:
+              - ${{ if parameters.USE_HOSTED_RUNNERS }}:
+                  - bash: node $(BENCH_SCRIPTS)/waitForBenchmarkIdle.js
+                    target:
+                      commands: restricted
+                      settableVariables: none
+                    displayName: Wait for idle before new benchmark
+                    condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true'))
 
-          - bash: |
-              set -exo pipefail
-              bash $(Pipeline.Workspace)/typescript-benchmarking/scripts/runBenchmarkSandbox.sh \
-                $(BWRAP_ARGS) node $(BENCH_SCRIPTS)/runTsPerf.js benchmark-$(TSPERF_JOB_KIND) \
-                --builtDir $(BUILT_TYPESCRIPT_DIR)/new \
-                --save $(BENCH_ARTIFACTS_DIR)/new_$(TSPERF_JOB_NAME).$(TSPERF_JOB_KIND).benchmark
-            target:
-              commands: restricted
-              settableVariables: none
-            displayName: Run new $(TSPERF_JOB_KIND) benchmark
-            condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true'))
+              - bash: |
+                  set -exo pipefail
+                  bash $(Pipeline.Workspace)/typescript-benchmarking/scripts/runBenchmarkSandbox.sh \
+                    $(BWRAP_ARGS) node $(BENCH_SCRIPTS)/runTsPerf.js benchmark-$(TSPERF_JOB_KIND) \
+                    --builtDir $(BUILT_TYPESCRIPT_DIR)/new \
+                    --save $(BENCH_ARTIFACTS_DIR)/new_$(TSPERF_JOB_NAME).$(TSPERF_JOB_KIND).benchmark
+                target:
+                  commands: restricted
+                  settableVariables: none
+                displayName: Run new $(TSPERF_JOB_KIND) benchmark
+                condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true'))
 
-          - bash: |
-              set -eo pipefail
-              # Special case for self benchmark
-              export TYPESCRIPT_COMMIT="$TYPESCRIPT_COMMIT_BASELINE"
-              bash $(Pipeline.Workspace)/typescript-benchmarking/scripts/setupScenario.sh
-            target:
-              commands: restricted
-              settableVariables: none
-            displayName: Set up scenario for baseline benchmark
-            condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true'))
-            retryCountOnTaskFailure: 3
-            env:
-              TYPESCRIPT_COMMIT_BASELINE: $(TYPESCRIPT_COMMIT_BASELINE)
-              TSPERF_VARIABLES_FILE: $(Pipeline.Workspace)/baseline-scenario-variables.json
+              - bash: |
+                  set -eo pipefail
+                  # Special case for self benchmark
+                  export TYPESCRIPT_COMMIT="$TYPESCRIPT_COMMIT_BASELINE"
+                  bash $(Pipeline.Workspace)/typescript-benchmarking/scripts/setupScenario.sh
+                target:
+                  commands: restricted
+                  settableVariables: none
+                displayName: Set up scenario for baseline benchmark
+                condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true'))
+                retryCountOnTaskFailure: 3
+                env:
+                  TYPESCRIPT_COMMIT_BASELINE: $(TYPESCRIPT_COMMIT_BASELINE)
+                  TSPERF_VARIABLES_FILE: $(Pipeline.Workspace)/baseline-scenario-variables.json
 
-          - bash: |
-              set -eo pipefail
-              node $(BENCH_SCRIPTS)/publishPipelineVariables.js --file $(Pipeline.Workspace)/baseline-scenario-variables.json --job
-            target:
-              commands: restricted
-              settableVariables:
-                - SCENARIO_REF
-            displayName: Publish scenario ref for baseline benchmark
-            condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true'))
+              - bash: |
+                  set -eo pipefail
+                  node $(BENCH_SCRIPTS)/publishPipelineVariables.js --file $(Pipeline.Workspace)/baseline-scenario-variables.json --job
+                target:
+                  commands: restricted
+                  settableVariables:
+                    - SCENARIO_REF
+                displayName: Publish scenario ref for baseline benchmark
+                condition: and(succeeded(), eq(variables['TSPERF_IS_COMPARISON'], 'true'))
 
           - ${{ if parameters.USE_HOSTED_RUNNERS }}:
               - bash: node $(BENCH_SCRIPTS)/waitForBenchmarkIdle.js
                 target:
                   commands: restricted
                   settableVariables: none
                 displayName: Wait for idle before baseline benchmark
+                ${{ if parameters.INTERLEAVE_BENCHMARKS }}:
+                  condition: and(succeeded(), ne(variables['TSPERF_IS_COMPARISON'], 'true'))
 
           - bash: |
               set -exo pipefail
@@ -611,6 +727,8 @@ jobs:
               commands: restricted
               settableVariables: none
             displayName: Run baseline $(TSPERF_JOB_KIND) benchmark
+            ${{ if parameters.INTERLEAVE_BENCHMARKS }}:
+              condition: and(succeeded(), ne(variables['TSPERF_IS_COMPARISON'], 'true'))
 
           # Publish only scheduled filenames, not everything in $(BENCH_ARTIFACTS_DIR).
           # This limits artifact selection; it does not authenticate result contents.

cases/common.sh

@@ -40,6 +40,10 @@ function clone_strada_scenario() {
     if is_corsa || test -z "${TYPESCRIPT_COMMIT:-}"; then
         clone_scenario https://github.com/microsoft/TypeScript.git strada
     else
+        # Interleaved comparisons must set this scenario up again when switching compilers.
+        if [ -n "${TSPERF_SCENARIO_COMMIT_DEPENDENT_FILE:-}" ]; then
+            : > "$TSPERF_SCENARIO_COMMIT_DEPENDENT_FILE"
+        fi
         clone_scenario https://github.com/microsoft/TypeScript.git "$TYPESCRIPT_COMMIT"
     fi
 }

scripts/src/__tests__/scenarioSetup.test.ts

@@ -271,6 +271,38 @@ if [ "$1" = "run" ]; then printf '%s\\n' "$@" >> "$DOCKER_ARGS"; fi
     return { result, args: fs.existsSync(argsFile) ? fs.readFileSync(argsFile, "utf8").split("\n") : [] };
 }
 
+test.each([
+    ["the compiler commit", false, true],
+    ["the strada branch for native compilers", true, false],
+])("marks commit-dependent setup only when cloning %s", (_, isCorsa, dependent) => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "typescript-benchmarking-strada-"));
+    tempDirs.push(tempDir);
+    const builtDir = path.join(tempDir, "built");
+    fs.mkdirSync(path.join(builtDir, "baseline"), { recursive: true });
+    if (isCorsa) {
+        fs.writeFileSync(path.join(builtDir, "baseline/tsc"), "");
+    }
+    const marker = path.join(tempDir, "commit-dependent");
+    const result = spawnSync(
+        "bash",
+        ["-c", 'set -eo pipefail; source "$COMMON_SCRIPT"; clone_scenario() { echo "$2"; }; clone_strada_scenario'],
+        {
+            env: {
+                ...process.env,
+                COMMON_SCRIPT: commonScript,
+                BUILT_TYPESCRIPT_DIR: builtDir,
+                TYPESCRIPT_COMMIT: "0123456789abcdef0123456789abcdef01234567",
+                TSPERF_SCENARIO_COMMIT_DEPENDENT_FILE: marker,
+            },
+            encoding: "utf8",
+            timeout: 10_000,
+        },
+    );
+    expect(result.status).toBe(0);
+    expect(result.stdout.trim()).toBe(isCorsa ? "strada" : "0123456789abcdef0123456789abcdef01234567");
+    expect(fs.existsSync(marker)).toBe(dependent);
+});
+
 test("passes a read-only Git metadata mount and disables Husky during installation", () => {
     const scenario = createScenario();
     const { result, args } = runInstaller(scenario);

ts-perf/packages/commands/src/merge/index.ts

@@ -1,8 +1,10 @@
 import { Benchmark, Command, CommandMap } from "@ts-perf/api";
+import { CommandLineParseError } from "power-options";
 
 export interface MergeOptions {
     files: string[];
     output: string;
+    iterations?: number;
 }
 
 export async function merge(options: MergeOptions) {
@@ -23,6 +25,9 @@ export async function merge(options: MergeOptions) {
         for (let i = 1; i < benchmarks.length; i++) {
             result = result.merge(benchmarks[i]);
         }
+        if (options.iterations !== undefined) {
+            result = result.with({ iterations: options.iterations });
+        }
         await result.saveAsync(options.output);
     }
     catch (e) {
@@ -47,6 +52,16 @@ const command: Command<MergeOptions> = {
             required: true,
             description: "Where to store the merged benchmark file.",
         },
+        iterations: {
+            type: "number",
+            param: "count",
+            description: "Records <count> iterations in the merged file, such as when merging rounds of one benchmark.",
+            validate: (value: number, arg: string) => {
+                if (!Number.isInteger(value) || value <= 0) {
+                    throw new CommandLineParseError(`Option '${arg}' expects a positive integer.`, /*help*/ true);
+                }
+            },
+        },
     },
     exec: ({ options }) => merge(options),
 };