From e37f0b59009a4bc199d08d8d3d501fd5e3e44b4e Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Fri, 16 Oct 2026 16:33:32 -0700 Subject: [PATCH 1/5] Fix benchmark filtering and measurement index roundtrips Remap scenario and host indices independently when filtering a report. Serialize scenarioIndex under the correct key while accepting existing scenarioArgs data. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 958b45cd-e40a-46fc-9de9-3bfde150ace3 --- ts-perf/packages/api/src/model/benchmark.ts | 6 ++- .../api/src/model/measurement.test.ts | 50 +++++++++++++++++++ ts-perf/packages/api/src/model/measurement.ts | 7 ++- 3 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 ts-perf/packages/api/src/model/measurement.test.ts diff --git a/ts-perf/packages/api/src/model/benchmark.ts b/ts-perf/packages/api/src/model/benchmark.ts index 085705b3..4ef66a8d 100644 --- a/ts-perf/packages/api/src/model/benchmark.ts +++ b/ts-perf/packages/api/src/model/benchmark.ts @@ -280,7 +280,9 @@ export class Benchmark { } } - if ((filteredScenarios || filteredHosts) && scenarioMap && hostMap) { + if (filteredScenarios || filteredHosts) { + scenarioMap ??= new Map(this.scenarios.map((_, index) => [index, index])); + hostMap ??= new Map(this.hosts.map((_, index) => [index, index])); filteredMeasurements = []; for (const measurement of this.measurements) { if ( @@ -292,7 +294,7 @@ export class Benchmark { filteredMeasurements.push(measurement.with({ scenarioIndex: scenarioMap.get(measurement.scenarioIndex!), - hostIndex: scenarioMap.get(measurement.hostIndex!), + hostIndex: hostMap.get(measurement.hostIndex!), })); } diff --git a/ts-perf/packages/api/src/model/measurement.test.ts b/ts-perf/packages/api/src/model/measurement.test.ts new file mode 100644 index 00000000..a6295db5 --- /dev/null +++ b/ts-perf/packages/api/src/model/measurement.test.ts @@ -0,0 +1,50 @@ +import { expect, test } from "vitest"; + +import { Benchmark } from "./benchmark"; +import { HostSpecifier } from "./hostSpecifier"; +import { Measurement } from "./measurement"; +import { computeMetrics } from "./value"; + +const measurement = new Measurement("test", 1, HostSpecifier.create({ name: "node", version: "v24.21.0" }), 1, { + totalTime: computeMetrics([1, 2], "Total time", "s", 4), +}); + +test("measurement indices survive serialization and legacy output", () => { + const restored = Measurement.create(JSON.parse(JSON.stringify(measurement))); + expect(restored.scenarioIndex).toBe(1); + expect(restored.hostIndex).toBe(1); + expect( + Measurement.create({ ...JSON.parse(JSON.stringify(measurement)), scenarioIndex: undefined, scenarioArgs: 7 }) + .scenarioIndex, + ).toBe(7); + expect(restored.values?.totalTime?.samples).toEqual([1, 2]); +}); + +function report() { + return Benchmark.create({ + scenarios: [ + { name: "unused", kind: "tsc", configFile: "/unused/scenario.json" }, + { name: "test", kind: "tsc", configFile: "/test/scenario.json" }, + ], + hosts: [{ name: "node", version: "v22.0.0" }, { name: "node", version: "v24.21.0" }], + measurements: [measurement], + }); +} + +test("host-only filtering remaps hostIndex and preserves scenarioIndex", () => { + const filtered = report().filter([], ["node@24.21.0"]); + expect(filtered.hosts).toHaveLength(1); + expect(filtered.measurements).toHaveLength(1); + expect(filtered.measurements[0].hostIndex).toBe(0); + expect(filtered.measurements[0].scenarioIndex).toBe(1); + expect(Measurement.create(JSON.parse(JSON.stringify(filtered.measurements[0]))).hostIndex).toBe(0); +}); + +test("scenario-only filtering remaps scenarioIndex and preserves hostIndex", () => { + const filtered = report().filter(["test"], []); + expect(filtered.scenarios).toHaveLength(1); + expect(filtered.measurements).toHaveLength(1); + expect(filtered.measurements[0].scenarioIndex).toBe(0); + expect(filtered.measurements[0].hostIndex).toBe(1); + expect(filtered.measurements[0].values?.totalTime?.samples).toEqual([1, 2]); +}); diff --git a/ts-perf/packages/api/src/model/measurement.ts b/ts-perf/packages/api/src/model/measurement.ts index 379954c5..f085a268 100644 --- a/ts-perf/packages/api/src/model/measurement.ts +++ b/ts-perf/packages/api/src/model/measurement.ts @@ -47,7 +47,10 @@ export class Measurement { } return new Measurement( components.scenarioName, - components.scenarioIndex, + components.scenarioIndex + ?? ("scenarioArgs" in components && typeof components.scenarioArgs === "number" + ? components.scenarioArgs + : undefined), components.host && HostSpecifier.create(components.host), components.hostIndex, values, @@ -153,7 +156,7 @@ export class Measurement { return { name: this.name, scenarioName: this.scenarioName, - scenarioArgs: this.scenarioIndex, + scenarioIndex: this.scenarioIndex, hostName: this.hostName, hostIndex: this.hostIndex, host: this.host, From da919cf88b0394771a3e75a0cb3a5618a80415db Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Fri, 16 Oct 2026 16:33:57 -0700 Subject: [PATCH 2/5] Add API benchmark runner and timing metrics Run declared Node scripts from prepared solution directories using existing argument expansion and measurement reporting. Require schema-v1 client and server timing dumps and record their scalar counters alongside whole-command wall time. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 958b45cd-e40a-46fc-9de9-3bfde150ace3 --- ts-perf/packages/api/src/expansion.ts | 2 + ts-perf/packages/api/src/model/sample.ts | 30 ++- ts-perf/packages/api/src/model/scenario.ts | 2 +- .../packages/commands/src/benchmark/api.ts | 17 ++ .../commands/src/benchmark/apiTiming.test.ts | 107 ++++++++ .../commands/src/benchmark/apiTiming.ts | 54 +++++ .../packages/commands/src/benchmark/index.ts | 3 + .../commands/src/benchmark/measure.ts | 16 ++ .../src/benchmark/runApiScenario.test.ts | 228 ++++++++++++++++++ .../commands/src/benchmark/runApiScenario.ts | 135 +++++++++++ 10 files changed, 591 insertions(+), 3 deletions(-) create mode 100644 ts-perf/packages/commands/src/benchmark/api.ts create mode 100644 ts-perf/packages/commands/src/benchmark/apiTiming.test.ts create mode 100644 ts-perf/packages/commands/src/benchmark/apiTiming.ts create mode 100644 ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts create mode 100644 ts-perf/packages/commands/src/benchmark/runApiScenario.ts diff --git a/ts-perf/packages/api/src/expansion.ts b/ts-perf/packages/api/src/expansion.ts index e28b3aed..28f5e233 100644 --- a/ts-perf/packages/api/src/expansion.ts +++ b/ts-perf/packages/api/src/expansion.ts @@ -12,6 +12,7 @@ export abstract class ExpansionProvider { | { kind: "tsserver"; options: TSServerOptions; } | { kind: "lsp"; options: TSServerOptions; } | { kind: "tsc"; options: CompilerOptions; } + | { kind: "api"; options: CompilerOptions; } | { kind: "startup"; options: StartupOptions; }; temp?: TempDirectories; scenario?: Scenario; @@ -24,6 +25,7 @@ export abstract class ExpansionProvider { if (config.runner) { switch (config.runner.kind) { case "tsc": + case "api": providers.add(new CompilerOptionsExpansionProvider(config.runner.options)); break; case "tsserver": diff --git a/ts-perf/packages/api/src/model/sample.ts b/ts-perf/packages/api/src/model/sample.ts index e116d3a6..ca24dff0 100644 --- a/ts-perf/packages/api/src/model/sample.ts +++ b/ts-perf/packages/api/src/model/sample.ts @@ -136,11 +136,37 @@ export function isStartupSampleKey(key: string): key is StartupSampleKey { export type StartupSample = { [K in StartupSampleKey]?: number; }; +export const apiTimingMetrics = { + apiClientInstances: { metric: "API client instances", unit: " clients", precision: 0 }, + apiRequests: { metric: "API client requests", unit: " requests", precision: 0 }, + apiRoundTripTime: { metric: "API client round trip (sum)", unit: "ms", precision: 3 }, + apiBytesSent: { metric: "API client bytes sent", unit: "B", precision: 0 }, + apiBytesReceived: { metric: "API client bytes received", unit: "B", precision: 0 }, + apiNodesMaterialized: { metric: "API client nodes materialized", unit: " nodes", precision: 0 }, + apiSourceFilesFetched: { metric: "API client source files fetched", unit: " files", precision: 0 }, + apiNodesFetched: { metric: "API client nodes fetched", unit: " nodes", precision: 0 }, + apiServerRequests: { metric: "API server-counter requests", unit: " requests", precision: 0 }, + apiServerProcessingTime: { metric: "API server-counter processing (sum)", unit: "ms", precision: 3 }, + apiTransportResidualTime: { + metric: "API client round trip minus server processing (not pure IPC)", + unit: "ms", + precision: 3, + }, +} as const; + +export type ApiTimingSampleKey = keyof typeof apiTimingMetrics; +export const apiTimingSampleKeys = Object.keys(apiTimingMetrics) as ApiTimingSampleKey[]; + +export function isApiTimingSampleKey(key: string): key is ApiTimingSampleKey { + return Object.hasOwn(apiTimingMetrics, key); +} + export type SampleKey = | CompilerSampleKey | TSServerSampleKey - | StartupSampleKey; + | StartupSampleKey + | ApiTimingSampleKey; export function isSampleKey(key: string): key is SampleKey { - return isCompilerSampleKey(key) || isTSServerSampleKey(key) || isStartupSampleKey(key); + return isCompilerSampleKey(key) || isTSServerSampleKey(key) || isStartupSampleKey(key) || isApiTimingSampleKey(key); } diff --git a/ts-perf/packages/api/src/model/scenario.ts b/ts-perf/packages/api/src/model/scenario.ts index bebaf2ff..b659ed26 100644 --- a/ts-perf/packages/api/src/model/scenario.ts +++ b/ts-perf/packages/api/src/model/scenario.ts @@ -9,7 +9,7 @@ import { TSServerConfig } from "./tsserverconfig"; const cachedScenarios = new HashMap(PathComparer.fileSystem); -export type ScenarioKind = "tsserver" | "tsc" | "startup" | "lsp"; +export type ScenarioKind = "tsserver" | "tsc" | "startup" | "lsp" | "api"; export interface TscConfig { usePublicApi?: boolean; diff --git a/ts-perf/packages/commands/src/benchmark/api.ts b/ts-perf/packages/commands/src/benchmark/api.ts new file mode 100644 index 00000000..5b5858da --- /dev/null +++ b/ts-perf/packages/commands/src/benchmark/api.ts @@ -0,0 +1,17 @@ +import { Command, CommandMap } from "@ts-perf/api"; + +import { benchmark } from "."; + +const command: Command = { + commandName: "api", + summary: "Benchmark external TypeScript 7 API consumers.", + description: "Benchmark pinned external consumers using the matching native compiler and JS API.", + include: ["startup"], + lock: true, + update: true, + exec: ({ options }, host) => benchmark({ kind: "api", options }, host), +}; + +export function registerCommands(commands: CommandMap) { + commands.api = command; +} diff --git a/ts-perf/packages/commands/src/benchmark/apiTiming.test.ts b/ts-perf/packages/commands/src/benchmark/apiTiming.test.ts new file mode 100644 index 00000000..434a948c --- /dev/null +++ b/ts-perf/packages/commands/src/benchmark/apiTiming.test.ts @@ -0,0 +1,107 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { expect, test } from "vitest"; + +import { readApiTimingDumps } from "./apiTiming"; + +const dump = { + schemaVersion: 1, + timing: { + enabled: true, + totals: { + requestCount: 3, + roundTripMs: 10, + bytesSent: 100, + bytesReceived: 200, + nodesMaterialized: 2, + sourceFilesFetched: 1, + nodesFetched: 4, + serverTimeMs: 6, + transportOverheadMs: 4, + }, + }, + serverTiming: { enabled: true, totals: { requestCount: 3, totalProcessingTimeMs: 6 } }, +}; + +async function withDirectory(callback: (directory: string) => Promise) { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "tsperf-timing-test-")); + try { + await callback(directory); + } + finally { + await fs.rm(directory, { recursive: true, force: true }); + } +} + +test("sums all client/server counters without double-counting server processing", async () => { + await withDirectory(async directory => { + await fs.writeFile(path.join(directory, "timing-one.json"), JSON.stringify(dump)); + await fs.writeFile(path.join(directory, "timing-two.json"), JSON.stringify(dump)); + await fs.writeFile(path.join(directory, "unrelated.txt"), ""); + expect(await readApiTimingDumps(directory)).toEqual({ + apiClientInstances: 2, + apiRequests: 6, + apiRoundTripTime: 20, + apiBytesSent: 200, + apiBytesReceived: 400, + apiNodesMaterialized: 4, + apiSourceFilesFetched: 2, + apiNodesFetched: 8, + apiServerRequests: 6, + apiServerProcessingTime: 12, + apiTransportResidualTime: 8, + }); + }); +}); + +test("requires timing files and rejects malformed JSON", async () => { + await withDirectory(async directory => { + await expect(readApiTimingDumps(directory)).rejects.toThrow("Missing API timing dumps"); + await expect(readApiTimingDumps(path.join(directory, "absent"))).rejects.toThrow(); + await fs.writeFile(path.join(directory, "timing-broken.json"), "{"); + await expect(readApiTimingDumps(directory)).rejects.toThrow(); + }); +}); + +test.each([ + { ...dump, schemaVersion: 2 }, + { ...dump, timing: undefined }, + { ...dump, timing: { ...dump.timing, enabled: false } }, + { ...dump, timing: { ...dump.timing, totals: undefined } }, + { ...dump, serverTiming: null }, + { ...dump, serverTiming: { enabled: false, totals: dump.serverTiming.totals } }, + { ...dump, serverTiming: { enabled: true, totals: {} } }, +])("rejects missing or unsupported timing totals: %j", async value => { + await withDirectory(async directory => { + await fs.writeFile(path.join(directory, "timing-one.json"), JSON.stringify(value)); + await expect(readApiTimingDumps(directory)).rejects.toThrow(); + }); +}); + +test.each([undefined, null, -1, Infinity])("rejects invalid counters rather than reporting zeros: %s", async value => { + await withDirectory(async directory => { + await fs.writeFile( + path.join(directory, "timing-one.json"), + JSON.stringify({ + ...dump, + timing: { ...dump.timing, totals: { ...dump.timing.totals, bytesSent: value } }, + }), + ); + await expect(readApiTimingDumps(directory)).rejects.toThrow("counter"); + }); +}); + +test("rejects zero-work telemetry", async () => { + await withDirectory(async directory => { + await fs.writeFile( + path.join(directory, "timing-one.json"), + JSON.stringify({ + ...dump, + timing: { ...dump.timing, totals: { ...dump.timing.totals, requestCount: 0 } }, + }), + ); + await expect(readApiTimingDumps(directory)).rejects.toThrow("no requests"); + }); +}); diff --git a/ts-perf/packages/commands/src/benchmark/apiTiming.ts b/ts-perf/packages/commands/src/benchmark/apiTiming.ts new file mode 100644 index 00000000..667a2abe --- /dev/null +++ b/ts-perf/packages/commands/src/benchmark/apiTiming.ts @@ -0,0 +1,54 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import path from "node:path"; + +import { ApiTimingSampleKey, apiTimingSampleKeys } from "@ts-perf/api"; + +export async function readApiTimingDumps(directory: string) { + const files = (await fs.readdir(directory)).filter(name => name.startsWith("timing-") && name.endsWith(".json")); + assert( + files.length, + "Missing API timing dumps. The compiler must support TS_API_TIMING_DIR and clients must close.", + ); + const values: Record = { + apiClientInstances: files.length, + apiRequests: 0, + apiRoundTripTime: 0, + apiBytesSent: 0, + apiBytesReceived: 0, + apiNodesMaterialized: 0, + apiSourceFilesFetched: 0, + apiNodesFetched: 0, + apiServerRequests: 0, + apiServerProcessingTime: 0, + apiTransportResidualTime: 0, + }; + for (const file of files) { + const dump = JSON.parse(await fs.readFile(path.join(directory, file), "utf8")); + assert.equal(dump.schemaVersion, 1, "Unsupported API timing schema"); + assert(dump.timing?.enabled && dump.serverTiming?.enabled, "Missing client or server timing collection"); + const client = dump.timing.totals; + const server = dump.serverTiming.totals; + assert(client && server, "Missing API timing totals"); + const sample: Record = { + apiClientInstances: 0, + apiRequests: client.requestCount, + apiRoundTripTime: client.roundTripMs, + apiBytesSent: client.bytesSent, + apiBytesReceived: client.bytesReceived, + apiNodesMaterialized: client.nodesMaterialized, + apiSourceFilesFetched: client.sourceFilesFetched, + apiNodesFetched: client.nodesFetched, + apiServerRequests: server.requestCount, + apiServerProcessingTime: server.totalProcessingTimeMs, + apiTransportResidualTime: client.transportOverheadMs, + }; + for (const key of apiTimingSampleKeys) { + assert(Number.isFinite(sample[key]) && sample[key] >= 0, `Missing or invalid API timing counter: ${key}`); + values[key] += sample[key]; + assert(Number.isFinite(values[key]), `API timing counter overflow: ${key}`); + } + } + assert(values.apiRequests > 0, "API timing dumps contain no requests"); + return values; +} diff --git a/ts-perf/packages/commands/src/benchmark/index.ts b/ts-perf/packages/commands/src/benchmark/index.ts index 6092d34e..847d4314 100644 --- a/ts-perf/packages/commands/src/benchmark/index.ts +++ b/ts-perf/packages/commands/src/benchmark/index.ts @@ -15,6 +15,7 @@ import { import { HostContext, LogEventType, ProcessExitError } from "@ts-perf/core"; import { CommandLineParseError } from "power-options"; +import * as api from "./api"; import * as lsp from "./lsp"; import { measureAndRunScenarios } from "./measure"; import { printBenchmark, printComparison } from "./print"; @@ -60,6 +61,7 @@ export type TSOptions = | { kind: "tsserver"; options: BenchmarkOptions & TSServerOptions; } | { kind: "lsp"; options: BenchmarkOptions & TSServerOptions; } | { kind: "tsc"; options: BenchmarkOptions & CompilerOptions; } + | { kind: "api"; options: BenchmarkOptions & CompilerOptions; } | { kind: "startup"; options: BenchmarkOptions & StartupOptions; }; export async function benchmark(tsoptions: TSOptions, host: HostContext) { @@ -395,6 +397,7 @@ tsc.registerCommands(command.commands); tsserver.registerCommands(command.commands); lsp.registerCommands(command.commands); startup.registerCommands(command.commands); +api.registerCommands(command.commands); export function registerCommands(commands: CommandMap) { commands.benchmark = command; diff --git a/ts-perf/packages/commands/src/benchmark/measure.ts b/ts-perf/packages/commands/src/benchmark/measure.ts index 3370cb02..9bbc7666 100644 --- a/ts-perf/packages/commands/src/benchmark/measure.ts +++ b/ts-perf/packages/commands/src/benchmark/measure.ts @@ -34,6 +34,7 @@ import { getTempDirectories, HostContext, ProcessExitError, SystemInfo } from "@ import { BenchmarkOptions, TSOptions } from "./"; import { resolveBuiltPath } from "./resolveBuiltPath"; +import { runApiScenario } from "./runApiScenario"; const diagnosticPattern = /^([a-z].+):\s+(.+?)[sk]?$/i; // Explicitly loose regex so --pretty will work. @@ -75,6 +76,9 @@ export async function measureAndRunScenarios({ kind, options }: TSOptions, host: throw new ProcessExitError(-1, "Scenario not found."); } const hosts = await Host.findHosts(options.hosts); + if (kind === "api" && hosts.length === 0) { + throw new Error("No matching Node host found for API benchmarks. Install the requested host before running."); + } host.info(`Scenarios: ${scenarios.map(x => x.name).join(", ")}`); host.info(`Hosts: ${hosts.map(x => formatTestHost(x)).join("; ")}`); const numHosts = hosts.length; @@ -93,6 +97,17 @@ export async function measureAndRunScenarios({ kind, options }: TSOptions, host: ); let measurement; switch (kind) { + case "api": + measurement = await runApiScenario( + scenario, + testHost, + options, + host, + scenarioIndex, + hostSpecifier, + hostIndex, + ); + break; case "tsserver": measurement = await runTSServerScenario( name, @@ -158,6 +173,7 @@ export async function measureAndRunScenarios({ kind, options }: TSOptions, host: measurements, options.cpus, options.predictable, + options.warmups, ); } diff --git a/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts b/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts new file mode 100644 index 00000000..5314117d --- /dev/null +++ b/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts @@ -0,0 +1,228 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { apiTimingSampleKeys, CommonExpansionProvider, Host, HostSpecifier, Measurement, Scenario } from "@ts-perf/api"; +import { HostContext, Logger } from "@ts-perf/core"; +import { expect, test, vi } from "vitest"; + +import { measureAndRunScenarios } from "./measure"; +import { runApiProcess, runApiScenario } from "./runApiScenario"; + +const commit = "a".repeat(40); +const context = new HostContext(new Logger(0)); + +test("propagates command crashes and missing executables", async () => { + const args = ["-e", "console.error('consumer-crash'); process.exit(1)"]; + await expect(runApiProcess(process.execPath, args)).rejects.toThrow(/1.*consumer-crash/s); + await expect(runApiProcess(process.execPath, ["-e", "process.exit(2)"])) + .rejects.toThrow("failed (2)"); + await expect(runApiProcess("/nonexistent/tsperf-node", [])).rejects.toThrow(); +}); + +async function fixture(callback: (root: string, scenario: Scenario) => Promise) { + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "tsperf-api-test-"))); + try { + await fs.writeFile(path.join(root, "info.json"), JSON.stringify({ commit })); + await fs.mkdir(path.join(root, "test-api")); + await fs.writeFile( + path.join(root, "runner.mjs"), + ` + import assert from "node:assert/strict"; + import fs from "node:fs/promises"; + import path from "node:path"; + import { randomUUID } from "node:crypto"; + import { execFileSync } from "node:child_process"; + import { fileURLToPath } from "node:url"; + const args = process.argv.slice(2); + if (args.some(arg => ["--prepare", "--check", "--builtDir", "--workDir"].includes(arg))) { + throw Error("unexpected injected argument"); + } + const root = path.dirname(fileURLToPath(import.meta.url)); + assert.deepEqual(args.slice(0, 3), [root, root, root]); + assert.equal(process.cwd(), path.join(root, "test-api")); + await new Promise(resolve => setTimeout(resolve, Number(args.at(-1)))); + if (args.includes("--fail-check")) throw Error("invalid output"); + if (args.includes("--verify-node")) execFileSync("node", ["-e", ""], { stdio: "inherit" }); + if (!args.includes("--missing-dumps")) { + const instanceId = randomUUID(); + const dump = { + schemaVersion: 1, instanceId, pid: process.pid, mode: "sync", transport: "spawn", + startedAt: new Date().toISOString(), closedAt: new Date().toISOString(), + timing: { + enabled: true, + totals: { + requestCount: 1, roundTripMs: 1, bytesSent: 1, bytesReceived: 1, + serverTimeMs: 0, transportOverheadMs: 1, + nodesMaterialized: 1, sourceFilesFetched: 1, nodesFetched: 1, + }, + recentRequests: [], + }, + serverTiming: { enabled: true, totals: { requestCount: 1, totalProcessingTimeMs: 0 } }, + }; + await fs.writeFile( + path.join(process.env.TS_API_TIMING_DIR, "timing-"+process.pid+"-"+instanceId+".json"), + JSON.stringify(dump), + ); + } + `, + ); + await callback( + root, + new Scenario("test-api", "api", path.join(root, "scenario.json"), [ + "${scenarioDirectory}/runner.mjs", + "${typescriptDirectory}", + "${suiteDirectory}", + "${scenarioDirectory}", + "5", + ]), + ); + } + finally { + await fs.rm(root, { recursive: true, force: true }); + } +} + +test("times the whole script with declared arguments, standard expansions and the solution working directory", async () => { + await fixture(async (root, scenario) => { + const run = (delay: number) => + runApiScenario( + scenario.with({ args: [...scenario.args!.slice(0, -1), String(delay)] }), + Host.current, + { + builtDir: path.relative(process.cwd(), root), + suiteDir: path.relative(process.cwd(), root), + iterations: 2, + warmups: 1, + }, + context, + 0, + HostSpecifier.create(Host.current), + 0, + ); + const slow = await run(120); + expect((await fs.stat(path.join(root, scenario.name))).isDirectory()).toBe(true); + expect(slow.values!.totalTime!.samples.every(sample => sample >= 0.12)).toBe(true); + const restored = Measurement.create(JSON.parse(JSON.stringify(slow))); + expect(restored.values!.totalTime!.samples).toHaveLength(2); + for (const key of apiTimingSampleKeys) { + expect(restored.values?.[key]?.samples).toEqual(slow.values?.[key]?.samples); + expect(restored.values?.[key]?.unit).toBe(slow.values?.[key]?.unit); + expect(restored.values?.[key]?.metric).toBe(slow.values?.[key]?.metric); + } + expect(restored.diff(slow)?.values.apiBytesSent?.benchmarkRelativeDelta).toBe(0); + expect(restored.merge(slow).values?.apiRoundTripTime?.samples).toEqual([1, 1, 1, 1]); + }); +}); + +test("expands the installed host path for both the wrapper and consumer Node subprocesses", async () => { + await fixture(async (root, scenario) => { + const bin = path.join(root, "bin"); + const executable = path.join(bin, "node"); + const log = path.join(root, "node-invocations"); + await fs.mkdir(bin); + await fs.writeFile( + executable, + `#!${process.execPath} + const fs = require("node:fs"); + const { spawnSync } = require("node:child_process"); + fs.appendFileSync(${JSON.stringify(log)}, "node\\n"); + const result = spawnSync(${JSON.stringify(process.execPath)}, process.argv.slice(2), { stdio: "inherit" }); + if (result.error) throw result.error; + process.exit(result.status ?? 1); + `, + ); + await fs.chmod(executable, 0o755); + const host = new Host("node", { + executableFile: "${hostsDirectory}/bin/node", + }); + const expansion = vi.spyOn(CommonExpansionProvider.prototype, "expand") + .mockImplementation(name => name === "hostsdirectory" ? root : undefined); + try { + await runApiScenario( + scenario.with({ args: [...scenario.args!.slice(0, -1), "--verify-node", "5"] }), + host, + { builtDir: root, suiteDir: root, iterations: 1 }, + context, + 0, + HostSpecifier.create(host), + 0, + ); + } + finally { + expansion.mockRestore(); + } + expect(await fs.readFile(log, "utf8")).toBe("node\nnode\n"); + }); +}); + +test("rejects predictable mode before starting an API consumer", async () => { + await fixture(async (root, scenario) => { + await expect(runApiScenario( + scenario, + Host.current, + { builtDir: root, suiteDir: root, iterations: 1, predictable: true }, + context, + 0, + HostSpecifier.create(Host.current), + 0, + )).rejects.toThrow("do not support --predictable"); + }); +}); + +test("requires the solution prepared by setup.sh rather than creating it during measurement", async () => { + await fixture(async (root, scenario) => { + await fs.rmdir(path.join(root, scenario.name)); + await expect(runApiScenario( + scenario, + Host.current, + { builtDir: root, suiteDir: root, iterations: 1 }, + context, + 0, + HostSpecifier.create(Host.current), + 0, + )).rejects.toThrow(`Run ${path.join(root, "setup.sh")} before benchmarking.`); + await expect(fs.access(path.join(root, scenario.name))).rejects.toMatchObject({ code: "ENOENT" }); + }); +}); + +test("failed output checks fail the benchmark without a result protocol", async () => { + await fixture(async (root, scenario) => { + await expect(runApiScenario( + scenario.with({ args: [...scenario.args!.slice(0, -1), "--fail-check", "5"] }), + Host.current, + { builtDir: root, suiteDir: root, iterations: 1 }, + context, + 0, + HostSpecifier.create(Host.current), + 0, + )).rejects.toThrow("invalid output"); + }); +}); + +test("preserves warmup counts and fails rather than omitting missing dumps", async () => { + await fixture(async (root, scenario) => { + const scenarios = vi.spyOn(Scenario, "findScenarios").mockResolvedValue([scenario]); + const hosts = vi.spyOn(Host, "findHosts").mockResolvedValue([Host.current]); + try { + const report = await measureAndRunScenarios({ + kind: "api", + options: { builtDir: root, suiteDir: root, iterations: 1, warmups: 2 }, + }, context); + expect(JSON.parse(JSON.stringify(report)).warmups).toBe(2); + expect(report.measurements[0].values!.totalTime!.samples).toHaveLength(1); + expect(report.measurements[0].values!.apiRequests!.samples).toEqual([1]); + scenarios.mockResolvedValue([scenario.with({ + args: [...scenario.args!.slice(0, -1), "--missing-dumps", "5"], + })]); + await expect(measureAndRunScenarios({ + kind: "api", + options: { builtDir: root, suiteDir: root, iterations: 1 }, + }, context)).rejects.toThrow("Missing API timing dumps"); + } + finally { + scenarios.mockRestore(); + hosts.mockRestore(); + } + }); +}); diff --git a/ts-perf/packages/commands/src/benchmark/runApiScenario.ts b/ts-perf/packages/commands/src/benchmark/runApiScenario.ts new file mode 100644 index 00000000..c2999598 --- /dev/null +++ b/ts-perf/packages/commands/src/benchmark/runApiScenario.ts @@ -0,0 +1,135 @@ +import { spawn } from "node:child_process"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { performance } from "node:perf_hooks"; +import * as readline from "node:readline"; + +import { + apiTimingMetrics, + apiTimingSampleKeys, + CommandLineArgumentsBuilder, + CompilerOptions, + computeMetrics, + ExpansionProvider, + Host, + HostSpecifier, + Measurement, + Scenario, + Value, +} from "@ts-perf/api"; +import { HostContext } from "@ts-perf/core"; + +import { BenchmarkOptions } from "."; +import { readApiTimingDumps } from "./apiTiming"; + +export function runApiProcess( + command: string, + args: string[], + env = process.env, + cwd?: string, + context?: HostContext, +): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { env, cwd, stdio: ["ignore", "pipe", "pipe"] }); + let output = ""; + readline.createInterface({ input: child.stdout }).on("line", line => { + output += line + "\n"; + context?.trace(`> ${line}`); + }); + readline.createInterface({ input: child.stderr }).on("line", line => { + output += line + "\n"; + context?.error(`>> ${line}`); + }); + child.once("error", reject); + child.once("close", (code, signal) => { + if (code !== 0) reject(new Error(`API consumer failed (${signal ?? code}):\n${output}`)); + else resolve(); + }); + }); +} + +export async function runApiScenario( + scenario: Scenario, + host: Host, + options: BenchmarkOptions & CompilerOptions, + context: HostContext, + scenarioIndex: number, + hostSpecifier: HostSpecifier, + hostIndex: number, +): Promise { + if (options.predictable) { + throw new Error("API benchmarks do not support --predictable for external consumer processes."); + } + if (host.name !== "node" || !host.executableFile) { + throw new Error("API benchmarks require an installed Node host, not the native host."); + } + if (!scenario.args?.length) throw new Error(`API scenario ${scenario.name} has no runner.`); + const workDir = path.resolve(options.suiteDir, scenario.name); + await fs.access(workDir).catch(error => { + if (error.code !== "ENOENT") throw error; + throw new Error(`Run ${path.join(scenario.configDirectory, "setup.sh")} before benchmarking.`); + }); + const expansion = ExpansionProvider.getProviders({ + runner: { + kind: "api", + options: { + ...options, + builtDir: path.resolve(options.builtDir), + suiteDir: path.resolve(options.suiteDir), + }, + }, + scenario, + host, + }); + const executable = expansion.expand(host.executableFile); + const run = new CommandLineArgumentsBuilder(expansion, host, false, options.cpus).add( + path.resolve(scenario.configDirectory, expansion.expand(scenario.args[0])), + ...scenario.args.slice(1), + ); + const samples: number[] = []; + const timingSamples: Awaited>[] = []; + const iterations = options.iterations ?? 5; + const warmups = options.warmups ?? 0; + const timingDir = await fs.mkdtemp(path.join(os.tmpdir(), "tsperf-api-")); + try { + for (let i = 0; i < iterations + warmups; i++) { + const dumpDir = await fs.mkdtemp(path.join(timingDir, "timing-")); + const before = performance.now(); + await runApiProcess( + run.cmd!, + run.args, + { + ...process.env, + PATH: `${path.dirname(executable)}${path.delimiter}${process.env.PATH}`, + TS_API_TIMING_DIR: dumpDir, + }, + workDir, + context, + ); + const elapsedMs = performance.now() - before; + const timing = await readApiTimingDumps(dumpDir); + context.info(`API ${scenario.name}: ${elapsedMs.toFixed(2)}ms${i < warmups ? " (warmup)" : ""}`); + if (i >= warmups) { + samples.push(elapsedMs / 1000); + timingSamples.push(timing); + } + } + const values: Partial> = { + totalTime: computeMetrics(samples, "Total time", "s", 4), + }; + for (const key of apiTimingSampleKeys) { + const definition = apiTimingMetrics[key]; + values[key] = computeMetrics( + timingSamples.map(sample => sample[key]), + definition.metric, + definition.unit, + definition.precision, + ); + } + return new Measurement(scenario.name, scenarioIndex, hostSpecifier, hostIndex, values); + } + finally { + await fs.rm(timingDir, { recursive: true, force: true }); + } +} From b88514053bda97245bac34ac0eff2430d5652a89 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Fri, 16 Oct 2026 16:34:04 -0700 Subject: [PATCH 3/5] Package matching native compiler and JS API artifacts Build and retain the TypeScript API and native platform packages for API jobs. Add benchmark-local unstable export aliases and verify package gitHead and executable resolution without a separate artifact manifest. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 958b45cd-e40a-46fc-9de9-3bfde150ace3 --- cases/api/artifact.mjs | 24 ++++ scripts/src/__tests__/apiArtifact.test.ts | 134 ++++++++++++++++++++++ scripts/src/apiArtifact.ts | 46 ++++++++ scripts/src/buildTypeScript.ts | 8 ++ 4 files changed, 212 insertions(+) create mode 100644 cases/api/artifact.mjs create mode 100644 scripts/src/__tests__/apiArtifact.test.ts create mode 100644 scripts/src/apiArtifact.ts diff --git a/cases/api/artifact.mjs b/cases/api/artifact.mjs new file mode 100644 index 00000000..0846bbe1 --- /dev/null +++ b/cases/api/artifact.mjs @@ -0,0 +1,24 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +export async function verifyArtifact(builtDir) { + const apiDir = path.resolve(builtDir, "api"); + const info = JSON.parse(await fs.readFile(path.join(builtDir, "info.json"), "utf8")); + assert.match(info.commit, /^[a-f0-9]{40}$/); + const packageDir = path.join(apiDir, "node_modules/typescript"); + const pkg = JSON.parse(await fs.readFile(path.join(packageDir, "package.json"), "utf8")); + assert.equal(pkg.gitHead, info.commit, "Wrong JS wrapper commit"); + const platformDir = path.join(apiDir, "node_modules/@typescript", `typescript-${process.platform}-${process.arch}`); + const native = JSON.parse(await fs.readFile(path.join(platformDir, "package.json"), "utf8")); + assert.equal(native.gitHead, info.commit, "Wrong native compiler commit"); + const { default: getExePath } = await import(pathToFileURL(path.join(packageDir, "lib/getExePath.js")).href); + assert.equal( + await fs.realpath(getExePath()), + await fs.realpath(path.join(platformDir, "lib", process.platform === "win32" ? "tsc.exe" : "tsc")), + "Registry binary fallback", + ); + await fs.access(getExePath(), fs.constants.X_OK); + return { commit: info.commit, packageDir, binary: getExePath() }; +} diff --git a/scripts/src/__tests__/apiArtifact.test.ts b/scripts/src/__tests__/apiArtifact.test.ts new file mode 100644 index 00000000..b779556b --- /dev/null +++ b/scripts/src/__tests__/apiArtifact.test.ts @@ -0,0 +1,134 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, expect, test } from "vitest"; + +import { preserveApiArtifact } from "../apiArtifact.js"; + +const temporaryDirectories: string[] = []; +const commit = "a".repeat(40); +const platformName = `typescript-${process.platform}-${process.arch}`; +const exe = `tsc${process.platform === "win32" ? ".exe" : ""}`; +const verifyModule = new URL("../../../cases/api/artifact.mjs", import.meta.url).href; + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map(directory => fs.rm(directory, { recursive: true, force: true })), + ); +}); + +async function fixture( + options?: { missingBinary?: boolean; wrongWrapper?: boolean; wrongBinary?: boolean; fallback?: boolean; }, +) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "tsperf-artifact-test-")); + temporaryDirectories.push(root); + const source = path.join(root, "source"); + const output = path.join(root, "output"); + const pkg = path.join(source, "built/npm/typescript"); + const platform = path.join(source, "built/npm", platformName); + for (const directory of ["lib", "dist/api/sync", "dist/api/async"]) { + await fs.mkdir(path.join(pkg, directory), { recursive: true }); + } + await fs.mkdir(path.join(platform, "lib"), { recursive: true }); + await fs.writeFile( + path.join(pkg, "package.json"), + JSON.stringify({ + name: "typescript", + type: "module", + gitHead: options?.wrongWrapper ? "b".repeat(40) : commit, + optionalDependencies: { [`@typescript/${platformName}`]: "7.1.0" }, + exports: { + "./sync": { default: "./dist/api/sync/api.js" }, + "./async": { default: "./dist/api/async/api.js" }, + "./ast/is": "./dist/ast/is.js", + }, + }), + ); + await fs.writeFile( + path.join(platform, "package.json"), + JSON.stringify({ + gitHead: options?.wrongBinary ? "b".repeat(40) : commit, + }), + ); + const binary = path.join(platform, "lib", exe); + if (!options?.missingBinary) { + await fs.writeFile(binary, "fake-native-binary\n"); + await fs.chmod(binary, 0o755); + } + await fs.writeFile(path.join(pkg, "dist/api/sync/api.js"), "export const sync = true;\n"); + await fs.writeFile(path.join(pkg, "dist/api/async/api.js"), "export const async = true;\n"); + await fs.writeFile( + path.join(pkg, "lib/getExePath.js"), + options?.fallback + ? "export default () => process.execPath;\n" + : `import { fileURLToPath } from "node:url"; export default () => fileURLToPath(new URL("../../@typescript/${platformName}/lib/${exe}", import.meta.url));\n`, + ); + return { + source, + output, + async preserve() { + await preserveApiArtifact(output, commit, source); + await fs.writeFile(path.join(output, "info.json"), JSON.stringify({ commit })); + }, + async verify() { + const { verifyArtifact } = await import(verifyModule); + return verifyArtifact(output); + }, + }; +} + +test("preserves matching wrapper/binary and adds local export aliases", async () => { + const f = await fixture(); + await f.preserve(); + const verified = await f.verify(); + expect(verified.commit).toBe(commit); + const pkg = JSON.parse(await fs.readFile(path.join(verified.packageDir, "package.json"), "utf8")); + expect(pkg.exports["./unstable/sync"]).toEqual(pkg.exports["./sync"]); + expect(pkg.exports["./unstable/ast/is"]).toEqual(pkg.exports["./ast/is"]); + expect(pkg.optionalDependencies).toBeUndefined(); + expect(await fs.readFile(verified.binary, "utf8")).toBe("fake-native-binary\n"); +}); + +test.each([{ wrongWrapper: true }, { wrongBinary: true }, { missingBinary: true }])( + "rejects mismatched or missing artifacts: %j", + async options => { + const f = await fixture(options); + await expect(f.preserve()).rejects.toThrow(); + }, +); + +test("rejects registry binary fallback", async () => { + const f = await fixture({ fallback: true }); + await f.preserve(); + await expect(f.verify()).rejects.toThrow("Registry binary fallback"); +}); + +test("rejects a missing packaged native binary", async () => { + const f = await fixture(); + await f.preserve(); + await fs.unlink(path.join(f.output, "api/node_modules/@typescript", platformName, "lib", exe)); + await expect(f.verify()).rejects.toThrow(); +}); + +test("rejects missing packaged artifacts", async () => { + const f = await fixture(); + await expect(f.verify()).rejects.toThrow(); +}); + +test("rejects different compiler identities", async () => { + const f = await fixture(); + await f.preserve(); + await fs.writeFile(path.join(f.output, "info.json"), JSON.stringify({ commit: "b".repeat(40) })); + await expect(f.verify()).rejects.toThrow("Wrong JS wrapper commit"); +}); + +test("rejects a different native package identity after packaging", async () => { + const f = await fixture(); + await f.preserve(); + await fs.writeFile( + path.join(f.output, "api/node_modules/@typescript", platformName, "package.json"), + JSON.stringify({ gitHead: "b".repeat(40) }), + ); + await expect(f.verify()).rejects.toThrow("Wrong native compiler commit"); +}); diff --git a/scripts/src/apiArtifact.ts b/scripts/src/apiArtifact.ts new file mode 100644 index 00000000..81fa63b5 --- /dev/null +++ b/scripts/src/apiArtifact.ts @@ -0,0 +1,46 @@ +import assert from "node:assert"; +import fs from "node:fs/promises"; +import path from "node:path"; + +export async function preserveApiArtifact( + outputDir: string, + commit: string, + sourceDir = ".", +) { + const platform = `${process.platform}-${process.arch}`; + const apiDir = path.join(outputDir, "api"); + const modules = path.join(apiDir, "node_modules"); + const packageDir = path.join(modules, "typescript"); + const platformName = `typescript-${platform}`; + await fs.cp(path.join(sourceDir, "built/npm/typescript"), packageDir, { recursive: true }); + await fs.cp(path.join(sourceDir, `built/npm/${platformName}`), path.join(modules, "@typescript", platformName), { + recursive: true, + }); + const packagePath = path.join(packageDir, "package.json"); + const pkg: { + name: string; + gitHead: string; + exports: Record; + optionalDependencies?: Record; + } = JSON.parse(await fs.readFile(packagePath, "utf8")); + assert.equal(pkg.name, "typescript", "API benchmarks require the current TypeScript package layout"); + assert.equal(pkg.gitHead, commit, "JS API package must match the compiler source commit"); + const platformPackage: { gitHead: string; } = JSON.parse( + await fs.readFile(path.join(modules, "@typescript", platformName, "package.json"), "utf8"), + ); + assert.equal(platformPackage.gitHead, commit, "Native package must match the compiler source commit"); + for (const [key, value] of Object.entries(pkg.exports)) { + if (key !== "." && key !== "./package.json" && !key.startsWith("./unstable/")) { + const alias = `./unstable/${key.slice(2)}`; + pkg.exports[alias] ??= value; + } + } + // Both packages are copied from this build; never allow an install to fetch a registry binary. + delete pkg.optionalDependencies; + await fs.writeFile(packagePath, JSON.stringify(pkg, undefined, 4) + "\n"); + await fs.access( + path.join(modules, "@typescript", platformName, "lib", process.platform === "win32" ? "tsc.exe" : "tsc"), + ); + await fs.access(path.join(packageDir, "dist/api/sync/api.js")); + await fs.access(path.join(packageDir, "dist/api/async/api.js")); +} diff --git a/scripts/src/buildTypeScript.ts b/scripts/src/buildTypeScript.ts index 34cdae9c..cddce257 100644 --- a/scripts/src/buildTypeScript.ts +++ b/scripts/src/buildTypeScript.ts @@ -5,6 +5,7 @@ import path from "node:path"; import minimist from "minimist"; import { Octokit } from "octokit"; +import { preserveApiArtifact } from "./apiArtifact.js"; import { detectTypeScriptImplementation } from "./typeScriptRepo.js"; import { $, $pipe, getNonEmptyEnv, parseBoolean, RepoInfo, retry, setOutputVariable } from "./utils.js"; @@ -24,6 +25,8 @@ assert(outputDir, "Expected output path as first argument"); const implementation = detectTypeScriptImplementation("."); assert(implementation, "Expected to be run from the TypeScript repo"); const isCorsa = implementation === "corsa"; +const buildApi = process.env.TSPERF_PROCESS_KINDS?.split(/\s+/).includes("api") ?? false; +assert(!buildApi || isCorsa, "The api preset requires native TypeScript 7"); await $`mkdir -p ${path.dirname(outputDir)}`; @@ -32,7 +35,12 @@ await retry(() => $`npm ci`); if (fs.existsSync("Herebyfile.mjs")) { if (isCorsa) { await $`npx hereby build`; + if (buildApi) { + await $`npx hereby typescript:build-main`; + await $`npx hereby typescript:build-platform --platform ${`${process.platform}-${process.arch}`}`; + } await $`mv built/local ${outputDir}`; + if (buildApi) await preserveApiArtifact(outputDir, commit); } else { await $`npx hereby lkg`; From 31aad7882e71a6843459df89a74409a5d8e74f17 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Fri, 16 Oct 2026 16:34:14 -0700 Subject: [PATCH 4/5] Add external API corpus and opt-in pipeline preset Pin native typescript-eslint floating/recommended workloads and the vscode-languageserver metamodel generator. Prepare them with existing sandboxed setup scripts, inject the built API package, and check upstream command output. Wire the api preset to the existing benchmark pipeline with a pinned Node host and document adding scenarios. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 958b45cd-e40a-46fc-9de9-3bfde150ace3 --- README.md | 10 ++++ cases/api/eslint-setup.mjs | 48 +++++++++++++++++++ cases/api/eslint-setup.sh | 23 +++++++++ cases/api/eslint.mjs | 47 ++++++++++++++++++ cases/api/lsp-metamodel.mjs | 23 +++++++++ cases/api/utils.mjs | 22 +++++++++ .../scenario.json | 5 ++ .../typescript-eslint-api-floating/setup.sh | 6 +++ .../scenario.json | 5 ++ .../setup.sh | 6 +++ .../scenario.json | 5 ++ .../setup.sh | 12 +++++ .../setupPipeline/api/compute.snap | 1 + .../setupPipeline/api/error.snap | 1 + .../setupPipeline/api/matrix.snap | 1 + .../setupPipeline/api/outputVariables.snap | 1 + .../setupPipeline/api/parameters.snap | 1 + .../setupPipeline/api_tsgo/compute.snap | 7 +++ .../setupPipeline/api_tsgo/error.snap | 1 + .../setupPipeline/api_tsgo/matrix.snap | 40 ++++++++++++++++ .../api_tsgo/outputVariables.snap | 23 +++++++++ .../setupPipeline/api_tsgo/parameters.snap | 11 +++++ scripts/src/__tests__/setupPipeline.test.ts | 43 +++++++++++++++++ scripts/src/runTsPerf.ts | 7 +++ scripts/src/setupPipeline.ts | 29 +++++++++-- 25 files changed, 374 insertions(+), 4 deletions(-) create mode 100644 cases/api/eslint-setup.mjs create mode 100644 cases/api/eslint-setup.sh create mode 100644 cases/api/eslint.mjs create mode 100644 cases/api/lsp-metamodel.mjs create mode 100644 cases/api/utils.mjs create mode 100644 cases/scenarios/typescript-eslint-api-floating/scenario.json create mode 100644 cases/scenarios/typescript-eslint-api-floating/setup.sh create mode 100644 cases/scenarios/typescript-eslint-api-recommended/scenario.json create mode 100644 cases/scenarios/typescript-eslint-api-recommended/setup.sh create mode 100644 cases/scenarios/vscode-languageserver-api-metamodel/scenario.json create mode 100644 cases/scenarios/vscode-languageserver-api-metamodel/setup.sh create mode 100644 scripts/src/__tests__/__file_snapshots__/setupPipeline/api/compute.snap create mode 100644 scripts/src/__tests__/__file_snapshots__/setupPipeline/api/error.snap create mode 100644 scripts/src/__tests__/__file_snapshots__/setupPipeline/api/matrix.snap create mode 100644 scripts/src/__tests__/__file_snapshots__/setupPipeline/api/outputVariables.snap create mode 100644 scripts/src/__tests__/__file_snapshots__/setupPipeline/api/parameters.snap create mode 100644 scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/compute.snap create mode 100644 scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/error.snap create mode 100644 scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/matrix.snap create mode 100644 scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/outputVariables.snap create mode 100644 scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/parameters.snap diff --git a/README.md b/README.md index d8047bb7..19ef36d6 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,16 @@ After these files are created, they must be added to `scenarioConfig` in `script > [!NOTE] > Benchmark capacity is limited; we may not accept all benchmarks or run them automatically in CI. +### Adding an API benchmark + +API benchmarks time an external program that uses the TypeScript 7 JS API. `cases/scenarios/vscode-languageserver-api-metamodel` is a good template. + +In `scenario.json`, set `"kind": "api"` and set `args` to a Node script (relative to the scenario directory) followed by its arguments. `${typescriptDirectory}` expands to the built compiler. The script runs in `cases/solutions/` and is timed from start to finish. It should point the program at the built `typescript` package, run it, and exit nonzero if the output is wrong. + +`ts-perf` sets `TS_API_TIMING_DIR`, and each API client writes timing data there when it is closed. A run that produces no timing data fails, so both compared compiler commits must include timing support (microsoft/TypeScript#64846). + +To run the benchmark in CI, add it to `allScenarios` in `scripts/src/setupPipeline.ts` with `kind: "api"` and `runIn: RunType.None`, then request the `api` preset. + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/cases/api/eslint-setup.mjs b/cases/api/eslint-setup.mjs new file mode 100644 index 00000000..0b32ebaf --- /dev/null +++ b/cases/api/eslint-setup.mjs @@ -0,0 +1,48 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const fileCount = 1024; + +const workDir = process.cwd(); +const rules = process.argv[2]; +assert(["floating", "recommended"].includes(rules)); +const performanceDir = path.join(workDir, "performance"); + +// Use the generator unchanged, with Prettier and ESLint fixed by the consumer's pnpm lockfile. +await fs.rm(path.join(performanceDir, "node_modules"), { force: true }); +await fs.symlink("../node_modules", path.join(performanceDir, "node_modules"), "junction"); +const { writeCaseFiles } = await import( + pathToFileURL(path.join(performanceDir, "src/creators/writeCaseFiles.ts")).href +); +const fixture = path.join(workDir, "fixture"); +await fs.mkdir(fixture, { recursive: true }); +await fs.writeFile(path.join(fixture, "package.json"), '{"type":"module","private":true}\n'); +await fs.rm(path.join(fixture, "node_modules"), { force: true }); +await fs.symlink("../node_modules", path.join(fixture, "node_modules"), "junction"); +const data = { + name: "api-lint", + files: fileCount, + layout: "even", + rules, + singleRun: false, + types: "native", +}; +await writeCaseFiles(data, fixture); +const packageFile = path.join(fixture, "package.json"); +const upstreamPackage = await import( + pathToFileURL(path.join(performanceDir, "src/creators/files/createPackageFile.ts")).href +); +await fs.writeFile( + packageFile, + JSON.stringify({ type: "module", scripts: upstreamPackage.createPackageFile(data).scripts }), +); +// ESLint's CLI does not clear parser caches; close its synchronous native client before process exit. +await fs.appendFile( + path.join(fixture, "eslint.config.js"), + '\nimport { clearCaches } from "@typescript-eslint/parser";\nprocess.once("exit", clearCaches);\n', +); +const files = (await fs.readdir(path.join(fixture, "src"), { recursive: true })) + .filter(file => file.endsWith(".ts")); +assert.equal(files.length, fileCount, "Generator produced an unexpected number of files"); diff --git a/cases/api/eslint-setup.sh b/cases/api/eslint-setup.sh new file mode 100644 index 00000000..f4a50c41 --- /dev/null +++ b/cases/api/eslint-setup.sh @@ -0,0 +1,23 @@ +clone_scenario https://github.com/JoshuaKGoldberg/typescript-eslint.git 30d55234e6d719d3386884c3eef5c3a6c9d255f3 + +run_sandboxed sh -c ' + set -ex + export NX_DAEMON=false NX_NO_CLOUD=true + PACKAGE_MANAGER=$(node -p "require(\"./package.json\").packageManager") + npx "$PACKAGE_MANAGER" --filter "typescript-eslint..." \ + --filter @typescript-eslint/ast-spec \ + --filter @typescript-eslint/rule-schema-to-typescript-types \ + --filter @typescript-eslint/rule-tester \ + --no-include-workspace-root install --frozen-lockfile --ignore-scripts \ + --package-import-method=copy --store-dir .pnpm-store + npx "$PACKAGE_MANAGER" exec nx build typescript-eslint --skip-nx-cache +' + +# Keep the generator checkout out of Nx's workspace discovery during the build. +git clone --filter=blob:none --no-checkout https://github.com/typescript-eslint/performance.git performance +git -C performance fetch origin 8b8a104c482f5d21ffeddf2d52a99b2eada22aa0 +git -C performance switch --detach FETCH_HEAD +# Leave disposable sources so clone_scenario's normal cleanup removes them on reruns. +rm -rf performance/.git +cp ../../api/eslint-setup.mjs api-setup.mjs +run_sandboxed node api-setup.mjs "$1" diff --git a/cases/api/eslint.mjs b/cases/api/eslint.mjs new file mode 100644 index 00000000..9837329c --- /dev/null +++ b/cases/api/eslint.mjs @@ -0,0 +1,47 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import path from "node:path"; + +import { verifyArtifact } from "./artifact.mjs"; +import { run, verifyResolution } from "./utils.mjs"; + +const workDir = process.cwd(); +const artifact = await verifyArtifact(process.argv[2]); +const checkoutDir = workDir; +// Workspace packages resolve peers locally; a fixture-level alias alone is insufficient. +for ( + const directory of [ + checkoutDir, + ...(await fs.readdir(path.join(checkoutDir, "packages"))) + .map(name => path.join(checkoutDir, "packages", name)), + ] +) { + const alias = path.join(directory, "node_modules/@typescript/native"); + const stat = await fs.lstat(alias).catch(error => { + if (error.code !== "ENOENT") throw error; + }); + if (!stat) continue; + assert(stat.isSymbolicLink(), "Refusing to replace a non-link native dependency"); + await fs.unlink(alias); + await fs.symlink(artifact.packageDir, alias, "junction"); +} +await verifyResolution(path.join(checkoutDir, "packages/typescript-estree"), artifact); +const fixture = await fs.realpath(path.join(workDir, "fixture")); +const output = path.join(fixture, "results.json"); +await fs.rm(output, { force: true }); +try { + run("npm", ["run", "lint", "--", "--format", "json", "--output-file", "results.json"], fixture); + assert.fail("Expected one lint finding"); +} +catch (error) { + if (error.status !== 1) throw error; +} +const results = JSON.parse(await fs.readFile(output, "utf8")); +assert.equal(results.length, 1024, "ESLint skipped fixture files"); +const messages = results.flatMap(result => { + assert.equal(result.fatalErrorCount, 0, "ESLint parser failure"); + return result.messages; +}); +assert.equal(messages.length, 1, "Unexpected lint findings"); +assert.equal(messages[0].ruleId, "@typescript-eslint/no-floating-promises"); +await fs.unlink(output); diff --git a/cases/api/lsp-metamodel.mjs b/cases/api/lsp-metamodel.mjs new file mode 100644 index 00000000..9a32f276 --- /dev/null +++ b/cases/api/lsp-metamodel.mjs @@ -0,0 +1,23 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import path from "node:path"; + +import { verifyArtifact } from "./artifact.mjs"; +import { run, verifyResolution } from "./utils.mjs"; + +const workDir = process.cwd(); +const artifact = await verifyArtifact(process.argv[2]); +const repo = workDir; +const alias = path.join(repo, "node_modules/@typescript/native"); +await fs.rm(alias, { recursive: true }); +await fs.symlink(artifact.packageDir, alias, "junction"); +await verifyResolution(path.join(repo, "tools"), artifact); +const output = path.join(repo, "protocol/metaModel.json"); +await fs.rm(output, { force: true }); +run("npm", ["run", "generate:metaModel"], repo); +assert.deepEqual( + JSON.parse(await fs.readFile(output, "utf8")), + JSON.parse(await fs.readFile(path.join(workDir, "expected.json"), "utf8")), + "Generated metamodel differs from the checked-in output", +); +await fs.unlink(output); diff --git a/cases/api/utils.mjs b/cases/api/utils.mjs new file mode 100644 index 00000000..5c34ef87 --- /dev/null +++ b/cases/api/utils.mjs @@ -0,0 +1,22 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import fs from "node:fs/promises"; +import { createRequire } from "node:module"; +import path from "node:path"; + +export function run(command, args, cwd) { + return execFileSync(command, args, { + cwd, + stdio: "inherit", + }); +} + +export async function verifyResolution(from, artifact) { + const actual = createRequire(path.join(from, "package.json")); + const expected = createRequire(path.join(artifact.packageDir, "package.json")); + assert.equal( + await fs.realpath(actual.resolve("@typescript/native/unstable/sync")), + await fs.realpath(expected.resolve("typescript/unstable/sync")), + "Consumer resolved a registry compiler", + ); +} diff --git a/cases/scenarios/typescript-eslint-api-floating/scenario.json b/cases/scenarios/typescript-eslint-api-floating/scenario.json new file mode 100644 index 00000000..f8604b1d --- /dev/null +++ b/cases/scenarios/typescript-eslint-api-floating/scenario.json @@ -0,0 +1,5 @@ +{ + "name": "typescript-eslint-api-floating", + "kind": "api", + "args": ["../../api/eslint.mjs", "${typescriptDirectory}"] +} diff --git a/cases/scenarios/typescript-eslint-api-floating/setup.sh b/cases/scenarios/typescript-eslint-api-floating/setup.sh new file mode 100644 index 00000000..621b355e --- /dev/null +++ b/cases/scenarios/typescript-eslint-api-floating/setup.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -exo pipefail +cd "${0%/*}" + +source ../../common.sh +source ../../api/eslint-setup.sh floating diff --git a/cases/scenarios/typescript-eslint-api-recommended/scenario.json b/cases/scenarios/typescript-eslint-api-recommended/scenario.json new file mode 100644 index 00000000..4523ddd4 --- /dev/null +++ b/cases/scenarios/typescript-eslint-api-recommended/scenario.json @@ -0,0 +1,5 @@ +{ + "name": "typescript-eslint-api-recommended", + "kind": "api", + "args": ["../../api/eslint.mjs", "${typescriptDirectory}"] +} diff --git a/cases/scenarios/typescript-eslint-api-recommended/setup.sh b/cases/scenarios/typescript-eslint-api-recommended/setup.sh new file mode 100644 index 00000000..e6faccf8 --- /dev/null +++ b/cases/scenarios/typescript-eslint-api-recommended/setup.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -exo pipefail +cd "${0%/*}" + +source ../../common.sh +source ../../api/eslint-setup.sh recommended diff --git a/cases/scenarios/vscode-languageserver-api-metamodel/scenario.json b/cases/scenarios/vscode-languageserver-api-metamodel/scenario.json new file mode 100644 index 00000000..b4a054e0 --- /dev/null +++ b/cases/scenarios/vscode-languageserver-api-metamodel/scenario.json @@ -0,0 +1,5 @@ +{ + "name": "vscode-languageserver-api-metamodel", + "kind": "api", + "args": ["../../api/lsp-metamodel.mjs", "${typescriptDirectory}"] +} diff --git a/cases/scenarios/vscode-languageserver-api-metamodel/setup.sh b/cases/scenarios/vscode-languageserver-api-metamodel/setup.sh new file mode 100644 index 00000000..8c4128ab --- /dev/null +++ b/cases/scenarios/vscode-languageserver-api-metamodel/setup.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -exo pipefail +cd "${0%/*}" + +source ../../common.sh +clone_scenario https://github.com/andrewbranch/vscode-languageserver-node.git fe89061e918472537a1d54132b17fe3cadd85741 +git show HEAD:protocol/metaModel.json > expected.json +run_sandboxed sh -c ' + set -ex + npm ci --ignore-scripts --no-audit --no-fund + node node_modules/typescript/bin/tsc -b tools/tsconfig.json types/tsconfig.json jsonrpc/tsconfig.json protocol/src/common/tsconfig.json +' diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/api/compute.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api/compute.snap new file mode 100644 index 00000000..66dc9051 --- /dev/null +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api/compute.snap @@ -0,0 +1 @@ +undefined \ No newline at end of file diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/api/error.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api/error.snap new file mode 100644 index 00000000..c31cc8ed --- /dev/null +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api/error.snap @@ -0,0 +1 @@ +[Error: The api preset requires the native TypeScript 7 compiler and its matching JS API.] \ No newline at end of file diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/api/matrix.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api/matrix.snap new file mode 100644 index 00000000..66dc9051 --- /dev/null +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api/matrix.snap @@ -0,0 +1 @@ +undefined \ No newline at end of file diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/api/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api/outputVariables.snap new file mode 100644 index 00000000..66dc9051 --- /dev/null +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api/outputVariables.snap @@ -0,0 +1 @@ +undefined \ No newline at end of file diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/api/parameters.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api/parameters.snap new file mode 100644 index 00000000..66dc9051 --- /dev/null +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api/parameters.snap @@ -0,0 +1 @@ +undefined \ No newline at end of file diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/compute.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/compute.snap new file mode 100644 index 00000000..06176bec --- /dev/null +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/compute.snap @@ -0,0 +1,7 @@ +{ + "parallel": "3m 40s", + "perAgent": { + "any": "11m", + }, + "total": "11m", +} \ No newline at end of file diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/error.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/error.snap new file mode 100644 index 00000000..66dc9051 --- /dev/null +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/error.snap @@ -0,0 +1 @@ +undefined \ No newline at end of file diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/matrix.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/matrix.snap new file mode 100644 index 00000000..0362e291 --- /dev/null +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/matrix.snap @@ -0,0 +1,40 @@ +{ + "any": { + "api_node_24_21_0_typescript_eslint_api_floating": { + "TSPERF_JOB_HOST": "node@24.21.0", + "TSPERF_JOB_ITERATIONS": 6, + "TSPERF_JOB_KIND": "api", + "TSPERF_JOB_NAME": "api_node_24_21_0_typescript_eslint_api_floating", + "TSPERF_JOB_SCENARIO": "typescript-eslint-api-floating", + "TSPERF_JOB_WARMUPS": 1, + }, + "api_node_24_21_0_typescript_eslint_api_recommended": { + "TSPERF_JOB_HOST": "node@24.21.0", + "TSPERF_JOB_ITERATIONS": 6, + "TSPERF_JOB_KIND": "api", + "TSPERF_JOB_NAME": "api_node_24_21_0_typescript_eslint_api_recommended", + "TSPERF_JOB_SCENARIO": "typescript-eslint-api-recommended", + "TSPERF_JOB_WARMUPS": 1, + }, + "api_node_24_21_0_vscode_languageserver_api_metamodel": { + "TSPERF_JOB_HOST": "node@24.21.0", + "TSPERF_JOB_ITERATIONS": 6, + "TSPERF_JOB_KIND": "api", + "TSPERF_JOB_NAME": "api_node_24_21_0_vscode_languageserver_api_metamodel", + "TSPERF_JOB_SCENARIO": "vscode-languageserver-api-metamodel", + "TSPERF_JOB_WARMUPS": 1, + }, + }, + "ts-perf1": {}, + "ts-perf10": {}, + "ts-perf11": {}, + "ts-perf12": {}, + "ts-perf2": {}, + "ts-perf3": {}, + "ts-perf4": {}, + "ts-perf5": {}, + "ts-perf6": {}, + "ts-perf7": {}, + "ts-perf8": {}, + "ts-perf9": {}, +} \ No newline at end of file diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/outputVariables.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/outputVariables.snap new file mode 100644 index 00000000..b1239892 --- /dev/null +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/outputVariables.snap @@ -0,0 +1,23 @@ +{ + "MATRIX_any": "{"api_node_24_21_0_typescript_eslint_api_floating":{"TSPERF_JOB_HOST":"node@24.21.0","TSPERF_JOB_ITERATIONS":6,"TSPERF_JOB_KIND":"api","TSPERF_JOB_NAME":"api_node_24_21_0_typescript_eslint_api_floating","TSPERF_JOB_SCENARIO":"typescript-eslint-api-floating","TSPERF_JOB_WARMUPS":1},"api_node_24_21_0_typescript_eslint_api_recommended":{"TSPERF_JOB_HOST":"node@24.21.0","TSPERF_JOB_ITERATIONS":6,"TSPERF_JOB_KIND":"api","TSPERF_JOB_NAME":"api_node_24_21_0_typescript_eslint_api_recommended","TSPERF_JOB_SCENARIO":"typescript-eslint-api-recommended","TSPERF_JOB_WARMUPS":1},"api_node_24_21_0_vscode_languageserver_api_metamodel":{"TSPERF_JOB_HOST":"node@24.21.0","TSPERF_JOB_ITERATIONS":6,"TSPERF_JOB_KIND":"api","TSPERF_JOB_NAME":"api_node_24_21_0_vscode_languageserver_api_metamodel","TSPERF_JOB_SCENARIO":"vscode-languageserver-api-metamodel","TSPERF_JOB_WARMUPS":1}}", + "MATRIX_ts_perf1": "{}", + "MATRIX_ts_perf10": "{}", + "MATRIX_ts_perf11": "{}", + "MATRIX_ts_perf12": "{}", + "MATRIX_ts_perf2": "{}", + "MATRIX_ts_perf3": "{}", + "MATRIX_ts_perf4": "{}", + "MATRIX_ts_perf5": "{}", + "MATRIX_ts_perf6": "{}", + "MATRIX_ts_perf7": "{}", + "MATRIX_ts_perf8": "{}", + "MATRIX_ts_perf9": "{}", + "TSPERF_BASELINE_COMMIT": "HEAD^1", + "TSPERF_BASELINE_NAME": "baseline", + "TSPERF_IS_COMPARISON": "true", + "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false", + "TSPERF_NEW_COMMIT": "HEAD", + "TSPERF_NEW_NAME": "pr", + "TSPERF_PREDICTABLE": "false", + "TSPERF_PROCESS_KINDS": "api", +} \ No newline at end of file diff --git a/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/parameters.snap b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/parameters.snap new file mode 100644 index 00000000..a38c6e77 --- /dev/null +++ b/scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/parameters.snap @@ -0,0 +1,11 @@ +{ + "baselineCommit": "HEAD^1", + "baselineName": "baseline", + "hosts": undefined, + "isComparison": true, + "isCustomCommitRange": false, + "newCommit": "HEAD", + "newName": "pr", + "predictable": undefined, + "preset": "api", +} \ No newline at end of file diff --git a/scripts/src/__tests__/setupPipeline.test.ts b/scripts/src/__tests__/setupPipeline.test.ts index 60644097..75e25e58 100644 --- a/scripts/src/__tests__/setupPipeline.test.ts +++ b/scripts/src/__tests__/setupPipeline.test.ts @@ -32,6 +32,49 @@ const inputs = [ "startup-only hosts=node@22.8.0", ]; +test("API preset is opt-in, uses pinned Node rather than native, and processes API reports", async () => { + const result = await setupPipeline({ + input: "api", + baselining: false, + isPr: true, + shouldLog: false, + gitParseRev: fakeGitRevParse, + implementation: "corsa", + }); + const jobs = Object.values(result.matrix.any); + expect(jobs.map(job => job.TSPERF_JOB_SCENARIO).sort()).toEqual([ + "typescript-eslint-api-floating", + "typescript-eslint-api-recommended", + "vscode-languageserver-api-metamodel", + ]); + expect(jobs.every(job => job.TSPERF_JOB_KIND === "api" && job.TSPERF_JOB_HOST === "node@24.21.0")).toBe(true); + expect(result.outputVariables.TSPERF_PROCESS_KINDS).toBe("api"); + expect(result.parameters.baselineCommit).toBe("HEAD^1"); + expect(result.parameters.newCommit).toBe("HEAD"); +}); + +test("API preset rejects an unpinned host", async () => { + await expect(setupPipeline({ + input: "api host=native", + baselining: false, + isPr: true, + shouldLog: false, + gitParseRev: fakeGitRevParse, + implementation: "corsa", + })).rejects.toThrow("pinned host"); +}); + +test("API preset rejects predictable mode", async () => { + await expect(setupPipeline({ + input: "api predictable", + baselining: false, + isPr: true, + shouldLog: false, + gitParseRev: fakeGitRevParse, + implementation: "corsa", + })).rejects.toThrow("does not support predictable mode"); +}); + async function fakeGitRevParse(query: string): Promise { switch (query) { case "release-5.3...release-5.4": diff --git a/scripts/src/runTsPerf.ts b/scripts/src/runTsPerf.ts index a24a7da4..0b4eff09 100644 --- a/scripts/src/runTsPerf.ts +++ b/scripts/src/runTsPerf.ts @@ -20,6 +20,7 @@ const commands: Record Promise) | undefined> = { "benchmark-lsp": benchmarkLsp, "benchmark-tsserver": benchmarkTsserver, "benchmark-startup": benchmarkStartup, + "benchmark-api": benchmarkApi, }; const fn = commands[subcommand]; @@ -157,3 +158,9 @@ async function benchmarkStartup() { await $`node ${tsperfExe} benchmark startup --builtDir ${builtDir} ${tsperfArgs}`; } + +async function benchmarkApi() { + const builtDir = checkNonEmpty(args.builtDir, "Expected non-empty --builtDir"); + const tsperfArgs = await getCommonBenchmarkArgs(); + await $`node ${tsperfExe} benchmark api --builtDir ${builtDir} ${tsperfArgs}`; +} diff --git a/scripts/src/setupPipeline.ts b/scripts/src/setupPipeline.ts index db6974c7..6df207f8 100644 --- a/scripts/src/setupPipeline.ts +++ b/scripts/src/setupPipeline.ts @@ -47,6 +47,7 @@ const defaultWarmups = 1; const hosts = { // This version is arbitrary (just what was latest on 2023-08-12). node20: "node@20.5.1", + node24: "node@24.21.0", // This matches a recent VS Code version via Electron. node18: "node@18.15.0", bun: "bun@1.1.24", @@ -54,7 +55,7 @@ const hosts = { native: "native", // Native Go binary; no host runtime needed. } as const satisfies Record; -const allJobKinds = ["tsc", "tsserver", "lsp", "startup"] as const; +const allJobKinds = ["tsc", "tsserver", "lsp", "startup", "api"] as const; type JobKind = typeof allJobKinds[number]; const enum RunType { @@ -79,6 +80,9 @@ interface BaseScenario { // DO NOT change the agents; they must remain the same forever to keep benchmarks comparable. const allScenarios: readonly BaseScenario[] = [ + { kind: "api", name: "vscode-languageserver-api-metamodel", agent: "ts-perf1", runIn: RunType.None, cost: 10 }, + { kind: "api", name: "typescript-eslint-api-floating", agent: "ts-perf3", runIn: RunType.None, cost: 10 }, + { kind: "api", name: "typescript-eslint-api-recommended", agent: "ts-perf4", runIn: RunType.None, cost: 10 }, { kind: "tsc", name: "angular-1", agent: "ts-perf1", runIn: RunType.Any, cost: 64 }, { kind: "tsc", name: "Compiler-Unions", agent: "ts-perf2", runIn: RunType.Any, cost: 17 }, { kind: "tsc", name: "vscode", agent: "ts-perf3", runIn: RunType.Any, cost: 107 }, @@ -222,6 +226,13 @@ function* generateBaselinePreset(scenarios: readonly BaseScenario[]): Iterable generateBaselinePreset(baselineScenarios), "full": () => generateBaselinePreset(onDemandScenarios), *"regular"() { @@ -320,7 +331,7 @@ function sanitizeJobName(name: string): JobName { } // This defines the sort order, which is seen in PR replies; tsc is the most important and should be first. -const kindOrder: readonly JobKind[] = ["tsc", "tsserver", "lsp", "startup"]; +const kindOrder: readonly JobKind[] = ["tsc", "tsserver", "lsp", "startup", "api"]; assert.deepStrictEqual([...allJobKinds].sort(), [...kindOrder].sort(), "kindOrder must contain all job kinds"); @@ -478,13 +489,14 @@ function* transformPreset( } } } - const scenarioHosts = isCorsa ? [hosts.native] : (parameters.hosts ?? [scenario.host]); + const scenarioHosts = isCorsa && scenario.kind !== "api" ? [hosts.native] + : (parameters.hosts ?? [scenario.host]); for (const host of scenarioHosts) { yield { ...scenario, host, - iterations: isCorsa ? corsaIterations : scenario.iterations, + iterations: isCorsa && scenario.kind !== "api" ? corsaIterations : scenario.iterations, }; } } @@ -511,6 +523,15 @@ export async function setupPipeline(input: SetupPipelineInput) { const { baselining, implementation, shouldLog } = input; const parameters = await parseInput(input); + if (parameters.preset === "api" && implementation !== "corsa") { + throw new Error("The api preset requires the native TypeScript 7 compiler and its matching JS API."); + } + if (parameters.preset === "api" && parameters.hosts?.some(host => host !== hosts.node24)) { + throw new Error(`The api preset requires its pinned host ${hosts.node24}.`); + } + if (parameters.preset === "api" && parameters.predictable) { + throw new Error("The api preset does not support predictable mode for external consumer processes."); + } if (shouldLog) { console.log("Parameters", parameters); } From 8d6f2c48cb8cb3b337c5d69ab7699c40321f1a95 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Mon, 19 Oct 2026 09:22:25 -0700 Subject: [PATCH 5/5] Fix clean-checkout tests and API sandbox invocation Resolve workspace packages to source in Vitest so tests do not require a preceding build. Invoke the ESLint fixture generator through sh -c, matching the sandbox helper's non-root execution path. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 958b45cd-e40a-46fc-9de9-3bfde150ace3 --- cases/api/eslint-setup.sh | 2 +- ts-perf/packages/api/vitest.config.mjs | 7 +++++++ ts-perf/packages/commands/vitest.config.mjs | 8 ++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/cases/api/eslint-setup.sh b/cases/api/eslint-setup.sh index f4a50c41..5fb1f35c 100644 --- a/cases/api/eslint-setup.sh +++ b/cases/api/eslint-setup.sh @@ -20,4 +20,4 @@ git -C performance switch --detach FETCH_HEAD # Leave disposable sources so clone_scenario's normal cleanup removes them on reruns. rm -rf performance/.git cp ../../api/eslint-setup.mjs api-setup.mjs -run_sandboxed node api-setup.mjs "$1" +run_sandboxed sh -c 'node api-setup.mjs "$1"' -- "$1" diff --git a/ts-perf/packages/api/vitest.config.mjs b/ts-perf/packages/api/vitest.config.mjs index 499930f8..47a7fdc8 100644 --- a/ts-perf/packages/api/vitest.config.mjs +++ b/ts-perf/packages/api/vitest.config.mjs @@ -1,6 +1,13 @@ +import { fileURLToPath } from "node:url"; + import { configDefaults, defineConfig } from "vitest/config"; export default defineConfig({ + resolve: { + alias: { + "@ts-perf/core": fileURLToPath(new URL("../core/src/index.ts", import.meta.url)), + }, + }, test: { exclude: [...configDefaults.exclude, "**/dist/**"], }, diff --git a/ts-perf/packages/commands/vitest.config.mjs b/ts-perf/packages/commands/vitest.config.mjs index 499930f8..a19bef78 100644 --- a/ts-perf/packages/commands/vitest.config.mjs +++ b/ts-perf/packages/commands/vitest.config.mjs @@ -1,6 +1,14 @@ +import { fileURLToPath } from "node:url"; + import { configDefaults, defineConfig } from "vitest/config"; export default defineConfig({ + resolve: { + alias: { + "@ts-perf/api": fileURLToPath(new URL("../api/src/index.ts", import.meta.url)), + "@ts-perf/core": fileURLToPath(new URL("../core/src/index.ts", import.meta.url)), + }, + }, test: { exclude: [...configDefaults.exclude, "**/dist/**"], },