From 63c105ebd809274a9b3e342bea3ec3a6f700879f Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:57:27 -0700 Subject: [PATCH 1/4] Stop passing an unsupported flag to the lsp-startup scenario tsgo's LSP mode does not accept --disableAutomaticTypingAcquisition and exits immediately with a usage error, so lsp-startup has been timing flag parsing rather than server startup. It also makes the scenario look like a crash once startup failures are detected. --- cases/scenarios/lsp-startup/scenario.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cases/scenarios/lsp-startup/scenario.json b/cases/scenarios/lsp-startup/scenario.json index 8462a42b..a9661807 100644 --- a/cases/scenarios/lsp-startup/scenario.json +++ b/cases/scenarios/lsp-startup/scenario.json @@ -4,7 +4,6 @@ "args": [ "tsgo", "--lsp", - "--stdio", - "--disableAutomaticTypingAcquisition" + "--stdio" ] } From 9b780387b5e36d0cb89bdbd35f7a429543dcb55a Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:57:27 -0700 Subject: [PATCH 2/4] Fail the server harnesses when the server exits unexpectedly If tsserver or the LSP server dies while a request is pending, the response promise never settles. Node then exits with status 0 once the event loop drains, so a crashed server is recorded as a successful run with partial or missing timings. --- ts-perf/packages/commands/src/benchmark/measurelsp.ts | 11 ++++++++++- .../commands/src/benchmark/measuretsserver.ts | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/ts-perf/packages/commands/src/benchmark/measurelsp.ts b/ts-perf/packages/commands/src/benchmark/measurelsp.ts index 4060f1dd..b86fa301 100644 --- a/ts-perf/packages/commands/src/benchmark/measurelsp.ts +++ b/ts-perf/packages/commands/src/benchmark/measurelsp.ts @@ -134,7 +134,15 @@ async function runPerf(options: CLIOpts) { connection.listen(); - serverProc.on("exit", code => console.log(code ? `Exited with code ${code}` : `Terminated`)); + let shuttingDown = false; + serverProc.on("exit", code => { + console.log(code ? `Exited with code ${code}` : `Terminated`); + // A crash can leave a request waiting forever for its response. + if (!shuttingDown) { + console.error("Server exited unexpectedly"); + process.exit(3); + } + }); // seq tracks request numbering; starts at 1 for initialize (analogous to configure in tsserver) let seq = 1; @@ -208,6 +216,7 @@ async function runPerf(options: CLIOpts) { } finally { // Shut down the LSP server + shuttingDown = true; try { await connection.sendRequest(protocol.ShutdownRequest.method); await connection.sendNotification(protocol.ExitNotification.method); diff --git a/ts-perf/packages/commands/src/benchmark/measuretsserver.ts b/ts-perf/packages/commands/src/benchmark/measuretsserver.ts index 9ff1fc34..cb39de60 100644 --- a/ts-perf/packages/commands/src/benchmark/measuretsserver.ts +++ b/ts-perf/packages/commands/src/benchmark/measuretsserver.ts @@ -110,8 +110,16 @@ async function runPerf(options: CLIOpts) { // Arguments to node execArgv, ); + let shuttingDown = false; try { - server.on("exit", code => console.log(code ? `Exited with code ${code}` : `Terminated`)); + server.on("exit", code => { + console.log(code ? `Exited with code ${code}` : `Terminated`); + // A crash can leave a request waiting forever for its response. + if (!shuttingDown) { + console.error("Server exited unexpectedly"); + process.exit(3); + } + }); if (options.cpus) { if (!server.pid) { @@ -182,6 +190,7 @@ async function runPerf(options: CLIOpts) { } finally { // Tell the server to shut down + shuttingDown = true; // eslint-disable-next-line no-useless-assignment await server.message({ seq: seq++, command: "exit" }); } From 5540b80204a6f0a3d464991a6ed2546a99c0a940 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:57:27 -0700 Subject: [PATCH 3/4] Represent failed measurements in benchmark results and reports A scenario that crashes the compiler under test has no samples, and an empty measurement currently makes the comparison step throw. The whole run then fails, and the PR gets a generic failure comment that hides the results of every other scenario. Record the failure explicitly so that results can still be merged, uploaded and compared. Reports show the scenario as failed in place of its metrics, so the crash is visible next to the other results. --- ts-perf/packages/api/src/format.ts | 17 ++++++++++ .../api/src/model/measurement.test.ts | 32 ++++++++++++++++++ ts-perf/packages/api/src/model/measurement.ts | 26 ++++++++++++++- .../api/src/model/measurementComparison.ts | 10 ++++++ .../src/model/measurementComparisonPivot.ts | 33 +++++++++++-------- .../api/src/model/measurementPivot.ts | 33 +++++++++++-------- .../src/benchmark/print/console/benchmark.ts | 18 +++++++--- .../src/benchmark/print/console/comparison.ts | 6 ++-- .../src/benchmark/print/html/benchmark.ts | 23 ++++++++++--- .../src/benchmark/print/html/comparison.ts | 12 +++++++ .../src/benchmark/print/markdown/benchmark.ts | 29 +++++++++++++--- .../benchmark/print/markdown/comparison.ts | 8 +++++ 12 files changed, 200 insertions(+), 47 deletions(-) diff --git a/ts-perf/packages/api/src/format.ts b/ts-perf/packages/api/src/format.ts index a9025d8d..daba55b9 100644 --- a/ts-perf/packages/api/src/format.ts +++ b/ts-perf/packages/api/src/format.ts @@ -145,44 +145,61 @@ export function formatTestHost(testHost: Host) { return name; } +export const failedMetricText = "⚠️ Failed"; + +/** Text for a side of a failed comparison: "failed" when that side failed, otherwise empty. */ +export function formatFailedSide(failed: boolean | undefined) { + return failed ? "failed" : ""; +} + export function formatComparisonMetric(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return failedMetricText; return comparison.metric || comparison.name; } export function formatComparisonBaseline(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return formatFailedSide(comparison.measurement.baseline?.failed); return formatMean(comparison.baseline!); } export function formatComparisonMidline(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return formatFailedSide(comparison.measurement.midline?.failed); return formatMean(comparison.midline!); } export function formatComparisonCurrent(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return formatFailedSide(comparison.measurement.benchmark.failed); return formatMean(comparison.benchmark); } export function formatComparisonBaselineDelta(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; return formatDelta(comparison.benchmark, comparison.baseline!, comparison.baselineRelativeIsSignificant); } export function formatComparisonMidlineDelta(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; return formatDelta(comparison.benchmark, comparison.midline!, comparison.midlineRelativeIsSignificant); } export function formatComparisonBaselinePValue(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; return formatPValue(comparison.baselineRelativePValue, comparison.benchmark, comparison.baseline!); } export function formatComparisonMidlinePValue(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; return formatPValue(comparison.midlineRelativePValue, comparison.benchmark, comparison.midline!); } export function formatComparisonBest(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; if (comparison.benchmark.allSamplesEqual) return "~"; return formatUnit(comparison.benchmark.minimum, comparison.benchmark); } export function formatComparisonWorst(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; if (comparison.benchmark.allSamplesEqual) return "~"; return formatUnit(comparison.benchmark.maximum, comparison.benchmark); } diff --git a/ts-perf/packages/api/src/model/measurement.test.ts b/ts-perf/packages/api/src/model/measurement.test.ts index a6295db5..465d1d0f 100644 --- a/ts-perf/packages/api/src/model/measurement.test.ts +++ b/ts-perf/packages/api/src/model/measurement.test.ts @@ -1,5 +1,6 @@ import { expect, test } from "vitest"; +import { formatComparisonBaseline, formatComparisonCurrent, formatComparisonMetric } from "../format"; import { Benchmark } from "./benchmark"; import { HostSpecifier } from "./hostSpecifier"; import { Measurement } from "./measurement"; @@ -48,3 +49,34 @@ test("scenario-only filtering remaps scenarioIndex and preserves hostIndex", () expect(filtered.measurements[0].hostIndex).toBe(1); expect(filtered.measurements[0].values?.totalTime?.samples).toEqual([1, 2]); }); + +const failed = Measurement.createFailed("test", 1, measurement.host!, 1); + +test("failed measurements survive serialization, filtering and merging", () => { + const restored = Measurement.create(JSON.parse(JSON.stringify(failed))); + expect(restored.failed).toBe(true); + expect(restored.values).toEqual({}); + expect(Measurement.create(JSON.parse(JSON.stringify(measurement))).failed).toBe(false); + + expect(report().with({ measurements: [failed] }).filter([], ["node@24.21.0"]).measurements[0].failed).toBe(true); + + expect(measurement.merge(failed).failed).toBe(true); + expect(failed.merge(measurement).failed).toBe(true); +}); + +test("comparisons with a failed side pivot to a single failed row", () => { + const passing = report(); + const crashed = passing.with({ measurements: [failed] }); + + for (const comparison of [crashed.diff(passing)!, passing.diff(crashed)!]) { + const pivots = comparison.measurements[0].pivot(); + expect(pivots).toHaveLength(1); + expect(pivots[0].failed).toBe(true); + } + + const [pivot] = crashed.diff(passing)!.measurements[0].pivot(); + expect(formatComparisonMetric(pivot)).toBe("⚠️ Failed"); + expect(formatComparisonBaseline(pivot)).toBe(""); + expect(formatComparisonCurrent(pivot)).toBe("failed"); + expect(failed.pivot().map(pivot => pivot.failed)).toEqual([true]); +}); diff --git a/ts-perf/packages/api/src/model/measurement.ts b/ts-perf/packages/api/src/model/measurement.ts index f085a268..c4219206 100644 --- a/ts-perf/packages/api/src/model/measurement.ts +++ b/ts-perf/packages/api/src/model/measurement.ts @@ -10,6 +10,7 @@ export interface MeasurementComponents extends Partial>, + /** The benchmarked process crashed or exited unsuccessfully, so no values were recorded. */ + public readonly failed = false, ) { } + public static createFailed( + scenarioName: string, + scenarioIndex: number, + host: HostSpecifier, + hostIndex: number, + ): Measurement { + return new Measurement(scenarioName, scenarioIndex, host, hostIndex, {}, /*failed*/ true); + } + public get name() { return `${this.scenarioName} - ${this.hostName}`; } @@ -53,7 +65,8 @@ export class Measurement { : undefined), components.host && HostSpecifier.create(components.host), components.hostIndex, - values, + components.failed ? {} : values, + !!components.failed, ); } @@ -63,6 +76,7 @@ export class Measurement { scenarioIndex: this.scenarioIndex, host: this.host, hostIndex: this.hostIndex, + failed: this.failed || undefined, ...this.values, }; } @@ -72,6 +86,7 @@ export class Measurement { const { scenarioIndex = this.scenarioIndex } = components; const { hostIndex = this.hostIndex } = components; const { host = this.host } = components; + const { failed = this.failed } = components; const values: Partial> = { ...this.values }; let equalValues = true; for (const key in components) { @@ -88,6 +103,7 @@ export class Measurement { && this.scenarioIndex === scenarioIndex && this.hostIndex === hostIndex && this.host === host + && this.failed === failed && equalValues ) { return this; @@ -97,6 +113,7 @@ export class Measurement { scenarioIndex, host, hostIndex, + failed, ...values, }); } @@ -110,6 +127,9 @@ export class Measurement { } public pivot() { + if (this.failed) { + return [new MeasurementPivot(this, /*value*/ undefined)]; + } const pivots: MeasurementPivot[] = []; for (const key in this.values) { if (hasProperty(this.values, key)) { @@ -132,6 +152,9 @@ export class Measurement { throw Error("Cannot merge measurements with different scenarios or hosts."); } + if (this.failed) return this; + if (other.failed) return other; + if (!this.values) return other; if (!other.values) return this; const thisKeys = Object.getOwnPropertyNames(this.values); @@ -160,6 +183,7 @@ export class Measurement { hostName: this.hostName, hostIndex: this.hostIndex, host: this.host, + failed: this.failed || undefined, ...this.values, }; } diff --git a/ts-perf/packages/api/src/model/measurementComparison.ts b/ts-perf/packages/api/src/model/measurementComparison.ts index 4fcc57e5..cabc74ea 100644 --- a/ts-perf/packages/api/src/model/measurementComparison.ts +++ b/ts-perf/packages/api/src/model/measurementComparison.ts @@ -23,6 +23,9 @@ export class MeasurementComparison { public readonly midline?: Measurement, ) { this.values = {}; + if (this.failed) { + return; + } for (const key in benchmark.values) { if (hasProperty(benchmark.values, key)) { this.values[key] = Value.diff(benchmark.values[key], baseline?.values?.[key], midline?.values?.[key]); @@ -30,6 +33,10 @@ export class MeasurementComparison { } } + public get failed() { + return this.benchmark.failed || !!this.baseline?.failed || !!this.midline?.failed; + } + public get name() { return `${this.scenarioName} - ${this.hostName}`; } @@ -82,6 +89,9 @@ export class MeasurementComparison { } public pivot() { + if (this.failed) { + return [new MeasurementComparisonPivot(this, /*value*/ undefined)]; + } const pivots: MeasurementComparisonPivot[] = []; for (const key in this.values) { if (hasProperty(this.values, key)) { diff --git a/ts-perf/packages/api/src/model/measurementComparisonPivot.ts b/ts-perf/packages/api/src/model/measurementComparisonPivot.ts index a65c43e2..4801ba7c 100644 --- a/ts-perf/packages/api/src/model/measurementComparisonPivot.ts +++ b/ts-perf/packages/api/src/model/measurementComparisonPivot.ts @@ -4,10 +4,15 @@ import { ValueComparison } from "./valueComparison"; export class MeasurementComparisonPivot { constructor( public readonly measurement: MeasurementComparison, - public readonly value: ValueComparison, + /** Undefined when the measurement failed; check {@link failed} before reading values. */ + public readonly value: ValueComparison | undefined, ) { } + public get failed() { + return this.value === undefined; + } + public get name() { return `${this.measurementName} - ${this.metric}`; } @@ -30,42 +35,42 @@ export class MeasurementComparisonPivot { return this.measurement.host; } public get metric() { - return this.value.metric; + return this.value?.metric ?? ""; } public get metricIndex() { - return this.value.metricIndex; + return this.value?.metricIndex ?? -1; } public get benchmark() { - return this.value.benchmark; + return this.value!.benchmark; } public get benchmarkRelativeDelta() { - return this.value.benchmarkRelativeDelta; + return this.value!.benchmarkRelativeDelta; } public get baseline() { - return this.value.baseline; + return this.value!.baseline; } public get baselineRelativeDelta() { - return this.value.baselineRelativeDelta; + return this.value!.baselineRelativeDelta; } public get baselineRelativePValue() { - return this.value.baselineRelativePValue; + return this.value!.baselineRelativePValue; } public get baselineRelativeIsSignificant() { - return this.value.baselineRelativeIsSignificant; + return this.value!.baselineRelativeIsSignificant; } public get midline() { - return this.value.midline; + return this.value!.midline; } public get midlineRelativeDelta() { - return this.value.midlineRelativeDelta; + return this.value!.midlineRelativeDelta; } public get midlineRelativePValue() { - return this.value.midlineRelativePValue; + return this.value!.midlineRelativePValue; } public get midlineRelativeIsSignificant() { - return this.value.midlineRelativeIsSignificant; + return this.value!.midlineRelativeIsSignificant; } public get worst() { - return this.value.worst; + return this.value!.worst; } } diff --git a/ts-perf/packages/api/src/model/measurementPivot.ts b/ts-perf/packages/api/src/model/measurementPivot.ts index cdb250c6..a19670b9 100644 --- a/ts-perf/packages/api/src/model/measurementPivot.ts +++ b/ts-perf/packages/api/src/model/measurementPivot.ts @@ -4,10 +4,15 @@ import { Value } from "./value"; export class MeasurementPivot { constructor( public readonly measurement: Measurement, - public readonly value: Value, + /** Undefined when the measurement failed; check {@link failed} before reading values. */ + public readonly value: Value | undefined, ) { } + public get failed() { + return this.value === undefined; + } + public get name() { return `${this.measurement.name} - ${this.metric}`; } @@ -27,42 +32,42 @@ export class MeasurementPivot { return this.measurement.hostIndex; } public get metric() { - return this.value.metric; + return this.value?.metric ?? ""; } public get metricIndex() { - return this.value.metricIndex; + return this.value?.metricIndex ?? -1; } public get samples() { - return this.value.samples; + return this.value!.samples; } public get mean() { - return this.value.mean; + return this.value!.mean; } public get minimum() { - return this.value.minimum; + return this.value!.minimum; } public get maximum() { - return this.value.maximum; + return this.value!.maximum; } public get marginOfError() { - return this.value.marginOfError; + return this.value!.marginOfError; } public get variance() { - return this.value.variance; + return this.value!.variance; } public get standardDeviation() { - return this.value.standardDeviation; + return this.value!.standardDeviation; } public get relativeMarginOfError() { - return this.value.relativeMarginOfError; + return this.value!.relativeMarginOfError; } public get unit() { - return this.value.unit; + return this.value!.unit; } public get precision() { - return this.value.precision; + return this.value!.precision; } public get allSamplesEqual() { - return this.value.allSamplesEqual; + return this.value!.allSamplesEqual; } } diff --git a/ts-perf/packages/commands/src/benchmark/print/console/benchmark.ts b/ts-perf/packages/commands/src/benchmark/print/console/benchmark.ts index 0cd63825..5a670b38 100644 --- a/ts-perf/packages/commands/src/benchmark/print/console/benchmark.ts +++ b/ts-perf/packages/commands/src/benchmark/print/console/benchmark.ts @@ -1,4 +1,4 @@ -import { Benchmark, formatMean, formatUnit, MeasurementPivot } from "@ts-perf/api"; +import { Benchmark, failedMetricText, formatMean, formatUnit, MeasurementPivot } from "@ts-perf/api"; import { Query } from "iterable-query"; import { Border, Table } from "table-style"; @@ -18,10 +18,18 @@ export function printBenchmark(benchmark: Benchmark, options: BenchmarkOptions, { by: x => x.measurementName }, ], columns: [ - { header: "Project", key: "metric", expression: x => x.metric }, - { header: "Average", expression: x => formatMean(x), align: "right" }, - { header: "Best", expression: x => x.allSamplesEqual ? "~" : formatUnit(x.minimum, x), align: "right" }, - { header: "Worst", expression: x => x.allSamplesEqual ? "~" : formatUnit(x.maximum, x), align: "right" }, + { header: "Project", key: "metric", expression: x => x.failed ? failedMetricText : x.metric }, + { header: "Average", expression: x => x.failed ? "failed" : formatMean(x), align: "right" }, + { + header: "Best", + expression: x => x.failed ? "" : x.allSamplesEqual ? "~" : formatUnit(x.minimum, x), + align: "right", + }, + { + header: "Worst", + expression: x => x.failed ? "" : x.allSamplesEqual ? "~" : formatUnit(x.maximum, x), + align: "right", + }, ], rowStyles: [ "*", diff --git a/ts-perf/packages/commands/src/benchmark/print/console/comparison.ts b/ts-perf/packages/commands/src/benchmark/print/console/comparison.ts index 7353f870..2ff3033d 100644 --- a/ts-perf/packages/commands/src/benchmark/print/console/comparison.ts +++ b/ts-perf/packages/commands/src/benchmark/print/console/comparison.ts @@ -89,17 +89,17 @@ export function printComparison( { className: "group header", border: { top: "double" }, foregroundColor: "cyan" }, { className: "body", - match: (x: MeasurementComparisonPivot) => x.benchmarkRelativeDelta < -0.1, + match: (x: MeasurementComparisonPivot) => !x.failed && x.benchmarkRelativeDelta < -0.1, foregroundColor: "green", }, { className: "body", - match: (x: MeasurementComparisonPivot) => x.benchmarkRelativeDelta > 0.1, + match: (x: MeasurementComparisonPivot) => !x.failed && x.benchmarkRelativeDelta > 0.1, foregroundColor: "yellow", }, { className: "body", - match: (x: MeasurementComparisonPivot) => x.benchmarkRelativeDelta > 0.3, + match: (x: MeasurementComparisonPivot) => x.failed || x.benchmarkRelativeDelta > 0.3, foregroundColor: "red", }, { diff --git a/ts-perf/packages/commands/src/benchmark/print/html/benchmark.ts b/ts-perf/packages/commands/src/benchmark/print/html/benchmark.ts index 4401e9c4..a5298939 100644 --- a/ts-perf/packages/commands/src/benchmark/print/html/benchmark.ts +++ b/ts-perf/packages/commands/src/benchmark/print/html/benchmark.ts @@ -1,4 +1,13 @@ -import { Benchmark, formatMean, formatTestHost, formatUnit, Host, Measurement, MeasurementPivot } from "@ts-perf/api"; +import { + Benchmark, + failedMetricText, + formatMean, + formatTestHost, + formatUnit, + Host, + Measurement, + MeasurementPivot, +} from "@ts-perf/api"; import { SystemInfo } from "@ts-perf/core"; import { from, Query } from "iterable-query"; @@ -231,16 +240,18 @@ function formatReport(benchmark: Benchmark, options: BenchmarkOptions, noStyles: html` ${ - cell(measurement.metric) + cell(measurement.failed ? failedMetricText : measurement.metric) } ${ - cell(formatMean(measurement)) + cell(measurement.failed ? "failed" : formatMean(measurement)) } ${ - measurement.allSamplesEqual ? "~" : cell(formatUnit(measurement.minimum, measurement)) + measurement.failed ? "" : measurement.allSamplesEqual ? "~" + : cell(formatUnit(measurement.minimum, measurement)) } ${ - measurement.allSamplesEqual ? "~" : cell(formatUnit(measurement.maximum, measurement)) + measurement.failed ? "" : measurement.allSamplesEqual ? "~" + : cell(formatUnit(measurement.maximum, measurement)) } ` ) @@ -276,12 +287,14 @@ function getMetricClassNames(measurement: MeasurementPivot) { } function getCurrentMeanClassNames(measurement: MeasurementPivot) { + if (measurement.failed) return new ClassNames("current", "mean", "bad"); return new ClassNames("current", "mean") .addRange(measurement.relativeMarginOfError, relativeMarginOfErrorClassNames); } function getMeasurementClassNames(measurement: MeasurementPivot) { return new ClassNames("measurement") + .addIf(measurement.failed, "failed") .addIf(measurement.metric === "Memory used", "memory") .addIf(measurement.metric === "Parse Time", "parse-time") .addIf(measurement.metric === "Bind Time", "bind-time") diff --git a/ts-perf/packages/commands/src/benchmark/print/html/comparison.ts b/ts-perf/packages/commands/src/benchmark/print/html/comparison.ts index c08e2407..121f3e1c 100644 --- a/ts-perf/packages/commands/src/benchmark/print/html/comparison.ts +++ b/ts-perf/packages/commands/src/benchmark/print/html/comparison.ts @@ -447,18 +447,27 @@ function getMetricClassNames(measurement: MeasurementComparisonPivot) { } function getBaselineMeanClassNames(measurement: MeasurementComparisonPivot, ranges: Record) { + if (measurement.failed) { + return new ClassNames("baseline", "mean").addIf(!!measurement.measurement.baseline?.failed, "bad").toString(); + } return new ClassNames("baseline", "mean") .addRange(measurement.baseline!.relativeMarginOfError, ranges) .toString(); } function getMidlineMeanClassNames(measurement: MeasurementComparisonPivot, ranges: Record) { + if (measurement.failed) { + return new ClassNames("midline", "mean").addIf(!!measurement.measurement.midline?.failed, "bad").toString(); + } return new ClassNames("midline", "mean") .addRange(measurement.midline!.relativeMarginOfError, ranges) .toString(); } function getCurrentMeanClassNames(measurement: MeasurementComparisonPivot, ranges: Record) { + if (measurement.failed) { + return new ClassNames("current", "mean").addIf(!!measurement.measurement.benchmark.failed, "bad").toString(); + } return new ClassNames("current", "mean") .addRange(measurement.benchmark.relativeMarginOfError, ranges) .toString(); @@ -518,6 +527,7 @@ function getMidlineDeltaMarkers( } function getBaselineDeltaClassNames(measurement: MeasurementComparisonPivot, ranges: Record) { + if (measurement.failed) return new ClassNames("baseline", "delta"); return new ClassNames("baseline", "delta") .addRangeIf( measurement.baselineRelativeIsSignificant, @@ -527,6 +537,7 @@ function getBaselineDeltaClassNames(measurement: MeasurementComparisonPivot, ran } function getMidlineDeltaClassNames(measurement: MeasurementComparisonPivot, ranges: Record) { + if (measurement.failed) return new ClassNames("midline", "delta"); return new ClassNames("midline", "delta") .addRangeIf( measurement.midlineRelativeIsSignificant, @@ -537,6 +548,7 @@ function getMidlineDeltaClassNames(measurement: MeasurementComparisonPivot, rang function getMeasurementClassNames(measurement: MeasurementComparisonPivot) { return new ClassNames("measurement") + .addIf(measurement.failed, "failed") .addIf(measurement.metric === "Memory used", "memory") .addIf(measurement.metric === "Parse Time", "parse-time") .addIf(measurement.metric === "Bind Time", "bind-time") diff --git a/ts-perf/packages/commands/src/benchmark/print/markdown/benchmark.ts b/ts-perf/packages/commands/src/benchmark/print/markdown/benchmark.ts index 5f81cd48..ac57d564 100644 --- a/ts-perf/packages/commands/src/benchmark/print/markdown/benchmark.ts +++ b/ts-perf/packages/commands/src/benchmark/print/markdown/benchmark.ts @@ -1,4 +1,13 @@ -import { Benchmark, formatMean, formatTestHost, formatUnit, Host, Measurement } from "@ts-perf/api"; +import { + Benchmark, + failedMetricText, + formatMean, + formatTestHost, + formatUnit, + Host, + Measurement, + MeasurementPivot, +} from "@ts-perf/api"; import { SystemInfo } from "@ts-perf/core"; import { from, Query } from "iterable-query"; @@ -86,10 +95,10 @@ ${ from(group).select(measurement => markdown` -${measurement.metric} -${formatMean(measurement)} -${measurement.allSamplesEqual ? "~" : formatUnit(measurement.minimum, measurement)} -${measurement.allSamplesEqual ? "~" : formatUnit(measurement.maximum, measurement)} +${measurement.failed ? failedMetricText : measurement.metric} +${measurement.failed ? "failed" : formatMean(measurement)} +${formatBest(measurement)} +${formatWorst(measurement)} ` ) } @@ -99,6 +108,16 @@ ${ \n`; } +function formatBest(measurement: MeasurementPivot) { + if (measurement.failed) return ""; + return measurement.allSamplesEqual ? "~" : formatUnit(measurement.minimum, measurement); +} + +function formatWorst(measurement: MeasurementPivot) { + if (measurement.failed) return ""; + return measurement.allSamplesEqual ? "~" : formatUnit(measurement.maximum, measurement); +} + function formatSummary(benchmark: Benchmark, options: BenchmarkOptions) { return markdown` ${markdown`**Summary** `} diff --git a/ts-perf/packages/commands/src/benchmark/print/markdown/comparison.ts b/ts-perf/packages/commands/src/benchmark/print/markdown/comparison.ts index d49a07a9..95a524e7 100644 --- a/ts-perf/packages/commands/src/benchmark/print/markdown/comparison.ts +++ b/ts-perf/packages/commands/src/benchmark/print/markdown/comparison.ts @@ -4,6 +4,7 @@ import { formatComparisonBest, formatComparisonMetric, formatComparisonWorst, + formatFailedSide, formatPercent, formatTestHost, formatUnit, @@ -153,14 +154,17 @@ ${midline ? markdown`${formatComparisonMidlinePVa } function formatComparisonBaseline(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return formatFailedSide(comparison.measurement.baseline?.failed); return formatMean(comparison.baseline!); } function formatComparisonMidline(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return formatFailedSide(comparison.measurement.midline?.failed); return formatMean(comparison.midline!); } function formatComparisonCurrent(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return formatFailedSide(comparison.measurement.benchmark.failed); return formatMean(comparison.benchmark); } @@ -176,18 +180,22 @@ function formatMean(measurement: Value | MeasurementPivot) { } function formatComparisonBaselineDelta(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; return formatDelta(comparison.benchmark, comparison.baseline!, comparison.baselineRelativeIsSignificant); } function formatComparisonMidlineDelta(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; return formatDelta(comparison.benchmark, comparison.midline!, comparison.midlineRelativeIsSignificant); } export function formatComparisonBaselinePValue(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; return formatPValue(comparison.baselineRelativePValue, comparison.benchmark, comparison.baseline!); } export function formatComparisonMidlinePValue(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; return formatPValue(comparison.midlineRelativePValue, comparison.benchmark, comparison.midline!); } From c83b7a8065d1770136ebc3bef8e035229205fa6c Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:57:27 -0700 Subject: [PATCH 4/4] Record crashed benchmark scenarios as failed instead of aborting When the compiler under test crashes, the benchmark should still finish and report, with the crash called out, rather than failing the whole pipeline or silently timing a broken run. Stop measuring a scenario at its first failed iteration, since later iterations of a crashing build only add time and noise. tsc exits nonzero for ordinary type errors, so a tsc run counts as failed only if it never reported its timing diagnostics. --- .../commands/src/benchmark/measure.ts | 80 ++++++++++++++++--- .../src/benchmark/runApiScenario.test.ts | 78 +++++++++++------- .../commands/src/benchmark/runApiScenario.ts | 33 +++++--- 3 files changed, 140 insertions(+), 51 deletions(-) diff --git a/ts-perf/packages/commands/src/benchmark/measure.ts b/ts-perf/packages/commands/src/benchmark/measure.ts index 124e8c8d..9d434c31 100644 --- a/ts-perf/packages/commands/src/benchmark/measure.ts +++ b/ts-perf/packages/commands/src/benchmark/measure.ts @@ -180,6 +180,19 @@ export async function measureAndRunScenarios({ kind, options }: TSOptions, host: ); } +function reportFailure( + context: HostContext, + name: string, + reason: string, + scenario: Scenario, + scenarioIndex: number, + hostSpecifier: HostSpecifier, + hostIndex: number, +) { + context.error(`Scenario '${name}' failed: ${reason}. Recording it as failed and skipping remaining iterations.`); + return Measurement.createFailed(scenario.name, scenarioIndex, hostSpecifier, hostIndex); +} + function isNativeBinary(filePath: string): boolean { return !filePath.endsWith(".js"); } @@ -277,7 +290,20 @@ async function runCompilerScenario( } catch {} - if (!isWarmup && values.totalTime) { + // Type errors also cause a nonzero exit, so a crash is detected by the missing diagnostics. + if (!values.totalTime) { + return reportFailure( + context, + name, + `no timing diagnostics were reported (exit status ${status})`, + scenario, + scenarioIndex, + hostSpecifier, + hostIndex, + ); + } + + if (!isWarmup) { samples.push(values); } } @@ -364,7 +390,7 @@ async function runTSServerScenario( context.error(`>> ${line}`); }); - return new Promise(resolve => childProcess.once("exit", resolve)); + return new Promise(resolve => childProcess.once("exit", resolve)); }; const status = await runAndParseOutput(); const after = performance.now(); @@ -380,6 +406,18 @@ async function runTSServerScenario( } catch {} + if (status !== 0) { + return reportFailure( + context, + name, + `exited with status ${status}`, + scenario, + scenarioIndex, + hostSpecifier, + hostIndex, + ); + } + if (!isWarmup) { samples.push(values); } @@ -461,7 +499,7 @@ async function runLSPScenario( context.error(`>> ${line}`); }); - return new Promise(resolve => childProcess.once("exit", resolve)); + return new Promise(resolve => childProcess.once("exit", resolve)); }; const status = await runAndParseOutput(); const after = performance.now(); @@ -477,6 +515,18 @@ async function runLSPScenario( } catch {} + if (status !== 0) { + return reportFailure( + context, + name, + `exited with status ${status}`, + scenario, + scenarioIndex, + hostSpecifier, + hostIndex, + ); + } + if (!isWarmup) { samples.push(values); } @@ -498,7 +548,7 @@ async function runLSPScenario( } async function runStartupScenario( - _name: string, + name: string, scenario: Scenario, host: Host, options: BenchmarkOptions & StartupOptions, @@ -536,12 +586,12 @@ async function runStartupScenario( // The startup times are pretty quick; multiply the number of iterations by // a scaling factor in order to get statistically significant results. const scale = 100; - function execute(): Promise { + function execute(): Promise { // We can't use the tsserver harness here, as it is incompatible with --cpus; // instead, we do the same thing as the other processes, as closing stdin // is also a valid way to exit the server. const childProcess = spawn(cmd!, args, { stdio: "ignore" }); - return new Promise(resolve => childProcess.once("exit", code => resolve(code || undefined))); + return new Promise(resolve => childProcess.once("exit", (code, signal) => resolve(code ?? signal))); } const samples: StartupSample[] = []; @@ -550,14 +600,24 @@ async function runStartupScenario( const runs = numIterations + numWarmups; for (let i = 0; i < runs; i++) { const isWarmup = i < numWarmups; - let exitCode: number | undefined; - const beforeAll = performance.now(); for (let j = 0; j < scale; j++) { const before = performance.now(); - exitCode ??= await execute(); + const status = await execute(); const after = performance.now(); + if (status !== 0) { + return reportFailure( + context, + name, + `exited with status ${status}`, + scenario, + scenarioIndex, + hostSpecifier, + hostIndex, + ); + } + if (!isWarmup) { samples.push({ executionTime: after - before, @@ -567,7 +627,7 @@ async function runStartupScenario( const afterAll = performance.now(); context.info( - ` ${formatProgress(i, runs)} Completed ${scale} iterations${exitCode ? " (with errors)" : ""} in ${ + ` ${formatProgress(i, runs)} Completed ${scale} iterations in ${ ((afterAll - beforeAll) / 1000).toFixed(2) }s.${isWarmup ? ` (warmup)` : ""}`, ); diff --git a/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts b/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts index bf41bd55..64595c8b 100644 --- a/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts +++ b/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts @@ -233,28 +233,45 @@ test("requires an installed host rather than falling back to ambient Node", asyn }); }); -test("propagates bundled npm failures without falling back to PATH", async () => { +async function expectFailedMeasurement(measurement: Promise, message: string) { + const error = vi.spyOn(context, "error"); + try { + expect((await measurement).failed).toBe(true); + expect(error).toHaveBeenCalledWith(expect.stringContaining(message)); + } + finally { + error.mockRestore(); + } +} + +test("reports bundled npm failures without falling back to PATH", async () => { await fixture(async (root, scenario, host) => { await fs.writeFile(path.join(root, "npm-cli.cjs"), "console.error('npm-workload-failed'); process.exit(2);"); - await expect(runApiScenario( - scenario.with({ args: [...scenario.args!.slice(0, -1), "--verify-npm", "5"] }), - host, - { builtDir: root, suiteDir: root, iterations: 1 }, - context, - 0, - HostSpecifier.create(host), - 0, - )).rejects.toThrow("npm-workload-failed"); + await expectFailedMeasurement( + runApiScenario( + scenario.with({ args: [...scenario.args!.slice(0, -1), "--verify-npm", "5"] }), + host, + { builtDir: root, suiteDir: root, iterations: 1 }, + context, + 0, + HostSpecifier.create(host), + 0, + ), + "npm-workload-failed", + ); await fs.unlink(path.join(root, "npm-cli.cjs")); - await expect(runApiScenario( - scenario.with({ args: [...scenario.args!.slice(0, -1), "--verify-npm", "5"] }), - host, - { builtDir: root, suiteDir: root, iterations: 1 }, - context, - 0, - HostSpecifier.create(host), - 0, - )).rejects.toThrow("MODULE_NOT_FOUND"); + await expectFailedMeasurement( + runApiScenario( + scenario.with({ args: [...scenario.args!.slice(0, -1), "--verify-npm", "5"] }), + host, + { builtDir: root, suiteDir: root, iterations: 1 }, + context, + 0, + HostSpecifier.create(host), + 0, + ), + "MODULE_NOT_FOUND", + ); }); }); @@ -274,17 +291,20 @@ test("requires the solution prepared by setup.sh rather than creating it during }); }); -test("failed output checks fail the benchmark without a result protocol", async () => { +test("failed output checks mark the measurement failed without a result protocol", async () => { await fixture(async (root, scenario, host) => { - await expect(runApiScenario( - scenario.with({ args: [...scenario.args!.slice(0, -1), "--fail-check", "5"] }), - host, - { builtDir: root, suiteDir: root, iterations: 1 }, - context, - 0, - HostSpecifier.create(host), - 0, - )).rejects.toThrow("invalid output"); + await expectFailedMeasurement( + runApiScenario( + scenario.with({ args: [...scenario.args!.slice(0, -1), "--fail-check", "5"] }), + host, + { builtDir: root, suiteDir: root, iterations: 1 }, + context, + 0, + HostSpecifier.create(host), + 0, + ), + "invalid output", + ); }); }); diff --git a/ts-perf/packages/commands/src/benchmark/runApiScenario.ts b/ts-perf/packages/commands/src/benchmark/runApiScenario.ts index 2cd5dcdb..fe104967 100644 --- a/ts-perf/packages/commands/src/benchmark/runApiScenario.ts +++ b/ts-perf/packages/commands/src/benchmark/runApiScenario.ts @@ -101,18 +101,27 @@ export async function runApiScenario( for (let i = 0; i < iterations + warmups; i++) { const dumpDir = await fs.mkdtemp(path.join(timingDir, "timing-")); const before = performance.now(); - await runApiProcess( - run.cmd!, - run.args, - { - ...process.env, - PATH: `${path.dirname(executable)}${path.delimiter}${process.env.PATH}`, - TSPERF_NPM_CLI: expansion.expand(path.resolve(host.configDirectory, host.npm)), - TS_API_TIMING_DIR: dumpDir, - }, - workDir, - context, - ); + try { + await runApiProcess( + run.cmd!, + run.args, + { + ...process.env, + PATH: `${path.dirname(executable)}${path.delimiter}${process.env.PATH}`, + TSPERF_NPM_CLI: expansion.expand(path.resolve(host.configDirectory, host.npm)), + TS_API_TIMING_DIR: dumpDir, + }, + workDir, + context, + ); + } + catch (e) { + context.error( + `Scenario '${scenario.name}' failed: ${e instanceof Error ? e.message : e}. ` + + `Recording it as failed and skipping remaining iterations.`, + ); + return Measurement.createFailed(scenario.name, scenarioIndex, hostSpecifier, hostIndex); + } const elapsedMs = performance.now() - before; const timing = await readApiTimingDumps(dumpDir); context.info(`API ${scenario.name}: ${elapsedMs.toFixed(2)}ms${i < warmups ? " (warmup)" : ""}`);