diff --git a/cases/scenarios/lsp-startup/scenario.json b/cases/scenarios/lsp-startup/scenario.json index 8462a42b..a9661807 100644 --- a/cases/scenarios/lsp-startup/scenario.json +++ b/cases/scenarios/lsp-startup/scenario.json @@ -4,7 +4,6 @@ "args": [ "tsgo", "--lsp", - "--stdio", - "--disableAutomaticTypingAcquisition" + "--stdio" ] } diff --git a/ts-perf/packages/api/src/format.ts b/ts-perf/packages/api/src/format.ts index a9025d8d..daba55b9 100644 --- a/ts-perf/packages/api/src/format.ts +++ b/ts-perf/packages/api/src/format.ts @@ -145,44 +145,61 @@ export function formatTestHost(testHost: Host) { return name; } +export const failedMetricText = "⚠️ Failed"; + +/** Text for a side of a failed comparison: "failed" when that side failed, otherwise empty. */ +export function formatFailedSide(failed: boolean | undefined) { + return failed ? "failed" : ""; +} + export function formatComparisonMetric(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return failedMetricText; return comparison.metric || comparison.name; } export function formatComparisonBaseline(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return formatFailedSide(comparison.measurement.baseline?.failed); return formatMean(comparison.baseline!); } export function formatComparisonMidline(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return formatFailedSide(comparison.measurement.midline?.failed); return formatMean(comparison.midline!); } export function formatComparisonCurrent(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return formatFailedSide(comparison.measurement.benchmark.failed); return formatMean(comparison.benchmark); } export function formatComparisonBaselineDelta(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; return formatDelta(comparison.benchmark, comparison.baseline!, comparison.baselineRelativeIsSignificant); } export function formatComparisonMidlineDelta(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; return formatDelta(comparison.benchmark, comparison.midline!, comparison.midlineRelativeIsSignificant); } export function formatComparisonBaselinePValue(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; return formatPValue(comparison.baselineRelativePValue, comparison.benchmark, comparison.baseline!); } export function formatComparisonMidlinePValue(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; return formatPValue(comparison.midlineRelativePValue, comparison.benchmark, comparison.midline!); } export function formatComparisonBest(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; if (comparison.benchmark.allSamplesEqual) return "~"; return formatUnit(comparison.benchmark.minimum, comparison.benchmark); } export function formatComparisonWorst(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; if (comparison.benchmark.allSamplesEqual) return "~"; return formatUnit(comparison.benchmark.maximum, comparison.benchmark); } diff --git a/ts-perf/packages/api/src/model/measurement.test.ts b/ts-perf/packages/api/src/model/measurement.test.ts index a6295db5..465d1d0f 100644 --- a/ts-perf/packages/api/src/model/measurement.test.ts +++ b/ts-perf/packages/api/src/model/measurement.test.ts @@ -1,5 +1,6 @@ import { expect, test } from "vitest"; +import { formatComparisonBaseline, formatComparisonCurrent, formatComparisonMetric } from "../format"; import { Benchmark } from "./benchmark"; import { HostSpecifier } from "./hostSpecifier"; import { Measurement } from "./measurement"; @@ -48,3 +49,34 @@ test("scenario-only filtering remaps scenarioIndex and preserves hostIndex", () expect(filtered.measurements[0].hostIndex).toBe(1); expect(filtered.measurements[0].values?.totalTime?.samples).toEqual([1, 2]); }); + +const failed = Measurement.createFailed("test", 1, measurement.host!, 1); + +test("failed measurements survive serialization, filtering and merging", () => { + const restored = Measurement.create(JSON.parse(JSON.stringify(failed))); + expect(restored.failed).toBe(true); + expect(restored.values).toEqual({}); + expect(Measurement.create(JSON.parse(JSON.stringify(measurement))).failed).toBe(false); + + expect(report().with({ measurements: [failed] }).filter([], ["node@24.21.0"]).measurements[0].failed).toBe(true); + + expect(measurement.merge(failed).failed).toBe(true); + expect(failed.merge(measurement).failed).toBe(true); +}); + +test("comparisons with a failed side pivot to a single failed row", () => { + const passing = report(); + const crashed = passing.with({ measurements: [failed] }); + + for (const comparison of [crashed.diff(passing)!, passing.diff(crashed)!]) { + const pivots = comparison.measurements[0].pivot(); + expect(pivots).toHaveLength(1); + expect(pivots[0].failed).toBe(true); + } + + const [pivot] = crashed.diff(passing)!.measurements[0].pivot(); + expect(formatComparisonMetric(pivot)).toBe("⚠️ Failed"); + expect(formatComparisonBaseline(pivot)).toBe(""); + expect(formatComparisonCurrent(pivot)).toBe("failed"); + expect(failed.pivot().map(pivot => pivot.failed)).toEqual([true]); +}); diff --git a/ts-perf/packages/api/src/model/measurement.ts b/ts-perf/packages/api/src/model/measurement.ts index f085a268..c4219206 100644 --- a/ts-perf/packages/api/src/model/measurement.ts +++ b/ts-perf/packages/api/src/model/measurement.ts @@ -10,6 +10,7 @@ export interface MeasurementComponents extends Partial>, + /** The benchmarked process crashed or exited unsuccessfully, so no values were recorded. */ + public readonly failed = false, ) { } + public static createFailed( + scenarioName: string, + scenarioIndex: number, + host: HostSpecifier, + hostIndex: number, + ): Measurement { + return new Measurement(scenarioName, scenarioIndex, host, hostIndex, {}, /*failed*/ true); + } + public get name() { return `${this.scenarioName} - ${this.hostName}`; } @@ -53,7 +65,8 @@ export class Measurement { : undefined), components.host && HostSpecifier.create(components.host), components.hostIndex, - values, + components.failed ? {} : values, + !!components.failed, ); } @@ -63,6 +76,7 @@ export class Measurement { scenarioIndex: this.scenarioIndex, host: this.host, hostIndex: this.hostIndex, + failed: this.failed || undefined, ...this.values, }; } @@ -72,6 +86,7 @@ export class Measurement { const { scenarioIndex = this.scenarioIndex } = components; const { hostIndex = this.hostIndex } = components; const { host = this.host } = components; + const { failed = this.failed } = components; const values: Partial> = { ...this.values }; let equalValues = true; for (const key in components) { @@ -88,6 +103,7 @@ export class Measurement { && this.scenarioIndex === scenarioIndex && this.hostIndex === hostIndex && this.host === host + && this.failed === failed && equalValues ) { return this; @@ -97,6 +113,7 @@ export class Measurement { scenarioIndex, host, hostIndex, + failed, ...values, }); } @@ -110,6 +127,9 @@ export class Measurement { } public pivot() { + if (this.failed) { + return [new MeasurementPivot(this, /*value*/ undefined)]; + } const pivots: MeasurementPivot[] = []; for (const key in this.values) { if (hasProperty(this.values, key)) { @@ -132,6 +152,9 @@ export class Measurement { throw Error("Cannot merge measurements with different scenarios or hosts."); } + if (this.failed) return this; + if (other.failed) return other; + if (!this.values) return other; if (!other.values) return this; const thisKeys = Object.getOwnPropertyNames(this.values); @@ -160,6 +183,7 @@ export class Measurement { hostName: this.hostName, hostIndex: this.hostIndex, host: this.host, + failed: this.failed || undefined, ...this.values, }; } diff --git a/ts-perf/packages/api/src/model/measurementComparison.ts b/ts-perf/packages/api/src/model/measurementComparison.ts index 4fcc57e5..cabc74ea 100644 --- a/ts-perf/packages/api/src/model/measurementComparison.ts +++ b/ts-perf/packages/api/src/model/measurementComparison.ts @@ -23,6 +23,9 @@ export class MeasurementComparison { public readonly midline?: Measurement, ) { this.values = {}; + if (this.failed) { + return; + } for (const key in benchmark.values) { if (hasProperty(benchmark.values, key)) { this.values[key] = Value.diff(benchmark.values[key], baseline?.values?.[key], midline?.values?.[key]); @@ -30,6 +33,10 @@ export class MeasurementComparison { } } + public get failed() { + return this.benchmark.failed || !!this.baseline?.failed || !!this.midline?.failed; + } + public get name() { return `${this.scenarioName} - ${this.hostName}`; } @@ -82,6 +89,9 @@ export class MeasurementComparison { } public pivot() { + if (this.failed) { + return [new MeasurementComparisonPivot(this, /*value*/ undefined)]; + } const pivots: MeasurementComparisonPivot[] = []; for (const key in this.values) { if (hasProperty(this.values, key)) { diff --git a/ts-perf/packages/api/src/model/measurementComparisonPivot.ts b/ts-perf/packages/api/src/model/measurementComparisonPivot.ts index a65c43e2..4801ba7c 100644 --- a/ts-perf/packages/api/src/model/measurementComparisonPivot.ts +++ b/ts-perf/packages/api/src/model/measurementComparisonPivot.ts @@ -4,10 +4,15 @@ import { ValueComparison } from "./valueComparison"; export class MeasurementComparisonPivot { constructor( public readonly measurement: MeasurementComparison, - public readonly value: ValueComparison, + /** Undefined when the measurement failed; check {@link failed} before reading values. */ + public readonly value: ValueComparison | undefined, ) { } + public get failed() { + return this.value === undefined; + } + public get name() { return `${this.measurementName} - ${this.metric}`; } @@ -30,42 +35,42 @@ export class MeasurementComparisonPivot { return this.measurement.host; } public get metric() { - return this.value.metric; + return this.value?.metric ?? ""; } public get metricIndex() { - return this.value.metricIndex; + return this.value?.metricIndex ?? -1; } public get benchmark() { - return this.value.benchmark; + return this.value!.benchmark; } public get benchmarkRelativeDelta() { - return this.value.benchmarkRelativeDelta; + return this.value!.benchmarkRelativeDelta; } public get baseline() { - return this.value.baseline; + return this.value!.baseline; } public get baselineRelativeDelta() { - return this.value.baselineRelativeDelta; + return this.value!.baselineRelativeDelta; } public get baselineRelativePValue() { - return this.value.baselineRelativePValue; + return this.value!.baselineRelativePValue; } public get baselineRelativeIsSignificant() { - return this.value.baselineRelativeIsSignificant; + return this.value!.baselineRelativeIsSignificant; } public get midline() { - return this.value.midline; + return this.value!.midline; } public get midlineRelativeDelta() { - return this.value.midlineRelativeDelta; + return this.value!.midlineRelativeDelta; } public get midlineRelativePValue() { - return this.value.midlineRelativePValue; + return this.value!.midlineRelativePValue; } public get midlineRelativeIsSignificant() { - return this.value.midlineRelativeIsSignificant; + return this.value!.midlineRelativeIsSignificant; } public get worst() { - return this.value.worst; + return this.value!.worst; } } diff --git a/ts-perf/packages/api/src/model/measurementPivot.ts b/ts-perf/packages/api/src/model/measurementPivot.ts index cdb250c6..a19670b9 100644 --- a/ts-perf/packages/api/src/model/measurementPivot.ts +++ b/ts-perf/packages/api/src/model/measurementPivot.ts @@ -4,10 +4,15 @@ import { Value } from "./value"; export class MeasurementPivot { constructor( public readonly measurement: Measurement, - public readonly value: Value, + /** Undefined when the measurement failed; check {@link failed} before reading values. */ + public readonly value: Value | undefined, ) { } + public get failed() { + return this.value === undefined; + } + public get name() { return `${this.measurement.name} - ${this.metric}`; } @@ -27,42 +32,42 @@ export class MeasurementPivot { return this.measurement.hostIndex; } public get metric() { - return this.value.metric; + return this.value?.metric ?? ""; } public get metricIndex() { - return this.value.metricIndex; + return this.value?.metricIndex ?? -1; } public get samples() { - return this.value.samples; + return this.value!.samples; } public get mean() { - return this.value.mean; + return this.value!.mean; } public get minimum() { - return this.value.minimum; + return this.value!.minimum; } public get maximum() { - return this.value.maximum; + return this.value!.maximum; } public get marginOfError() { - return this.value.marginOfError; + return this.value!.marginOfError; } public get variance() { - return this.value.variance; + return this.value!.variance; } public get standardDeviation() { - return this.value.standardDeviation; + return this.value!.standardDeviation; } public get relativeMarginOfError() { - return this.value.relativeMarginOfError; + return this.value!.relativeMarginOfError; } public get unit() { - return this.value.unit; + return this.value!.unit; } public get precision() { - return this.value.precision; + return this.value!.precision; } public get allSamplesEqual() { - return this.value.allSamplesEqual; + return this.value!.allSamplesEqual; } } diff --git a/ts-perf/packages/commands/src/benchmark/measure.ts b/ts-perf/packages/commands/src/benchmark/measure.ts index 124e8c8d..9d434c31 100644 --- a/ts-perf/packages/commands/src/benchmark/measure.ts +++ b/ts-perf/packages/commands/src/benchmark/measure.ts @@ -180,6 +180,19 @@ export async function measureAndRunScenarios({ kind, options }: TSOptions, host: ); } +function reportFailure( + context: HostContext, + name: string, + reason: string, + scenario: Scenario, + scenarioIndex: number, + hostSpecifier: HostSpecifier, + hostIndex: number, +) { + context.error(`Scenario '${name}' failed: ${reason}. Recording it as failed and skipping remaining iterations.`); + return Measurement.createFailed(scenario.name, scenarioIndex, hostSpecifier, hostIndex); +} + function isNativeBinary(filePath: string): boolean { return !filePath.endsWith(".js"); } @@ -277,7 +290,20 @@ async function runCompilerScenario( } catch {} - if (!isWarmup && values.totalTime) { + // Type errors also cause a nonzero exit, so a crash is detected by the missing diagnostics. + if (!values.totalTime) { + return reportFailure( + context, + name, + `no timing diagnostics were reported (exit status ${status})`, + scenario, + scenarioIndex, + hostSpecifier, + hostIndex, + ); + } + + if (!isWarmup) { samples.push(values); } } @@ -364,7 +390,7 @@ async function runTSServerScenario( context.error(`>> ${line}`); }); - return new Promise(resolve => childProcess.once("exit", resolve)); + return new Promise(resolve => childProcess.once("exit", resolve)); }; const status = await runAndParseOutput(); const after = performance.now(); @@ -380,6 +406,18 @@ async function runTSServerScenario( } catch {} + if (status !== 0) { + return reportFailure( + context, + name, + `exited with status ${status}`, + scenario, + scenarioIndex, + hostSpecifier, + hostIndex, + ); + } + if (!isWarmup) { samples.push(values); } @@ -461,7 +499,7 @@ async function runLSPScenario( context.error(`>> ${line}`); }); - return new Promise(resolve => childProcess.once("exit", resolve)); + return new Promise(resolve => childProcess.once("exit", resolve)); }; const status = await runAndParseOutput(); const after = performance.now(); @@ -477,6 +515,18 @@ async function runLSPScenario( } catch {} + if (status !== 0) { + return reportFailure( + context, + name, + `exited with status ${status}`, + scenario, + scenarioIndex, + hostSpecifier, + hostIndex, + ); + } + if (!isWarmup) { samples.push(values); } @@ -498,7 +548,7 @@ async function runLSPScenario( } async function runStartupScenario( - _name: string, + name: string, scenario: Scenario, host: Host, options: BenchmarkOptions & StartupOptions, @@ -536,12 +586,12 @@ async function runStartupScenario( // The startup times are pretty quick; multiply the number of iterations by // a scaling factor in order to get statistically significant results. const scale = 100; - function execute(): Promise { + function execute(): Promise { // We can't use the tsserver harness here, as it is incompatible with --cpus; // instead, we do the same thing as the other processes, as closing stdin // is also a valid way to exit the server. const childProcess = spawn(cmd!, args, { stdio: "ignore" }); - return new Promise(resolve => childProcess.once("exit", code => resolve(code || undefined))); + return new Promise(resolve => childProcess.once("exit", (code, signal) => resolve(code ?? signal))); } const samples: StartupSample[] = []; @@ -550,14 +600,24 @@ async function runStartupScenario( const runs = numIterations + numWarmups; for (let i = 0; i < runs; i++) { const isWarmup = i < numWarmups; - let exitCode: number | undefined; - const beforeAll = performance.now(); for (let j = 0; j < scale; j++) { const before = performance.now(); - exitCode ??= await execute(); + const status = await execute(); const after = performance.now(); + if (status !== 0) { + return reportFailure( + context, + name, + `exited with status ${status}`, + scenario, + scenarioIndex, + hostSpecifier, + hostIndex, + ); + } + if (!isWarmup) { samples.push({ executionTime: after - before, @@ -567,7 +627,7 @@ async function runStartupScenario( const afterAll = performance.now(); context.info( - ` ${formatProgress(i, runs)} Completed ${scale} iterations${exitCode ? " (with errors)" : ""} in ${ + ` ${formatProgress(i, runs)} Completed ${scale} iterations in ${ ((afterAll - beforeAll) / 1000).toFixed(2) }s.${isWarmup ? ` (warmup)` : ""}`, ); diff --git a/ts-perf/packages/commands/src/benchmark/measurelsp.ts b/ts-perf/packages/commands/src/benchmark/measurelsp.ts index 4060f1dd..b86fa301 100644 --- a/ts-perf/packages/commands/src/benchmark/measurelsp.ts +++ b/ts-perf/packages/commands/src/benchmark/measurelsp.ts @@ -134,7 +134,15 @@ async function runPerf(options: CLIOpts) { connection.listen(); - serverProc.on("exit", code => console.log(code ? `Exited with code ${code}` : `Terminated`)); + let shuttingDown = false; + serverProc.on("exit", code => { + console.log(code ? `Exited with code ${code}` : `Terminated`); + // A crash can leave a request waiting forever for its response. + if (!shuttingDown) { + console.error("Server exited unexpectedly"); + process.exit(3); + } + }); // seq tracks request numbering; starts at 1 for initialize (analogous to configure in tsserver) let seq = 1; @@ -208,6 +216,7 @@ async function runPerf(options: CLIOpts) { } finally { // Shut down the LSP server + shuttingDown = true; try { await connection.sendRequest(protocol.ShutdownRequest.method); await connection.sendNotification(protocol.ExitNotification.method); diff --git a/ts-perf/packages/commands/src/benchmark/measuretsserver.ts b/ts-perf/packages/commands/src/benchmark/measuretsserver.ts index 9ff1fc34..cb39de60 100644 --- a/ts-perf/packages/commands/src/benchmark/measuretsserver.ts +++ b/ts-perf/packages/commands/src/benchmark/measuretsserver.ts @@ -110,8 +110,16 @@ async function runPerf(options: CLIOpts) { // Arguments to node execArgv, ); + let shuttingDown = false; try { - server.on("exit", code => console.log(code ? `Exited with code ${code}` : `Terminated`)); + server.on("exit", code => { + console.log(code ? `Exited with code ${code}` : `Terminated`); + // A crash can leave a request waiting forever for its response. + if (!shuttingDown) { + console.error("Server exited unexpectedly"); + process.exit(3); + } + }); if (options.cpus) { if (!server.pid) { @@ -182,6 +190,7 @@ async function runPerf(options: CLIOpts) { } finally { // Tell the server to shut down + shuttingDown = true; // eslint-disable-next-line no-useless-assignment await server.message({ seq: seq++, command: "exit" }); } diff --git a/ts-perf/packages/commands/src/benchmark/print/console/benchmark.ts b/ts-perf/packages/commands/src/benchmark/print/console/benchmark.ts index 0cd63825..5a670b38 100644 --- a/ts-perf/packages/commands/src/benchmark/print/console/benchmark.ts +++ b/ts-perf/packages/commands/src/benchmark/print/console/benchmark.ts @@ -1,4 +1,4 @@ -import { Benchmark, formatMean, formatUnit, MeasurementPivot } from "@ts-perf/api"; +import { Benchmark, failedMetricText, formatMean, formatUnit, MeasurementPivot } from "@ts-perf/api"; import { Query } from "iterable-query"; import { Border, Table } from "table-style"; @@ -18,10 +18,18 @@ export function printBenchmark(benchmark: Benchmark, options: BenchmarkOptions, { by: x => x.measurementName }, ], columns: [ - { header: "Project", key: "metric", expression: x => x.metric }, - { header: "Average", expression: x => formatMean(x), align: "right" }, - { header: "Best", expression: x => x.allSamplesEqual ? "~" : formatUnit(x.minimum, x), align: "right" }, - { header: "Worst", expression: x => x.allSamplesEqual ? "~" : formatUnit(x.maximum, x), align: "right" }, + { header: "Project", key: "metric", expression: x => x.failed ? failedMetricText : x.metric }, + { header: "Average", expression: x => x.failed ? "failed" : formatMean(x), align: "right" }, + { + header: "Best", + expression: x => x.failed ? "" : x.allSamplesEqual ? "~" : formatUnit(x.minimum, x), + align: "right", + }, + { + header: "Worst", + expression: x => x.failed ? "" : x.allSamplesEqual ? "~" : formatUnit(x.maximum, x), + align: "right", + }, ], rowStyles: [ "*", diff --git a/ts-perf/packages/commands/src/benchmark/print/console/comparison.ts b/ts-perf/packages/commands/src/benchmark/print/console/comparison.ts index 7353f870..2ff3033d 100644 --- a/ts-perf/packages/commands/src/benchmark/print/console/comparison.ts +++ b/ts-perf/packages/commands/src/benchmark/print/console/comparison.ts @@ -89,17 +89,17 @@ export function printComparison( { className: "group header", border: { top: "double" }, foregroundColor: "cyan" }, { className: "body", - match: (x: MeasurementComparisonPivot) => x.benchmarkRelativeDelta < -0.1, + match: (x: MeasurementComparisonPivot) => !x.failed && x.benchmarkRelativeDelta < -0.1, foregroundColor: "green", }, { className: "body", - match: (x: MeasurementComparisonPivot) => x.benchmarkRelativeDelta > 0.1, + match: (x: MeasurementComparisonPivot) => !x.failed && x.benchmarkRelativeDelta > 0.1, foregroundColor: "yellow", }, { className: "body", - match: (x: MeasurementComparisonPivot) => x.benchmarkRelativeDelta > 0.3, + match: (x: MeasurementComparisonPivot) => x.failed || x.benchmarkRelativeDelta > 0.3, foregroundColor: "red", }, { diff --git a/ts-perf/packages/commands/src/benchmark/print/html/benchmark.ts b/ts-perf/packages/commands/src/benchmark/print/html/benchmark.ts index 4401e9c4..a5298939 100644 --- a/ts-perf/packages/commands/src/benchmark/print/html/benchmark.ts +++ b/ts-perf/packages/commands/src/benchmark/print/html/benchmark.ts @@ -1,4 +1,13 @@ -import { Benchmark, formatMean, formatTestHost, formatUnit, Host, Measurement, MeasurementPivot } from "@ts-perf/api"; +import { + Benchmark, + failedMetricText, + formatMean, + formatTestHost, + formatUnit, + Host, + Measurement, + MeasurementPivot, +} from "@ts-perf/api"; import { SystemInfo } from "@ts-perf/core"; import { from, Query } from "iterable-query"; @@ -231,16 +240,18 @@ function formatReport(benchmark: Benchmark, options: BenchmarkOptions, noStyles: html` ${ - cell(measurement.metric) + cell(measurement.failed ? failedMetricText : measurement.metric) } ${ - cell(formatMean(measurement)) + cell(measurement.failed ? "failed" : formatMean(measurement)) } ${ - measurement.allSamplesEqual ? "~" : cell(formatUnit(measurement.minimum, measurement)) + measurement.failed ? "" : measurement.allSamplesEqual ? "~" + : cell(formatUnit(measurement.minimum, measurement)) } ${ - measurement.allSamplesEqual ? "~" : cell(formatUnit(measurement.maximum, measurement)) + measurement.failed ? "" : measurement.allSamplesEqual ? "~" + : cell(formatUnit(measurement.maximum, measurement)) } ` ) @@ -276,12 +287,14 @@ function getMetricClassNames(measurement: MeasurementPivot) { } function getCurrentMeanClassNames(measurement: MeasurementPivot) { + if (measurement.failed) return new ClassNames("current", "mean", "bad"); return new ClassNames("current", "mean") .addRange(measurement.relativeMarginOfError, relativeMarginOfErrorClassNames); } function getMeasurementClassNames(measurement: MeasurementPivot) { return new ClassNames("measurement") + .addIf(measurement.failed, "failed") .addIf(measurement.metric === "Memory used", "memory") .addIf(measurement.metric === "Parse Time", "parse-time") .addIf(measurement.metric === "Bind Time", "bind-time") diff --git a/ts-perf/packages/commands/src/benchmark/print/html/comparison.ts b/ts-perf/packages/commands/src/benchmark/print/html/comparison.ts index c08e2407..121f3e1c 100644 --- a/ts-perf/packages/commands/src/benchmark/print/html/comparison.ts +++ b/ts-perf/packages/commands/src/benchmark/print/html/comparison.ts @@ -447,18 +447,27 @@ function getMetricClassNames(measurement: MeasurementComparisonPivot) { } function getBaselineMeanClassNames(measurement: MeasurementComparisonPivot, ranges: Record) { + if (measurement.failed) { + return new ClassNames("baseline", "mean").addIf(!!measurement.measurement.baseline?.failed, "bad").toString(); + } return new ClassNames("baseline", "mean") .addRange(measurement.baseline!.relativeMarginOfError, ranges) .toString(); } function getMidlineMeanClassNames(measurement: MeasurementComparisonPivot, ranges: Record) { + if (measurement.failed) { + return new ClassNames("midline", "mean").addIf(!!measurement.measurement.midline?.failed, "bad").toString(); + } return new ClassNames("midline", "mean") .addRange(measurement.midline!.relativeMarginOfError, ranges) .toString(); } function getCurrentMeanClassNames(measurement: MeasurementComparisonPivot, ranges: Record) { + if (measurement.failed) { + return new ClassNames("current", "mean").addIf(!!measurement.measurement.benchmark.failed, "bad").toString(); + } return new ClassNames("current", "mean") .addRange(measurement.benchmark.relativeMarginOfError, ranges) .toString(); @@ -518,6 +527,7 @@ function getMidlineDeltaMarkers( } function getBaselineDeltaClassNames(measurement: MeasurementComparisonPivot, ranges: Record) { + if (measurement.failed) return new ClassNames("baseline", "delta"); return new ClassNames("baseline", "delta") .addRangeIf( measurement.baselineRelativeIsSignificant, @@ -527,6 +537,7 @@ function getBaselineDeltaClassNames(measurement: MeasurementComparisonPivot, ran } function getMidlineDeltaClassNames(measurement: MeasurementComparisonPivot, ranges: Record) { + if (measurement.failed) return new ClassNames("midline", "delta"); return new ClassNames("midline", "delta") .addRangeIf( measurement.midlineRelativeIsSignificant, @@ -537,6 +548,7 @@ function getMidlineDeltaClassNames(measurement: MeasurementComparisonPivot, rang function getMeasurementClassNames(measurement: MeasurementComparisonPivot) { return new ClassNames("measurement") + .addIf(measurement.failed, "failed") .addIf(measurement.metric === "Memory used", "memory") .addIf(measurement.metric === "Parse Time", "parse-time") .addIf(measurement.metric === "Bind Time", "bind-time") diff --git a/ts-perf/packages/commands/src/benchmark/print/markdown/benchmark.ts b/ts-perf/packages/commands/src/benchmark/print/markdown/benchmark.ts index 5f81cd48..ac57d564 100644 --- a/ts-perf/packages/commands/src/benchmark/print/markdown/benchmark.ts +++ b/ts-perf/packages/commands/src/benchmark/print/markdown/benchmark.ts @@ -1,4 +1,13 @@ -import { Benchmark, formatMean, formatTestHost, formatUnit, Host, Measurement } from "@ts-perf/api"; +import { + Benchmark, + failedMetricText, + formatMean, + formatTestHost, + formatUnit, + Host, + Measurement, + MeasurementPivot, +} from "@ts-perf/api"; import { SystemInfo } from "@ts-perf/core"; import { from, Query } from "iterable-query"; @@ -86,10 +95,10 @@ ${ from(group).select(measurement => markdown` -${measurement.metric} -${formatMean(measurement)} -${measurement.allSamplesEqual ? "~" : formatUnit(measurement.minimum, measurement)} -${measurement.allSamplesEqual ? "~" : formatUnit(measurement.maximum, measurement)} +${measurement.failed ? failedMetricText : measurement.metric} +${measurement.failed ? "failed" : formatMean(measurement)} +${formatBest(measurement)} +${formatWorst(measurement)} ` ) } @@ -99,6 +108,16 @@ ${ \n`; } +function formatBest(measurement: MeasurementPivot) { + if (measurement.failed) return ""; + return measurement.allSamplesEqual ? "~" : formatUnit(measurement.minimum, measurement); +} + +function formatWorst(measurement: MeasurementPivot) { + if (measurement.failed) return ""; + return measurement.allSamplesEqual ? "~" : formatUnit(measurement.maximum, measurement); +} + function formatSummary(benchmark: Benchmark, options: BenchmarkOptions) { return markdown` ${markdown`**Summary** `} diff --git a/ts-perf/packages/commands/src/benchmark/print/markdown/comparison.ts b/ts-perf/packages/commands/src/benchmark/print/markdown/comparison.ts index d49a07a9..95a524e7 100644 --- a/ts-perf/packages/commands/src/benchmark/print/markdown/comparison.ts +++ b/ts-perf/packages/commands/src/benchmark/print/markdown/comparison.ts @@ -4,6 +4,7 @@ import { formatComparisonBest, formatComparisonMetric, formatComparisonWorst, + formatFailedSide, formatPercent, formatTestHost, formatUnit, @@ -153,14 +154,17 @@ ${midline ? markdown`${formatComparisonMidlinePVa } function formatComparisonBaseline(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return formatFailedSide(comparison.measurement.baseline?.failed); return formatMean(comparison.baseline!); } function formatComparisonMidline(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return formatFailedSide(comparison.measurement.midline?.failed); return formatMean(comparison.midline!); } function formatComparisonCurrent(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return formatFailedSide(comparison.measurement.benchmark.failed); return formatMean(comparison.benchmark); } @@ -176,18 +180,22 @@ function formatMean(measurement: Value | MeasurementPivot) { } function formatComparisonBaselineDelta(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; return formatDelta(comparison.benchmark, comparison.baseline!, comparison.baselineRelativeIsSignificant); } function formatComparisonMidlineDelta(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; return formatDelta(comparison.benchmark, comparison.midline!, comparison.midlineRelativeIsSignificant); } export function formatComparisonBaselinePValue(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; return formatPValue(comparison.baselineRelativePValue, comparison.benchmark, comparison.baseline!); } export function formatComparisonMidlinePValue(comparison: MeasurementComparisonPivot) { + if (comparison.failed) return ""; return formatPValue(comparison.midlineRelativePValue, comparison.benchmark, comparison.midline!); } diff --git a/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts b/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts index bf41bd55..64595c8b 100644 --- a/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts +++ b/ts-perf/packages/commands/src/benchmark/runApiScenario.test.ts @@ -233,28 +233,45 @@ test("requires an installed host rather than falling back to ambient Node", asyn }); }); -test("propagates bundled npm failures without falling back to PATH", async () => { +async function expectFailedMeasurement(measurement: Promise, message: string) { + const error = vi.spyOn(context, "error"); + try { + expect((await measurement).failed).toBe(true); + expect(error).toHaveBeenCalledWith(expect.stringContaining(message)); + } + finally { + error.mockRestore(); + } +} + +test("reports bundled npm failures without falling back to PATH", async () => { await fixture(async (root, scenario, host) => { await fs.writeFile(path.join(root, "npm-cli.cjs"), "console.error('npm-workload-failed'); process.exit(2);"); - await expect(runApiScenario( - scenario.with({ args: [...scenario.args!.slice(0, -1), "--verify-npm", "5"] }), - host, - { builtDir: root, suiteDir: root, iterations: 1 }, - context, - 0, - HostSpecifier.create(host), - 0, - )).rejects.toThrow("npm-workload-failed"); + await expectFailedMeasurement( + runApiScenario( + scenario.with({ args: [...scenario.args!.slice(0, -1), "--verify-npm", "5"] }), + host, + { builtDir: root, suiteDir: root, iterations: 1 }, + context, + 0, + HostSpecifier.create(host), + 0, + ), + "npm-workload-failed", + ); await fs.unlink(path.join(root, "npm-cli.cjs")); - await expect(runApiScenario( - scenario.with({ args: [...scenario.args!.slice(0, -1), "--verify-npm", "5"] }), - host, - { builtDir: root, suiteDir: root, iterations: 1 }, - context, - 0, - HostSpecifier.create(host), - 0, - )).rejects.toThrow("MODULE_NOT_FOUND"); + await expectFailedMeasurement( + runApiScenario( + scenario.with({ args: [...scenario.args!.slice(0, -1), "--verify-npm", "5"] }), + host, + { builtDir: root, suiteDir: root, iterations: 1 }, + context, + 0, + HostSpecifier.create(host), + 0, + ), + "MODULE_NOT_FOUND", + ); }); }); @@ -274,17 +291,20 @@ test("requires the solution prepared by setup.sh rather than creating it during }); }); -test("failed output checks fail the benchmark without a result protocol", async () => { +test("failed output checks mark the measurement failed without a result protocol", async () => { await fixture(async (root, scenario, host) => { - await expect(runApiScenario( - scenario.with({ args: [...scenario.args!.slice(0, -1), "--fail-check", "5"] }), - host, - { builtDir: root, suiteDir: root, iterations: 1 }, - context, - 0, - HostSpecifier.create(host), - 0, - )).rejects.toThrow("invalid output"); + await expectFailedMeasurement( + runApiScenario( + scenario.with({ args: [...scenario.args!.slice(0, -1), "--fail-check", "5"] }), + host, + { builtDir: root, suiteDir: root, iterations: 1 }, + context, + 0, + HostSpecifier.create(host), + 0, + ), + "invalid output", + ); }); }); diff --git a/ts-perf/packages/commands/src/benchmark/runApiScenario.ts b/ts-perf/packages/commands/src/benchmark/runApiScenario.ts index 2cd5dcdb..fe104967 100644 --- a/ts-perf/packages/commands/src/benchmark/runApiScenario.ts +++ b/ts-perf/packages/commands/src/benchmark/runApiScenario.ts @@ -101,18 +101,27 @@ export async function runApiScenario( for (let i = 0; i < iterations + warmups; i++) { const dumpDir = await fs.mkdtemp(path.join(timingDir, "timing-")); const before = performance.now(); - await runApiProcess( - run.cmd!, - run.args, - { - ...process.env, - PATH: `${path.dirname(executable)}${path.delimiter}${process.env.PATH}`, - TSPERF_NPM_CLI: expansion.expand(path.resolve(host.configDirectory, host.npm)), - TS_API_TIMING_DIR: dumpDir, - }, - workDir, - context, - ); + try { + await runApiProcess( + run.cmd!, + run.args, + { + ...process.env, + PATH: `${path.dirname(executable)}${path.delimiter}${process.env.PATH}`, + TSPERF_NPM_CLI: expansion.expand(path.resolve(host.configDirectory, host.npm)), + TS_API_TIMING_DIR: dumpDir, + }, + workDir, + context, + ); + } + catch (e) { + context.error( + `Scenario '${scenario.name}' failed: ${e instanceof Error ? e.message : e}. ` + + `Recording it as failed and skipping remaining iterations.`, + ); + return Measurement.createFailed(scenario.name, scenarioIndex, hostSpecifier, hostIndex); + } const elapsedMs = performance.now() - before; const timing = await readApiTimingDumps(dumpDir); context.info(`API ${scenario.name}: ${elapsedMs.toFixed(2)}ms${i < warmups ? " (warmup)" : ""}`);