Handle crashing benchmarks better

cases/scenarios/lsp-startup/scenario.json

@@ -4,7 +4,6 @@
     "args": [
         "tsgo",
         "--lsp",
-        "--stdio",
-        "--disableAutomaticTypingAcquisition"
+        "--stdio"
     ]
 }

ts-perf/packages/api/src/format.ts

@@ -145,44 +145,61 @@ export function formatTestHost(testHost: Host) {
     return name;
 }
 
+export const failedMetricText = "⚠️ Failed";
+
+/** Text for a side of a failed comparison: "failed" when that side failed, otherwise empty. */
+export function formatFailedSide(failed: boolean | undefined) {
+    return failed ? "failed" : "";
+}
+
 export function formatComparisonMetric(comparison: MeasurementComparisonPivot) {
+    if (comparison.failed) return failedMetricText;
     return comparison.metric || comparison.name;
 }
 
 export function formatComparisonBaseline(comparison: MeasurementComparisonPivot) {
+    if (comparison.failed) return formatFailedSide(comparison.measurement.baseline?.failed);
     return formatMean(comparison.baseline!);
 }
 
 export function formatComparisonMidline(comparison: MeasurementComparisonPivot) {
+    if (comparison.failed) return formatFailedSide(comparison.measurement.midline?.failed);
     return formatMean(comparison.midline!);
 }
 
 export function formatComparisonCurrent(comparison: MeasurementComparisonPivot) {
+    if (comparison.failed) return formatFailedSide(comparison.measurement.benchmark.failed);
     return formatMean(comparison.benchmark);
 }
 
 export function formatComparisonBaselineDelta(comparison: MeasurementComparisonPivot) {
+    if (comparison.failed) return "";
     return formatDelta(comparison.benchmark, comparison.baseline!, comparison.baselineRelativeIsSignificant);
 }
 
 export function formatComparisonMidlineDelta(comparison: MeasurementComparisonPivot) {
+    if (comparison.failed) return "";
     return formatDelta(comparison.benchmark, comparison.midline!, comparison.midlineRelativeIsSignificant);
 }
 
 export function formatComparisonBaselinePValue(comparison: MeasurementComparisonPivot) {
+    if (comparison.failed) return "";
     return formatPValue(comparison.baselineRelativePValue, comparison.benchmark, comparison.baseline!);
 }
 
 export function formatComparisonMidlinePValue(comparison: MeasurementComparisonPivot) {
+    if (comparison.failed) return "";
     return formatPValue(comparison.midlineRelativePValue, comparison.benchmark, comparison.midline!);
 }
 
 export function formatComparisonBest(comparison: MeasurementComparisonPivot) {
+    if (comparison.failed) return "";
     if (comparison.benchmark.allSamplesEqual) return "~";
     return formatUnit(comparison.benchmark.minimum, comparison.benchmark);
 }
 
 export function formatComparisonWorst(comparison: MeasurementComparisonPivot) {
+    if (comparison.failed) return "";
     if (comparison.benchmark.allSamplesEqual) return "~";
     return formatUnit(comparison.benchmark.maximum, comparison.benchmark);
 }

ts-perf/packages/api/src/model/measurement.test.ts

@@ -1,5 +1,6 @@
 import { expect, test } from "vitest";
 
+import { formatComparisonBaseline, formatComparisonCurrent, formatComparisonMetric } from "../format";
 import { Benchmark } from "./benchmark";
 import { HostSpecifier } from "./hostSpecifier";
 import { Measurement } from "./measurement";
@@ -48,3 +49,34 @@ test("scenario-only filtering remaps scenarioIndex and preserves hostIndex", ()
     expect(filtered.measurements[0].hostIndex).toBe(1);
     expect(filtered.measurements[0].values?.totalTime?.samples).toEqual([1, 2]);
 });
+
+const failed = Measurement.createFailed("test", 1, measurement.host!, 1);
+
+test("failed measurements survive serialization, filtering and merging", () => {
+    const restored = Measurement.create(JSON.parse(JSON.stringify(failed)));
+    expect(restored.failed).toBe(true);
+    expect(restored.values).toEqual({});
+    expect(Measurement.create(JSON.parse(JSON.stringify(measurement))).failed).toBe(false);
+
+    expect(report().with({ measurements: [failed] }).filter([], ["node@24.21.0"]).measurements[0].failed).toBe(true);
+
+    expect(measurement.merge(failed).failed).toBe(true);
+    expect(failed.merge(measurement).failed).toBe(true);
+});
+
+test("comparisons with a failed side pivot to a single failed row", () => {
+    const passing = report();
+    const crashed = passing.with({ measurements: [failed] });
+
+    for (const comparison of [crashed.diff(passing)!, passing.diff(crashed)!]) {
+        const pivots = comparison.measurements[0].pivot();
+        expect(pivots).toHaveLength(1);
+        expect(pivots[0].failed).toBe(true);
+    }
+
+    const [pivot] = crashed.diff(passing)!.measurements[0].pivot();
+    expect(formatComparisonMetric(pivot)).toBe("⚠️ Failed");
+    expect(formatComparisonBaseline(pivot)).toBe("");
+    expect(formatComparisonCurrent(pivot)).toBe("failed");
+    expect(failed.pivot().map(pivot => pivot.failed)).toEqual([true]);
+});

ts-perf/packages/api/src/model/measurement.ts

@@ -10,6 +10,7 @@ export interface MeasurementComponents extends Partial<Record<SampleKey, Value |
     scenarioIndex?: number;
     host?: HostSpecifier | HostSpecifierComponents;
     hostIndex?: number;
+    failed?: boolean;
 }
 
 const hasOwnProperty = Object.prototype.hasOwnProperty;
@@ -24,9 +25,20 @@ export class Measurement {
         public readonly host?: HostSpecifier,
         public readonly hostIndex?: number,
         public readonly values?: Partial<Record<string, Value>>,
+        /** The benchmarked process crashed or exited unsuccessfully, so no values were recorded. */
+        public readonly failed = false,
     ) {
     }
 
+    public static createFailed(
+        scenarioName: string,
+        scenarioIndex: number,
+        host: HostSpecifier,
+        hostIndex: number,
+    ): Measurement {
+        return new Measurement(scenarioName, scenarioIndex, host, hostIndex, {}, /*failed*/ true);
+    }
+
     public get name() {
         return `${this.scenarioName} - ${this.hostName}`;
     }
@@ -53,7 +65,8 @@ export class Measurement {
                     : undefined),
             components.host && HostSpecifier.create(components.host),
             components.hostIndex,
-            values,
+            components.failed ? {} : values,
+            !!components.failed,
         );
     }
 
@@ -63,6 +76,7 @@ export class Measurement {
             scenarioIndex: this.scenarioIndex,
             host: this.host,
             hostIndex: this.hostIndex,
+            failed: this.failed || undefined,
             ...this.values,
         };
     }
@@ -72,6 +86,7 @@ export class Measurement {
         const { scenarioIndex = this.scenarioIndex } = components;
         const { hostIndex = this.hostIndex } = components;
         const { host = this.host } = components;
+        const { failed = this.failed } = components;
         const values: Partial<Record<string, ValueComponents>> = { ...this.values };
         let equalValues = true;
         for (const key in components) {
@@ -88,6 +103,7 @@ export class Measurement {
             && this.scenarioIndex === scenarioIndex
             && this.hostIndex === hostIndex
             && this.host === host
+            && this.failed === failed
             && equalValues
         ) {
             return this;
@@ -97,6 +113,7 @@ export class Measurement {
             scenarioIndex,
             host,
             hostIndex,
+            failed,
             ...values,
         });
     }
@@ -110,6 +127,9 @@ export class Measurement {
     }
 
     public pivot() {
+        if (this.failed) {
+            return [new MeasurementPivot(this, /*value*/ undefined)];
+        }
         const pivots: MeasurementPivot[] = [];
         for (const key in this.values) {
             if (hasProperty(this.values, key)) {
@@ -132,6 +152,9 @@ export class Measurement {
             throw Error("Cannot merge measurements with different scenarios or hosts.");
         }
 
+        if (this.failed) return this;
+        if (other.failed) return other;
+
         if (!this.values) return other;
         if (!other.values) return this;
         const thisKeys = Object.getOwnPropertyNames(this.values);
@@ -160,6 +183,7 @@ export class Measurement {
             hostName: this.hostName,
             hostIndex: this.hostIndex,
             host: this.host,
+            failed: this.failed || undefined,
             ...this.values,
         };
     }

ts-perf/packages/api/src/model/measurementComparison.ts

@@ -23,13 +23,20 @@ export class MeasurementComparison {
         public readonly midline?: Measurement,
     ) {
         this.values = {};
+        if (this.failed) {
+            return;
+        }
         for (const key in benchmark.values) {
             if (hasProperty(benchmark.values, key)) {
                 this.values[key] = Value.diff(benchmark.values[key], baseline?.values?.[key], midline?.values?.[key]);
             }
         }
     }
 
+    public get failed() {
+        return this.benchmark.failed || !!this.baseline?.failed || !!this.midline?.failed;
+    }
+
     public get name() {
         return `${this.scenarioName} - ${this.hostName}`;
     }
@@ -82,6 +89,9 @@ export class MeasurementComparison {
     }
 
     public pivot() {
+        if (this.failed) {
+            return [new MeasurementComparisonPivot(this, /*value*/ undefined)];
+        }
         const pivots: MeasurementComparisonPivot[] = [];
         for (const key in this.values) {
             if (hasProperty(this.values, key)) {

ts-perf/packages/api/src/model/measurementComparisonPivot.ts

@@ -4,10 +4,15 @@ import { ValueComparison } from "./valueComparison";
 export class MeasurementComparisonPivot {
     constructor(
         public readonly measurement: MeasurementComparison,
-        public readonly value: ValueComparison,
+        /** Undefined when the measurement failed; check {@link failed} before reading values. */
+        public readonly value: ValueComparison | undefined,
     ) {
     }
 
+    public get failed() {
+        return this.value === undefined;
+    }
+
     public get name() {
         return `${this.measurementName} - ${this.metric}`;
     }
@@ -30,42 +35,42 @@ export class MeasurementComparisonPivot {
         return this.measurement.host;
     }
     public get metric() {
-        return this.value.metric;
+        return this.value?.metric ?? "";
     }
     public get metricIndex() {
-        return this.value.metricIndex;
+        return this.value?.metricIndex ?? -1;
     }
     public get benchmark() {
-        return this.value.benchmark;
+        return this.value!.benchmark;
     }
     public get benchmarkRelativeDelta() {
-        return this.value.benchmarkRelativeDelta;
+        return this.value!.benchmarkRelativeDelta;
     }
     public get baseline() {
-        return this.value.baseline;
+        return this.value!.baseline;
     }
     public get baselineRelativeDelta() {
-        return this.value.baselineRelativeDelta;
+        return this.value!.baselineRelativeDelta;
     }
     public get baselineRelativePValue() {
-        return this.value.baselineRelativePValue;
+        return this.value!.baselineRelativePValue;
     }
     public get baselineRelativeIsSignificant() {
-        return this.value.baselineRelativeIsSignificant;
+        return this.value!.baselineRelativeIsSignificant;
     }
     public get midline() {
-        return this.value.midline;
+        return this.value!.midline;
     }
     public get midlineRelativeDelta() {
-        return this.value.midlineRelativeDelta;
+        return this.value!.midlineRelativeDelta;
     }
     public get midlineRelativePValue() {
-        return this.value.midlineRelativePValue;
+        return this.value!.midlineRelativePValue;
     }
     public get midlineRelativeIsSignificant() {
-        return this.value.midlineRelativeIsSignificant;
+        return this.value!.midlineRelativeIsSignificant;
     }
     public get worst() {
-        return this.value.worst;
+        return this.value!.worst;
     }
 }

ts-perf/packages/api/src/model/measurementPivot.ts

@@ -4,10 +4,15 @@ import { Value } from "./value";
 export class MeasurementPivot {
     constructor(
         public readonly measurement: Measurement,
-        public readonly value: Value,
+        /** Undefined when the measurement failed; check {@link failed} before reading values. */
+        public readonly value: Value | undefined,
     ) {
     }
 
+    public get failed() {
+        return this.value === undefined;
+    }
+
     public get name() {
         return `${this.measurement.name} - ${this.metric}`;
     }
@@ -27,42 +32,42 @@ export class MeasurementPivot {
         return this.measurement.hostIndex;
     }
     public get metric() {
-        return this.value.metric;
+        return this.value?.metric ?? "";
     }
     public get metricIndex() {
-        return this.value.metricIndex;
+        return this.value?.metricIndex ?? -1;
     }
     public get samples() {
-        return this.value.samples;
+        return this.value!.samples;
     }
     public get mean() {
-        return this.value.mean;
+        return this.value!.mean;
     }
     public get minimum() {
-        return this.value.minimum;
+        return this.value!.minimum;
     }
     public get maximum() {
-        return this.value.maximum;
+        return this.value!.maximum;
     }
     public get marginOfError() {
-        return this.value.marginOfError;
+        return this.value!.marginOfError;
     }
     public get variance() {
-        return this.value.variance;
+        return this.value!.variance;
     }
     public get standardDeviation() {
-        return this.value.standardDeviation;
+        return this.value!.standardDeviation;
     }
     public get relativeMarginOfError() {
-        return this.value.relativeMarginOfError;
+        return this.value!.relativeMarginOfError;
     }
     public get unit() {
-        return this.value.unit;
+        return this.value!.unit;
     }
     public get precision() {
-        return this.value.precision;
+        return this.value!.precision;
     }
     public get allSamplesEqual() {
-        return this.value.allSamplesEqual;
+        return this.value!.allSamplesEqual;
     }
 }