diff --git a/benchmarks/browser/browser.bench.ts b/benchmarks/browser/browser.bench.ts index cdc6836c..2cd50cb7 100644 --- a/benchmarks/browser/browser.bench.ts +++ b/benchmarks/browser/browser.bench.ts @@ -26,6 +26,23 @@ export const config = defineBenchmarkConfig({ iterations: 2, concurrency: 1, participants: browserProviders, + display: { + description: 'Browser create→connect→navigate→release lifecycle latency.', + metrics: [ + { key: 'totalMs', label: 'Total lifecycle', unit: 'ms', direction: 'lower-better', decimals: 0 }, + { key: 'createMs', label: 'Session creation', unit: 'ms', direction: 'lower-better', decimals: 0 }, + { key: 'connectMs', label: 'CDP connection', unit: 'ms', direction: 'lower-better', decimals: 0 }, + { key: 'navigateMs', label: 'Page navigation', unit: 'ms', direction: 'lower-better', decimals: 0 }, + { key: 'releaseMs', label: 'Session release', unit: 'ms', direction: 'lower-better', decimals: 0 }, + ], + steps: [ + { key: 'create', label: 'Create session' }, + { key: 'connect', label: 'Connect CDP' }, + { key: 'navigate', label: 'Navigate page' }, + { key: 'release', label: 'Release session' }, + ], + overview: { defaultMetric: 'totalMs', defaultLayout: 'ranking' }, + }, onScore: (lowerIsBetter) => ({ metrics: [ lowerIsBetter('totalMs', { unit: 'ms', ceiling: 10000, weights: { median: 0.40, p95: 0.20, p99: 0.10 } }), diff --git a/benchmarks/sandbox/dax.bench.ts b/benchmarks/sandbox/dax.bench.ts index 8f6cf660..f20fd5cd 100644 --- a/benchmarks/sandbox/dax.bench.ts +++ b/benchmarks/sandbox/dax.bench.ts @@ -43,6 +43,37 @@ export const config = defineBenchmarkConfig({ groupBy: 'round', defaultProviders: ['e2b', 'modal', 'tensorlake'], participants: providers, + display: { + description: 'OpenCode build lifecycle latency per phase.', + metrics: [ + { key: 'totalMs', label: 'Total build time', unit: 'ms', direction: 'lower-better', decimals: 0 }, + { key: 'prepareMs', label: 'Prepare', unit: 'ms', direction: 'lower-better', decimals: 0 }, + { key: 'cacheClearMs', label: 'Cache clear', unit: 'ms', direction: 'lower-better', decimals: 0 }, + { key: 'bunDownloadMs', label: 'Bun download', unit: 'ms', direction: 'lower-better', decimals: 0 }, + { key: 'bunUnpackMs', label: 'Bun unpack', unit: 'ms', direction: 'lower-better', decimals: 0 }, + { key: 'cloneMs', label: 'Clone', unit: 'ms', direction: 'lower-better', decimals: 0 }, + { key: 'installMs', label: 'Install', unit: 'ms', direction: 'lower-better', decimals: 0 }, + { key: 'typecheckMs', label: 'Typecheck', unit: 'ms', direction: 'lower-better', decimals: 0 }, + { key: 'phasesCompleted', label: 'Phases completed', direction: 'higher-better', decimals: 0 }, + { key: 'phasesTotal', label: 'Phases total', direction: 'higher-better', decimals: 0 }, + { key: 'diskAfterClone', label: 'Disk after clone', unit: 'bytes', direction: 'lower-better', decimals: 0 }, + { key: 'diskAfterInstall', label: 'Disk after install', unit: 'bytes', direction: 'lower-better', decimals: 0 }, + { key: 'diskAfterTypecheck', label: 'Disk after typecheck', unit: 'bytes', direction: 'lower-better', decimals: 0 }, + ], + steps: [ + { key: 'create', label: 'Create sandbox' }, + { key: 'build', label: 'Build' }, + { key: 'destroy', label: 'Destroy sandbox' }, + { key: 'prepare', label: 'Prepare' }, + { key: 'cache_clear', label: 'Cache clear' }, + { key: 'bun_download', label: 'Bun download' }, + { key: 'bun_unpack', label: 'Bun unpack' }, + { key: 'clone', label: 'Clone' }, + { key: 'install', label: 'Install' }, + { key: 'typecheck', label: 'Typecheck' }, + ], + overview: { defaultMetric: 'totalMs', defaultLayout: 'ranking' }, + }, onComplete: (outcome) => writeDaxLegacyResults(outcome.participants, { resultsDir: path.resolve(__dirname, '../../results/sandbox-dax'), @@ -220,7 +251,10 @@ function daxPhaseSteps(t: DaxTimingResult): TaskStepRecord[] { ]; return phases .filter((entry): entry is [string, number] => typeof entry[1] === 'number') - .map(([name, latencyMs]) => ({ name, status: 'success', latencyMs })); + .map(([name, latencyMs]) => { + const metricKey = name.replace(/_(.)/g, (_, c) => c.toUpperCase()) + 'Ms'; + return { name, status: 'success', latencyMs, data: { [metricKey]: latencyMs } }; + }); } export const task = defineTask(async (ctx) => { @@ -238,7 +272,17 @@ export const task = defineTask(async (ctx) => { let timing: DaxTimingResult; try { - timing = await ctx.step('build', () => runDaxBuild(sandbox, p.name, p.timeout ?? timeout)); + timing = await ctx.step('build', async () => { + const t = await runDaxBuild(sandbox, p.name, p.timeout ?? timeout); + const buildMetrics: JsonObject = { totalMs: t.totalMs }; + if (t.phasesCompleted !== undefined) buildMetrics.phasesCompleted = t.phasesCompleted; + if (t.phasesTotal !== undefined) buildMetrics.phasesTotal = t.phasesTotal; + if (t.diskAfterClone !== undefined) buildMetrics.diskAfterClone = t.diskAfterClone; + if (t.diskAfterInstall !== undefined) buildMetrics.diskAfterInstall = t.diskAfterInstall; + if (t.diskAfterTypecheck !== undefined) buildMetrics.diskAfterTypecheck = t.diskAfterTypecheck; + ctx.measure(buildMetrics); + return t; + }); } finally { await ctx .step('destroy', () => withTimeout(sandbox.destroy(), p.destroyTimeoutMs ?? destroyTimeoutMs, 'Destroy timeout'), { diff --git a/benchmarks/sandbox/tti.bench.ts b/benchmarks/sandbox/tti.bench.ts index bccbd5c2..539db41d 100644 --- a/benchmarks/sandbox/tti.bench.ts +++ b/benchmarks/sandbox/tti.bench.ts @@ -39,6 +39,18 @@ export const config = defineBenchmarkConfig({ iterations: 2, concurrency: 1, participants: providers, + display: { + description: 'Sandbox time-to-interactive from create through first successful command.', + metrics: [ + { key: 'ttiMs', label: 'Time to interactive', unit: 'ms', direction: 'lower-better', decimals: 0 }, + ], + steps: [ + { key: 'create', label: 'Create sandbox' }, + { key: 'exec.task', label: 'Run first command' }, + { key: 'destroy', label: 'Destroy sandbox' }, + ], + overview: { defaultMetric: 'ttiMs', defaultLayout: 'ranking' }, + }, onScore: (lowerIsBetter) => ({ metrics: [ lowerIsBetter('ttiMs', { @@ -78,6 +90,7 @@ export const task = defineTask(async (ctx) => { ), ); + let ttiMs: number | undefined; try { await step('exec.task', async () => { const result = await withTimeout( @@ -88,12 +101,18 @@ export const task = defineTask(async (ctx) => { if (result.exitCode !== 0) { throw new Error(`Command failed with exit code ${result.exitCode}: ${result.stderr || 'Unknown error'}`); } + ttiMs = performance.now() - start; + measure({ ttiMs }); }); - measure({ ttiMs: performance.now() - start }); } finally { await step('destroy', () => withTimeout(sandbox.destroy(), participant.destroyTimeoutMs ?? DESTROY_TIMEOUT_MS, 'Destroy timeout'), { reportConcurrency: false }, ).catch((err) => console.warn(` [cleanup] destroy failed: ${formatError(err)}`)); } + + if (ttiMs === undefined) { + throw new Error('exec.task did not produce a ttiMs measurement'); + } + return { data: { ttiMs } }; }); diff --git a/packages/benchsdk-runner/src/__tests__/bench-config.test.ts b/packages/benchsdk-runner/src/__tests__/bench-config.test.ts index 5c59d0cd..160f3826 100644 --- a/packages/benchsdk-runner/src/__tests__/bench-config.test.ts +++ b/packages/benchsdk-runner/src/__tests__/bench-config.test.ts @@ -109,6 +109,49 @@ describe('defineBenchmarkConfig', () => { defineBenchmarkConfig({ benchmarkSlug: 's', benchmarkName: 'n', participants, shapes: { s: { slug: 'ok', staggerDelayMs: -1 } } }), ).toThrow('staggerDelayMs'); }); + + it('carries a display manifest when valid', () => { + const config = defineBenchmarkConfig({ + benchmarkSlug: 's', + benchmarkName: 'n', + participants, + display: { + description: 'A test benchmark', + metrics: [{ key: 'throughputMbps', label: 'Throughput', unit: 'Mbps', direction: 'higher-better' }], + steps: [{ key: 'create', label: 'Create sandbox' }], + overview: { defaultMetric: 'throughputMbps', defaultLayout: 'ranking' }, + }, + }); + expect(config.display?.overview?.defaultLayout).toBe('ranking'); + expect(config.display?.metrics?.[0].key).toBe('throughputMbps'); + }); + + it('rejects an invalid display direction', () => { + expect(() => + defineBenchmarkConfig({ + benchmarkSlug: 's', + benchmarkName: 'n', + participants, + display: { metrics: [{ key: 'x', label: 'X', direction: 'up' as any }] }, + }), + ).toThrow('direction'); + }); + + it('rejects duplicate display metric keys', () => { + expect(() => + defineBenchmarkConfig({ + benchmarkSlug: 's', + benchmarkName: 'n', + participants, + display: { + metrics: [ + { key: 'x', label: 'X' }, + { key: 'x', label: 'X2' }, + ], + }, + }), + ).toThrow('duplicate'); + }); }); describe('defineTask', () => { diff --git a/packages/benchsdk-runner/src/bench-config.ts b/packages/benchsdk-runner/src/bench-config.ts index 289a0fb6..10c58d89 100644 --- a/packages/benchsdk-runner/src/bench-config.ts +++ b/packages/benchsdk-runner/src/bench-config.ts @@ -162,6 +162,55 @@ export interface ResolvedRunConfig { providers?: string[]; } +/** Display metadata for a single custom metric a benchmark reports via `ctx.measure`. */ +export interface BenchmarkMetricDisplay { + /** Stable metric key, matching the key in `ctx.measure` or `data`. */ + key: string; + /** Human-readable label shown in the platform UI. */ + label: string; + /** Optional unit shown after the value (e.g. `Mbps`, `/s`, `ms`). */ + unit?: string; + /** Number of decimal places when formatting numeric values. Defaults to the display format. */ + decimals?: number; + /** Whether higher or lower values rank better. */ + direction?: 'higher-better' | 'lower-better'; + /** Optional ordering hint for metric lists. */ + order?: number; +} + +/** Display metadata for a single task lifecycle step. */ +export interface BenchmarkStepDisplay { + /** Stable step name, matching the string passed to `ctx.step`. */ + key: string; + /** Human-readable label shown in the platform UI. */ + label: string; + /** Optional ordering hint for step lists. */ + order?: number; +} + +/** Display defaults for the benchmark overview page. */ +export interface BenchmarkOverviewDisplay { + /** Metric key to rank participants by by default (falls back to overall task latency). */ + defaultMetric?: string; + /** Default overview layout. */ + defaultLayout?: 'ranking' | 'cards' | 'chart' | 'leaderboard'; +} + +/** + * Optional platform display manifest. A `*.bench.ts` file owns not only how the + * benchmark runs, but how it should be rendered, without a platform code change. + */ +export interface BenchmarkDisplayConfig { + /** Optional human-readable description shown on the benchmark listing. */ + description?: string; + /** Metric catalog — labels, units, and ranking direction for `ctx.measure` keys. */ + metrics?: BenchmarkMetricDisplay[]; + /** Step catalog — human labels for lifecycle steps reported via `ctx.step`. */ + steps?: BenchmarkStepDisplay[]; + /** Overview defaults. */ + overview?: BenchmarkOverviewDisplay; +} + /** * Result of a benchmark run, passed to `config.onComplete`. Exposes the raw * per-participant records so completion hooks can write legacy local results. @@ -237,6 +286,18 @@ export interface BenchmarkConfig { * writers). This is the run-level counterpart to per-step `ctx.measure`. */ onComplete?: (outcome: BenchmarkRunOutcome) => void | Promise; + /** + * Optional display manifest. Lets the bench author configure metric labels, + * step labels, and overview defaults without editing the platform. + */ + display?: BenchmarkDisplayConfig; +} + +function assertNonEmptyString(value: unknown, field: string): string { + if (typeof value !== 'string' || value.trim() === '') { + throw new Error(`${field} must be a non-empty string`); + } + return value; } function assertPositiveInt(value: number | undefined, field: string): void { @@ -296,6 +357,68 @@ export function defineBenchmarkConfig(); + for (let i = 0; i < config.display.metrics.length; i++) { + const metric = config.display.metrics[i]; + if (metric === null || typeof metric !== 'object' || Array.isArray(metric)) { + throw new Error(`display.metrics[${i}] must be an object`); + } + const key = assertNonEmptyString(metric.key, `display.metrics[${i}].key`); + if (seenMetricKeys.has(key)) { + throw new Error(`duplicate display metric key: ${key}`); + } + seenMetricKeys.add(key); + assertNonEmptyString(metric.label, `display.metrics[${i}].label`); + if (metric.direction !== undefined && metric.direction !== 'higher-better' && metric.direction !== 'lower-better') { + throw new Error(`display.metrics[${i}].direction must be 'higher-better' or 'lower-better'`); + } + if (metric.decimals !== undefined && (!Number.isInteger(metric.decimals) || metric.decimals < 0)) { + throw new Error(`display.metrics[${i}].decimals must be a non-negative integer`); + } + if (metric.order !== undefined && (!Number.isInteger(metric.order) || metric.order < 0)) { + throw new Error(`display.metrics[${i}].order must be a non-negative integer`); + } + } + } + if (config.display.steps !== undefined) { + if (!Array.isArray(config.display.steps)) { + throw new Error('display.steps must be an array'); + } + const seenStepKeys = new Set(); + for (let i = 0; i < config.display.steps.length; i++) { + const step = config.display.steps[i]; + if (step === null || typeof step !== 'object' || Array.isArray(step)) { + throw new Error(`display.steps[${i}] must be an object`); + } + const key = assertNonEmptyString(step.key, `display.steps[${i}].key`); + if (seenStepKeys.has(key)) { + throw new Error(`duplicate display step key: ${key}`); + } + seenStepKeys.add(key); + assertNonEmptyString(step.label, `display.steps[${i}].label`); + if (step.order !== undefined && (!Number.isInteger(step.order) || step.order < 0)) { + throw new Error(`display.steps[${i}].order must be a non-negative integer`); + } + } + } + if (config.display.overview !== undefined) { + if (typeof config.display.overview !== 'object' || config.display.overview === null || Array.isArray(config.display.overview)) { + throw new Error('display.overview must be an object'); + } + const { defaultLayout } = config.display.overview; + if (defaultLayout !== undefined && !['ranking', 'cards', 'chart', 'leaderboard'].includes(defaultLayout)) { + throw new Error("display.overview.defaultLayout must be 'ranking', 'cards', 'chart', or 'leaderboard'"); + } + } + } return config; } diff --git a/packages/benchsdk-runner/src/index.ts b/packages/benchsdk-runner/src/index.ts index b91e831c..ab443e9a 100644 --- a/packages/benchsdk-runner/src/index.ts +++ b/packages/benchsdk-runner/src/index.ts @@ -10,6 +10,10 @@ export type { ParticipantRecords, ResolvedRunConfig, BenchmarkRunOutcome, + BenchmarkDisplayConfig, + BenchmarkMetricDisplay, + BenchmarkStepDisplay, + BenchmarkOverviewDisplay, } from './bench-config.js'; export { NoAvailableParticipantsError } from './no-available-participants.js'; export { runBenchmark, parseCliArgs, mergeConfig } from './runner.js'; diff --git a/packages/benchsdk-runner/src/runner.ts b/packages/benchsdk-runner/src/runner.ts index c542ada9..3335efd2 100644 --- a/packages/benchsdk-runner/src/runner.ts +++ b/packages/benchsdk-runner/src/runner.ts @@ -478,6 +478,7 @@ export async function runBenchmark( if (identityIsOurs) { await client!.upsertBenchmark(config.benchmarkSlug, { name: config.benchmarkName, + ...(config.display ? { config: { display: config.display as unknown as JsonObject } } : {}), }); }