diff --git a/.github/workflows/storage-benchmarks.yml b/.github/workflows/storage-benchmarks.yml index 33729381..950ad7c4 100644 --- a/.github/workflows/storage-benchmarks.yml +++ b/.github/workflows/storage-benchmarks.yml @@ -13,7 +13,7 @@ on: workflow_dispatch: inputs: iterations: - description: 'Iterations per provider' + description: 'Iterations per provider, per file size' required: false default: '100' storage_concurrency: @@ -21,7 +21,7 @@ on: required: false default: '1' file_size: - description: 'File size to test (leave empty to run all)' + description: 'File size to test (leave empty to run all four in one run)' required: false default: '' type: choice @@ -48,10 +48,11 @@ permissions: jobs: bench: - name: Bench ${{ matrix.provider }} ${{ matrix.file_size }} + name: Bench ${{ matrix.provider }} runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe # runs-on: self-hosted - timeout-minutes: 90 + # One job now covers all four sizes, so it does ~4x the work a per-size job did. + timeout-minutes: 300 strategy: fail-fast: false matrix: @@ -65,7 +66,6 @@ jobs: - tensorlake - archil - neon - file_size: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.file_size != '' && fromJson(format('["{0}"]', github.event.inputs.file_size))) || (github.event_name == 'push' && fromJson('["1MB"]')) || fromJson('["1MB","4MB","10MB","16MB"]') }} steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 @@ -87,11 +87,11 @@ jobs: run: | . benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|TENSORLAKE_API_KEY|TENSORLAKE_API_URL|TENSORLAKE_ORGANIZATION_ID|TENSORLAKE_PROJECT_ID|TENSORLAKE_FILESYSTEM|ARCHIL_S3_ACCESS_KEY_ID|ARCHIL_S3_SECRET_ACCESS_KEY|ARCHIL_BUCKET|ARCHIL_REGION|ARCHIL_BRANCH|NEON_BUCKET|NEON_ENDPOINT|NEON_ACCESS_KEY_ID|NEON_SECRET_ACCESS_KEY|NEON_REGION|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' - FILE_SIZE="${{ matrix.file_size }}" - # Matrix jobs for the same file size share one platform run (the benchmark - # slug is size-independent, so the size is part of the key). The attempt - # number is included so re-runs do not rejoin completed runs. - RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${FILE_SIZE}" + # Every size runs in one job, so a run covers all of them and the + # dashboard can compare sizes within it. The attempt number is included + # so re-runs do not rejoin completed runs. + FILE_SIZE="${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.file_size != '' && github.event.inputs.file_size) || (github.event_name == 'push' && '1MB') || '1MB,4MB,10MB,16MB' }}" + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/storage.bench.ts \ --provider ${{ matrix.provider }} \ --file-size $FILE_SIZE \ @@ -103,7 +103,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: storage-results-${{ matrix.provider }}-${{ matrix.file_size }} + name: storage-results-${{ matrix.provider }} path: results/storage/ if-no-files-found: ignore retention-days: 7 @@ -162,16 +162,19 @@ jobs: const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; // Only render sizes that actually ran in this workflow. The checkout // carries committed latest.json files for every size (from the weekly - // run), so we derive the size list from this run's downloaded - // artifacts (storage-results--) instead of hardcoding. + // run), so we derive the size list from the size directories inside + // this run's downloaded artifacts instead of hardcoding. const sizeOrder = ['1mb', '4mb', '10mb', '16mb']; const ranSizes = new Set(); - if (fs.existsSync('artifacts')) { - for (const name of fs.readdirSync('artifacts')) { - if (!name.startsWith('storage-results-')) continue; - ranSizes.add(name.split('-').pop().toLowerCase()); + function collectSizes(dir) { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const name = entry.name.toLowerCase(); + if (sizeOrder.includes(name)) ranSizes.add(name); + else collectSizes(path.join(dir, entry.name)); } } + if (fs.existsSync('artifacts')) collectSizes('artifacts'); const sizes = sizeOrder.filter(s => ranSizes.has(s)); let body = '## Storage Benchmark Results\n\n'; let hasResults = false; diff --git a/benchmarks/ai-gateway/ai-gateway-gemini.bench.ts b/benchmarks/ai-gateway/ai-gateway-gemini.bench.ts index 364ef995..5f2e7014 100644 --- a/benchmarks/ai-gateway/ai-gateway-gemini.bench.ts +++ b/benchmarks/ai-gateway/ai-gateway-gemini.bench.ts @@ -35,29 +35,20 @@ export const config = defineBenchmarkConfig({ phases, groupBy: 'round', participants: providers, - onScore: (lowerIsBetter, higherIsBetter) => ({ + scoring: { metrics: [ - lowerIsBetter('coldE2eMs', { - unit: 'ms', - ceiling: 20000, - value: (record) => ((record.data?.phase as string | undefined) === 'cold' && typeof record.latencyMs === 'number' ? record.latencyMs : undefined), - weights: { median: 0.30, p95: 0.15, p99: 0 }, - }), - lowerIsBetter('warmTtftMs', { - unit: 'ms', - ceiling: 20000, - value: (record) => ((record.data?.phase as string | undefined) === 'warm' && typeof record.latencyMs === 'number' ? record.latencyMs : undefined), - weights: { median: 0.30, p95: 0.15, p99: 0 }, - }), - higherIsBetter('outputTokensPerSec', { + { key: 'coldE2eMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } }, + { key: 'warmTtftMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } }, + { + key: 'outputTokensPerSec', unit: 'tokens/sec', floor: 5, ceiling: 200, - value: (record) => (typeof record.data?.outputTokensPerSec === 'number' ? record.data.outputTokensPerSec : undefined), + higherIsBetter: true, weights: { median: 0.10, p95: 0, p99: 0 }, - }), + }, ], - }), + }, onComplete: (outcome) => writeAIGatewayLegacyResults(outcome.participants, { resultsDir: path.resolve(__dirname, '../../results/ai-gateway-latency/gemini'), diff --git a/benchmarks/ai-gateway/ai-gateway-kimi.bench.ts b/benchmarks/ai-gateway/ai-gateway-kimi.bench.ts index 74c1cbe8..88eeace5 100644 --- a/benchmarks/ai-gateway/ai-gateway-kimi.bench.ts +++ b/benchmarks/ai-gateway/ai-gateway-kimi.bench.ts @@ -52,29 +52,20 @@ export const config = defineBenchmarkConfig({ phases, groupBy: 'round', participants: providers, - onScore: (lowerIsBetter, higherIsBetter) => ({ + scoring: { metrics: [ - lowerIsBetter('coldE2eMs', { - unit: 'ms', - ceiling: 20000, - value: (record) => ((record.data?.phase as string | undefined) === 'cold' && typeof record.latencyMs === 'number' ? record.latencyMs : undefined), - weights: { median: 0.30, p95: 0.15, p99: 0 }, - }), - lowerIsBetter('warmTtftMs', { - unit: 'ms', - ceiling: 20000, - value: (record) => ((record.data?.phase as string | undefined) === 'warm' && typeof record.latencyMs === 'number' ? record.latencyMs : undefined), - weights: { median: 0.30, p95: 0.15, p99: 0 }, - }), - higherIsBetter('outputTokensPerSec', { + { key: 'coldE2eMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } }, + { key: 'warmTtftMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } }, + { + key: 'outputTokensPerSec', unit: 'tokens/sec', floor: 5, ceiling: 200, - value: (record) => (typeof record.data?.outputTokensPerSec === 'number' ? record.data.outputTokensPerSec : undefined), + higherIsBetter: true, weights: { median: 0.10, p95: 0, p99: 0 }, - }), + }, ], - }), + }, onComplete: (outcome) => writeAIGatewayLegacyResults(outcome.participants, { resultsDir: path.resolve(__dirname, '../../results/ai-gateway-latency/kimi'), diff --git a/benchmarks/ai-gateway/ai-gateway-openai.bench.ts b/benchmarks/ai-gateway/ai-gateway-openai.bench.ts index ccded987..4cd5c533 100644 --- a/benchmarks/ai-gateway/ai-gateway-openai.bench.ts +++ b/benchmarks/ai-gateway/ai-gateway-openai.bench.ts @@ -35,29 +35,20 @@ export const config = defineBenchmarkConfig({ phases, groupBy: 'round', participants: providers, - onScore: (lowerIsBetter, higherIsBetter) => ({ + scoring: { metrics: [ - lowerIsBetter('coldE2eMs', { - unit: 'ms', - ceiling: 20000, - value: (record) => ((record.data?.phase as string | undefined) === 'cold' && typeof record.latencyMs === 'number' ? record.latencyMs : undefined), - weights: { median: 0.30, p95: 0.15, p99: 0 }, - }), - lowerIsBetter('warmTtftMs', { - unit: 'ms', - ceiling: 20000, - value: (record) => ((record.data?.phase as string | undefined) === 'warm' && typeof record.latencyMs === 'number' ? record.latencyMs : undefined), - weights: { median: 0.30, p95: 0.15, p99: 0 }, - }), - higherIsBetter('outputTokensPerSec', { + { key: 'coldE2eMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } }, + { key: 'warmTtftMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } }, + { + key: 'outputTokensPerSec', unit: 'tokens/sec', floor: 5, ceiling: 200, - value: (record) => (typeof record.data?.outputTokensPerSec === 'number' ? record.data.outputTokensPerSec : undefined), + higherIsBetter: true, weights: { median: 0.10, p95: 0, p99: 0 }, - }), + }, ], - }), + }, onComplete: (outcome) => writeAIGatewayLegacyResults(outcome.participants, { resultsDir: path.resolve(__dirname, '../../results/ai-gateway-latency/openai'), diff --git a/benchmarks/ai-gateway/ai-gateway.bench.ts b/benchmarks/ai-gateway/ai-gateway.bench.ts index d9c99c46..0e0fc306 100644 --- a/benchmarks/ai-gateway/ai-gateway.bench.ts +++ b/benchmarks/ai-gateway/ai-gateway.bench.ts @@ -52,29 +52,20 @@ export const config = defineBenchmarkConfig({ phases, groupBy: 'round', participants: providers, - onScore: (lowerIsBetter, higherIsBetter) => ({ + scoring: { metrics: [ - lowerIsBetter('coldE2eMs', { - unit: 'ms', - ceiling: 20000, - value: (record) => ((record.data?.phase as string | undefined) === 'cold' && typeof record.latencyMs === 'number' ? record.latencyMs : undefined), - weights: { median: 0.30, p95: 0.15, p99: 0 }, - }), - lowerIsBetter('warmTtftMs', { - unit: 'ms', - ceiling: 20000, - value: (record) => ((record.data?.phase as string | undefined) === 'warm' && typeof record.latencyMs === 'number' ? record.latencyMs : undefined), - weights: { median: 0.30, p95: 0.15, p99: 0 }, - }), - higherIsBetter('outputTokensPerSec', { + { key: 'coldE2eMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } }, + { key: 'warmTtftMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } }, + { + key: 'outputTokensPerSec', unit: 'tokens/sec', floor: 5, ceiling: 200, - value: (record) => (typeof record.data?.outputTokensPerSec === 'number' ? record.data.outputTokensPerSec : undefined), + higherIsBetter: true, weights: { median: 0.10, p95: 0, p99: 0 }, - }), + }, ], - }), + }, onComplete: (outcome) => writeAIGatewayLegacyResults(outcome.participants, { resultsDir: path.resolve(__dirname, '../../results/ai-gateway-latency/anthropic'), diff --git a/benchmarks/ai-gateway/shared-task.ts b/benchmarks/ai-gateway/shared-task.ts index eb73ca8e..4e565425 100644 --- a/benchmarks/ai-gateway/shared-task.ts +++ b/benchmarks/ai-gateway/shared-task.ts @@ -3,7 +3,7 @@ * OpenAI, Gemini, Kimi). Deliberately holds only what's identical across * every family — the probe task itself, phase/step shaping, and CLI * iteration-flag parsing — not the family's own identity (`benchmarkSlug`, - * `providers`, `onScore`, `onComplete`), which each `*.bench.ts` file + * `providers`, `scoring`, `onComplete`), which each `*.bench.ts` file * declares directly via `defineBenchmarkConfig`, mirroring * `ai-gateway.bench.ts` (the Anthropic family, the original of the four). * Config/scoring intentionally isn't abstracted here even though it's @@ -85,8 +85,15 @@ function phaseSteps(result: PhaseProbeResult): TaskStepRecord[] { } function probeData(result: PhaseProbeResult): JsonObject { + // The scored latency is reported under a phase-specific key (`coldE2eMs` / + // `warmTtftMs`) so cold and warm are separate metrics without a scoring-time + // phase filter. + const scoredLatencyMs = result.mode === 'cold' ? result.coldE2eMs ?? result.ttftMs : result.ttftMs; return { mode: result.mode, + ...(typeof scoredLatencyMs === 'number' && Number.isFinite(scoredLatencyMs) + ? { [result.mode === 'cold' ? 'coldE2eMs' : 'warmTtftMs']: scoredLatencyMs } + : {}), ...(result.outputTokens !== undefined ? { outputTokens: result.outputTokens } : {}), ...(result.outputTokensPerSec !== undefined ? { outputTokensPerSec: result.outputTokensPerSec } : {}), ...(result.resolvedProvider !== undefined ? { resolvedProvider: result.resolvedProvider } : {}), diff --git a/benchmarks/browser/browser-throughput.bench.ts b/benchmarks/browser/browser-throughput.bench.ts index 85a2dc71..76ac5b2f 100644 --- a/benchmarks/browser/browser-throughput.bench.ts +++ b/benchmarks/browser/browser-throughput.bench.ts @@ -35,31 +35,24 @@ export const config = defineBenchmarkConfig({ iterations: 2, groupBy: 'round', participants: throughputProviders, - onScore: (lowerIsBetter, higherIsBetter) => ({ - success: (record) => record.status === 'success' && typeof record.data?.actionsCompleted === 'number' && record.data.actionsCompleted === ACTIONS_PER_SESSION, + scoring: { + // A session that dropped actions isn't comparable to one that completed + // them all, so it counts against the success rate and its timings are + // excluded from the metrics. + success: { requireData: { actionsCompleted: ACTIONS_PER_SESSION } }, metrics: [ - higherIsBetter('actionsPerSecond', { + { + key: 'actionsPerSecond', unit: 'actions/sec', floor: 0, ceiling: 10, + higherIsBetter: true, weights: { median: 0.40, p95: 0, p99: 0 }, - }), - lowerIsBetter('taskMs', { - unit: 'ms', - ceiling: 30000, - weights: { median: 0.25, p95: 0.20, p99: 0 }, - }), - lowerIsBetter('screenshot', { - unit: 'ms', - ceiling: 30000, - value: (record) => { - const actions = Array.isArray(record.data?.actions) ? (record.data!.actions as any[]) : []; - return actions.filter((a) => a.type === 'screenshot' && a.success).map((a) => a.durationMs); - }, - weights: { median: 0.15, p95: 0, p99: 0 }, - }), + }, + { key: 'taskMs', unit: 'ms', ceiling: 30000, weights: { median: 0.25, p95: 0.20, p99: 0 } }, + { key: 'screenshotMs', unit: 'ms', ceiling: 30000, weights: { median: 0.15, p95: 0, p99: 0 } }, ], - }), + }, onComplete: (outcome) => writeThroughputLegacyResults(outcome.participants, { resultsDir: path.resolve(__dirname, '../../results/browser-throughput'), @@ -231,6 +224,17 @@ async function runActionLoop(page: Page, results: ActionResult[], navigateUrl: s } } +/** Median duration of the session's successful screenshots, or undefined if it took none. */ +function medianScreenshotMs(actions: ActionResult[]): number | undefined { + const durations = actions + .filter((a) => a.type === 'screenshot' && a.success) + .map((a) => a.durationMs) + .sort((a, b) => a - b); + if (durations.length === 0) return undefined; + const mid = Math.floor(durations.length / 2); + return durations.length % 2 === 0 ? (durations[mid - 1] + durations[mid]) / 2 : durations[mid]; +} + /** One provider instance per participant, reused across that provider's sessions. */ const providerCache = new Map(); @@ -254,6 +258,7 @@ export const task = defineTask(async (ctx) => { let taskMs = 0; let actionsCompleted = 0; let actionsPerSecond = 0; + let screenshotMs: number | undefined; let session: { sessionId: string; connectUrl: string } | undefined; let browser: Browser | undefined; @@ -292,7 +297,8 @@ export const task = defineTask(async (ctx) => { taskMs = actions.reduce((sum, a) => sum + a.durationMs, 0); actionsCompleted = actions.filter((a) => a.success).length; actionsPerSecond = taskMs > 0 ? actionsCompleted / (taskMs / 1000) : 0; - measure({ actionsPerSecond, actionsCompleted }); + screenshotMs = medianScreenshotMs(actions); + measure({ actionsPerSecond, actionsCompleted, ...(screenshotMs !== undefined ? { screenshotMs } : {}) }); }); } catch (err) { iterationError = err instanceof Error ? err.message : String(err); @@ -329,6 +335,7 @@ export const task = defineTask(async (ctx) => { taskMs, actionsCompleted, actionsPerSecond, + ...(screenshotMs !== undefined ? { screenshotMs } : {}), actions: actions as unknown as JsonValue, ...(iterationError ? { errorMessage: iterationError } : {}), }; diff --git a/benchmarks/browser/browser.bench.ts b/benchmarks/browser/browser.bench.ts index cdc6836c..21cc9e47 100644 --- a/benchmarks/browser/browser.bench.ts +++ b/benchmarks/browser/browser.bench.ts @@ -26,12 +26,12 @@ export const config = defineBenchmarkConfig({ iterations: 2, concurrency: 1, participants: browserProviders, - onScore: (lowerIsBetter) => ({ + scoring: { metrics: [ - lowerIsBetter('totalMs', { unit: 'ms', ceiling: 10000, weights: { median: 0.40, p95: 0.20, p99: 0.10 } }), - lowerIsBetter('createMs', { unit: 'ms', ceiling: 10000, weights: { median: 0.30, p95: 0, p99: 0 } }), + { key: 'totalMs', unit: 'ms', ceiling: 10000, weights: { median: 0.40, p95: 0.20, p99: 0.10 } }, + { key: 'createMs', unit: 'ms', ceiling: 10000, weights: { median: 0.30, p95: 0, p99: 0 } }, ], - }), + }, onComplete: (outcome) => writeBrowserLegacyResults(outcome.participants, { resultsDir: path.resolve(__dirname, '../../results/browser'), diff --git a/benchmarks/sandbox/tti.bench.ts b/benchmarks/sandbox/tti.bench.ts index bccbd5c2..fc63b770 100644 --- a/benchmarks/sandbox/tti.bench.ts +++ b/benchmarks/sandbox/tti.bench.ts @@ -39,15 +39,11 @@ export const config = defineBenchmarkConfig({ iterations: 2, concurrency: 1, participants: providers, - onScore: (lowerIsBetter) => ({ + scoring: { metrics: [ - lowerIsBetter('ttiMs', { - unit: 'ms', - ceiling: 10000, - weights: { median: 0.60, p95: 0.25, p99: 0.15 }, - }), + { key: 'ttiMs', unit: 'ms', ceiling: 10000, weights: { median: 0.60, p95: 0.25, p99: 0.15 } }, ], - }), + }, // Legacy JSON labels a burst run 'concurrent' (see merge-results / // generate-svg) — that's the shape carrying the wall-clock/ramp fields. The // `results/` directory name predates this file and is kept verbatim so the diff --git a/benchmarks/storage/snapshot-fork.bench.ts b/benchmarks/storage/snapshot-fork.bench.ts index 994f0ae9..a571a898 100644 --- a/benchmarks/storage/snapshot-fork.bench.ts +++ b/benchmarks/storage/snapshot-fork.bench.ts @@ -53,16 +53,18 @@ export const config = defineBenchmarkConfig({ iterations: 2, concurrency: 1, participants, - onScore: (lowerIsBetter) => ({ - dimensions: { dataset }, - success: (record) => record.status === 'success' && record.data?.verified === true, + dimensions: { dataset }, + scoring: { + // A fork whose read-back didn't match is not a usable fork, so its timings + // must not be scored as if it were. + success: { requireData: { verified: true } }, metrics: [ - lowerIsBetter('snapshotCreateMs', { unit: 'ms', ceiling: 60000, weights: { median: 0.40, p95: 0, p99: 0 } }), - lowerIsBetter('forkFromSnapshotMs', { unit: 'ms', ceiling: 60000, weights: { median: 0.35, p95: 0, p99: 0 } }), - lowerIsBetter('forkFromLiveMs', { unit: 'ms', ceiling: 60000, weights: { median: 0.15, p95: 0, p99: 0 } }), - lowerIsBetter('forkFirstReadMs', { unit: 'ms', ceiling: 60000, weights: { median: 0.10, p95: 0, p99: 0 } }), + { key: 'snapshotCreateMs', unit: 'ms', ceiling: 60000, weights: { median: 0.40, p95: 0, p99: 0 } }, + { key: 'forkFromSnapshotMs', unit: 'ms', ceiling: 60000, weights: { median: 0.35, p95: 0, p99: 0 } }, + { key: 'forkFromLiveMs', unit: 'ms', ceiling: 60000, weights: { median: 0.15, p95: 0, p99: 0 } }, + { key: 'forkFirstReadMs', unit: 'ms', ceiling: 60000, weights: { median: 0.10, p95: 0, p99: 0 } }, ], - }), + }, onComplete: (outcome) => writeSnapshotForkLegacyResults(outcome.participants, { resultsDir: path.resolve(__dirname, `../../results/snapshot-fork/${dataset}`), diff --git a/benchmarks/storage/storage.bench.ts b/benchmarks/storage/storage.bench.ts index b330c387..c7c96c9d 100644 --- a/benchmarks/storage/storage.bench.ts +++ b/benchmarks/storage/storage.bench.ts @@ -6,12 +6,13 @@ * * bench run benchmarks/storage/storage.bench.ts * bench run benchmarks/storage/storage.bench.ts --file-size 10MB --iterations 5 --provider aws-s3 + * bench run benchmarks/storage/storage.bench.ts --file-size 1MB,10MB --provider aws-s3 */ import '../src/env.js'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import crypto from 'node:crypto'; -import { defineBenchmarkConfig, defineTask, TaskError } from '@benchsdk/runner'; +import { defineBenchmarkConfig, defineTask, TaskError, type BenchmarkRunOutcome } from '@benchsdk/runner'; import type { Storage } from '@storagesdk/core'; import { withTimeout } from '../src/util/timeout.js'; import { formatError } from '../src/util/error.js'; @@ -23,44 +24,83 @@ import type { StorageFileSize, StorageProviderConfig } from './types.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // --file-size is unknown to @benchsdk/runner and passes through untouched, so we -// parse it ourselves from process.argv (default '10MB', matching run.ts). +// parse it ourselves from process.argv. Accept a comma-separated list to run +// multiple file sizes in a single benchmark run; each size becomes a phase and +// the scoring spec groups summary rows by `file_size`. const args = process.argv.slice(2); -function getArgValue(argv: string[], flag: string): string | undefined { +function getArgValues(argv: string[], flag: string): string[] | undefined { const idx = argv.indexOf(flag); - return idx !== -1 && idx + 1 < argv.length ? argv[idx + 1] : undefined; + if (idx === -1 || idx + 1 >= argv.length) return undefined; + const values = argv[idx + 1].split(',').map((s) => s.trim()).filter(Boolean); + // A flag that was passed but names no size is an invalid size, not an absent + // flag: keep the raw value so validation below rejects it by name. + return values.length > 0 ? values : [argv[idx + 1]]; } -const fileSizeArg = getArgValue(args, '--file-size') ?? '10MB'; -const validSizes = Object.keys(FILE_SIZE_BYTES) as StorageFileSize[]; -if (!(fileSizeArg in FILE_SIZE_BYTES)) { - console.error(`Invalid --file-size "${fileSizeArg}". Valid sizes: ${validSizes.join(', ')}`); - process.exit(1); +const fileSizeArgs = getArgValues(args, '--file-size') ?? ['10MB']; +for (const size of fileSizeArgs) { + if (!(size in FILE_SIZE_BYTES)) { + const validSizes = Object.keys(FILE_SIZE_BYTES); + console.error(`Invalid --file-size "${size}". Valid sizes: ${validSizes.join(', ')}`); + process.exit(1); + } } -const fileSizeLabel = fileSizeArg as StorageFileSize; -const fileSizeBytes = FILE_SIZE_BYTES[fileSizeLabel]; - -const testData = crypto.randomBytes(fileSizeBytes); +const fileSizes = fileSizeArgs as StorageFileSize[]; +const defaultFileSize = fileSizes[0]; +const isMultiSize = fileSizes.length > 1; -export const config = defineBenchmarkConfig({ +const baseConfig = { benchmarkSlug: `storage-lifecycle${process.env.DAILY_BENCH_SLUG ? `-${process.env.DAILY_BENCH_SLUG}` : ''}`, benchmarkName: `Storage Lifecycle${process.env.DAILY_BENCH_NAME ? ` - ${process.env.DAILY_BENCH_NAME}` : ''}`, - iterations: 2, concurrency: 1, participants: storageProviders, - onScore: (lowerIsBetter, higherIsBetter) => ({ - dimensions: { file_size: fileSizeLabel }, + scoring: { + groupBy: 'file_size', metrics: [ - lowerIsBetter('uploadMs', { unit: 'ms', ceiling: 30000, weights: { median: 0.25, p95: 0.10, p99: 0.05 } }), - lowerIsBetter('downloadMs', { unit: 'ms', ceiling: 30000, weights: { median: 0.35, p95: 0.15, p99: 0.05 } }), - higherIsBetter('throughputMbps', { unit: 'mbps', floor: 1, ceiling: 1000, weights: { median: 0.05, p95: 0, p99: 0 } }), + { key: 'uploadMs', unit: 'ms', ceiling: 30000, weights: { median: 0.25, p95: 0.10, p99: 0.05 } }, + { key: 'downloadMs', unit: 'ms', ceiling: 30000, weights: { median: 0.35, p95: 0.15, p99: 0.05 } }, + { key: 'throughputMbps', unit: 'mbps', floor: 1, ceiling: 1000, higherIsBetter: true, weights: { median: 0.05, p95: 0, p99: 0 } }, ], - }), - onComplete: (outcome) => + }, +}; + +const singleSizeConfig = { + ...baseConfig, + iterations: 2, + dimensions: { file_size: defaultFileSize }, + // A single-size run needs the `file_size` tag but not a separate group row, + // otherwise the run-wide aggregate and the group row have identical dimensions + // and identical metrics and get reported twice. + scoring: { metrics: baseConfig.scoring.metrics }, + onComplete: (outcome: BenchmarkRunOutcome) => writeStorageLegacyResults(outcome.participants, { - resultsDir: path.resolve(__dirname, `../../results/storage/${fileSizeLabel.toLowerCase()}`), - fileSizeBytes, + resultsDir: path.resolve(__dirname, `../../results/storage/${defaultFileSize.toLowerCase()}`), + fileSizeBytes: FILE_SIZE_BYTES[defaultFileSize], providers: storageProviders, }), -}); +}; + +const multiSizeConfig = { + ...baseConfig, + phases: fileSizes.map((size) => ({ name: size, iterations: 2 })), + // The legacy results tree is one directory per size, so a multi-size run + // splits its records back out by `file_size` and writes each size's + // directory as a single-size run would. + onComplete: async (outcome: BenchmarkRunOutcome) => { + for (const size of fileSizes) { + const participants = outcome.participants.map((p) => ({ + ...p, + records: p.records.filter((r) => (r.data as { file_size?: string } | undefined)?.file_size === size), + })); + await writeStorageLegacyResults(participants, { + resultsDir: path.resolve(__dirname, `../../results/storage/${size.toLowerCase()}`), + fileSizeBytes: FILE_SIZE_BYTES[size], + providers: storageProviders, + }); + } + }, +}; + +export const config = defineBenchmarkConfig(isMultiSize ? multiSizeConfig : singleSizeConfig); function randomId(): string { return Math.random().toString(36).substring(2, 15); @@ -76,13 +116,28 @@ export const task = defineTask(async (ctx) => { const { participant, step, measure } = ctx; const timeout = participant.timeout ?? 30_000; + // When running multiple file sizes per run, each phase is named after the size. + const fileSizeLabel = (ctx.phase as StorageFileSize | undefined) ?? defaultFileSize; + const fileSizeBytes = FILE_SIZE_BYTES[fileSizeLabel]; + let storage = storageCache.get(participant.name); if (!storage) { storage = participant.createStorage(); storageCache.set(participant.name, storage); } + // Measured up front rather than only returned: on failure the thrown data is + // not what lands on the record in participant mode, and the scoring spec + // groups by `file_size` — an untagged failure would form its own group and + // leave the real group looking fully successful. + measure({ file_size: fileSizeLabel, fileSizeBytes }); + const key = `benchmark-${Date.now()}-${randomId()}`; + const testData = crypto.randomBytes(fileSizeBytes); + + let uploadMs = 0; + let downloadMs = 0; + let throughputMbps = 0; try { // Upload timing @@ -90,14 +145,12 @@ export const task = defineTask(async (ctx) => { await step('upload', () => withTimeout(storage!.upload(key, testData), timeout, 'Upload timed out'), ); - const uploadMs = performance.now() - uploadStart; + uploadMs = performance.now() - uploadStart; // Download timing — request raw bytes so we measure a full object fetch. // Throughput (Mbps) is a rate, not a duration, so it can't be inferred // from the step's latency; measure it inside the `download` step so it // lands on that step's data (platform step_data_json). - let downloadMs = 0; - let throughputMbps = 0; await step('download', async () => { const downloadStart = performance.now(); await withTimeout(storage!.download(key, { as: 'bytes' }), timeout, 'Download timed out'); @@ -113,7 +166,7 @@ export const task = defineTask(async (ctx) => { { reportConcurrency: false }, ).catch((err) => console.warn(` [cleanup] delete failed: ${formatError(err)}`)); - return { data: { uploadMs, downloadMs, throughputMbps, fileSizeBytes } }; + return { data: { file_size: fileSizeLabel, uploadMs, downloadMs, throughputMbps, fileSizeBytes } }; } catch (err) { // Attempt cleanup even on failure. try { @@ -124,7 +177,7 @@ export const task = defineTask(async (ctx) => { const message = formatError(err); throw new TaskError(message, { code: 'STORAGE_ERROR', - data: { uploadMs: 0, downloadMs: 0, throughputMbps: 0, fileSizeBytes }, + data: { file_size: fileSizeLabel, uploadMs: 0, downloadMs: 0, throughputMbps: 0, fileSizeBytes }, }); } }); diff --git a/packages/benchsdk-api/src/types.ts b/packages/benchsdk-api/src/types.ts index 98a24716..7fd0ad67 100644 --- a/packages/benchsdk-api/src/types.ts +++ b/packages/benchsdk-api/src/types.ts @@ -635,6 +635,7 @@ export interface BenchmarkRunSummaryRunMetadata { export interface BenchmarkRunSummaryInput { run: BenchmarkRunSummaryRunMetadata; results: BenchmarkRunSummaryResult[]; + scoring?: JsonObject; } export interface BenchmarkClient { diff --git a/packages/benchsdk-runner/package.json b/packages/benchsdk-runner/package.json index 686ba7af..2f8038b9 100644 --- a/packages/benchsdk-runner/package.json +++ b/packages/benchsdk-runner/package.json @@ -3,7 +3,7 @@ "version": "0.2.0", "private": false, "type": "module", - "description": "Benchmark framework: task + step + config primitives and the local orchestrator that drives @benchsdk/client", + "description": "Benchmark framework: task + step + config primitives and the local orchestrator that drives @benchsdk/worker", "author": "Garrison", "license": "MIT", "main": "./dist/index.cjs", @@ -52,7 +52,8 @@ "node": ">=18.0.0" }, "dependencies": { - "@benchsdk/client": "workspace:*" + "@benchsdk/api": "workspace:*", + "@benchsdk/worker": "workspace:*" }, "devDependencies": { "@types/node": "^20.0.0", diff --git a/packages/benchsdk-runner/src/__tests__/bench-config.test.ts b/packages/benchsdk-runner/src/__tests__/bench-config.test.ts index 5c59d0cd..ab5fcfca 100644 --- a/packages/benchsdk-runner/src/__tests__/bench-config.test.ts +++ b/packages/benchsdk-runner/src/__tests__/bench-config.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import { defineBenchmarkConfig, defineTask, TaskError } from '../bench-config'; -import type { BaseParticipant } from '@benchsdk/client'; +import type { BaseParticipant } from '@benchsdk/worker'; const participants: BaseParticipant[] = [{ name: 'e2b', requiredEnvVars: [] }]; diff --git a/packages/benchsdk-runner/src/__tests__/runner.test.ts b/packages/benchsdk-runner/src/__tests__/runner.test.ts index 51b0ccdf..e51f2baa 100644 --- a/packages/benchsdk-runner/src/__tests__/runner.test.ts +++ b/packages/benchsdk-runner/src/__tests__/runner.test.ts @@ -1,12 +1,15 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const createBenchmarkClient = vi.fn(); +const runWorker = vi.fn(); const reporterClaim = vi.fn(); -vi.mock('@benchsdk/client', () => ({ +vi.mock('@benchsdk/api', () => ({ createBenchmarkClient: (...args: unknown[]) => createBenchmarkClient(...args), +})); + +vi.mock('@benchsdk/worker', () => ({ BenchmarkReporter: { claim: (...args: unknown[]) => reporterClaim(...args) }, - selectParticipants: (all: any[], names?: string[]) => (names ? all.filter((p) => names.includes(p.name)) : all), filterParticipantsByEnv: (ps: any[]) => { const available: any[] = []; const skipped: { name: string; missing: string[] }[] = []; @@ -17,13 +20,15 @@ vi.mock('@benchsdk/client', () => ({ } return { available, skipped }; }, + runWorker: (...args: unknown[]) => runWorker(...args), + selectParticipants: (all: any[], names?: string[]) => (names ? all.filter((p) => names.includes(p.name)) : all), })); import { parseCliArgs, mergeConfig, runBenchmark } from '../runner'; import { TaskError, defineTask } from '../bench-config'; import { NoAvailableParticipantsError } from '../no-available-participants'; import type { BenchmarkConfig } from '../bench-config'; -import type { TaskResultRecord } from '@benchsdk/client'; +import type { TaskResultRecord } from '@benchsdk/api'; describe('parseCliArgs', () => { it('parses space-separated flags', () => { @@ -152,7 +157,20 @@ describe('mergeConfig', () => { expect(mergeConfig(withDefaults, { providers: ['modal'] }).providers).toEqual(['modal']); }); - it('derives iterations from phases (sum) and ignores --iterations with a warning', () => { + it('derives iterations from phases (sum), and applies --iterations to each phase', () => { + const phased: BenchmarkConfig = { + benchmarkSlug: 's', + benchmarkName: 'n', + phases: [{ name: '1MB', iterations: 2 }, { name: '16MB', iterations: 2 }], + participants: [], + }; + expect(mergeConfig(phased, {})).toMatchObject({ iterations: 4, phaseIterations: undefined }); + // Phases are the arms of one comparison, so the flag scales every arm + // rather than being split between them. + expect(mergeConfig(phased, { iterations: 10 })).toMatchObject({ iterations: 20, phaseIterations: 10 }); + }); + + it('keeps individually sized phases over --iterations, with a warning', () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); const phased: BenchmarkConfig = { benchmarkSlug: 's', @@ -160,8 +178,7 @@ describe('mergeConfig', () => { phases: [{ name: 'cold', iterations: 3 }, { name: 'warm', iterations: 2 }], participants: [], }; - expect(mergeConfig(phased, {}).iterations).toBe(5); - expect(mergeConfig(phased, { iterations: 99 }).iterations).toBe(5); + expect(mergeConfig(phased, { iterations: 99 })).toMatchObject({ iterations: 5, phaseIterations: undefined }); expect(warn).toHaveBeenCalled(); warn.mockRestore(); }); @@ -181,6 +198,7 @@ describe('runBenchmark', () => { taskRangeStart = 0; vi.restoreAllMocks(); reporterClaim.mockReset(); + runWorker.mockReset(); vi.spyOn(console, 'log').mockImplementation(() => {}); vi.spyOn(console, 'error').mockImplementation(() => {}); vi.spyOn(console, 'warn').mockImplementation(() => {}); @@ -198,41 +216,41 @@ describe('runBenchmark', () => { calls.getRun.push([slug, runId]); return { id: runId, totalTasks: 3, participantSized: runId === 'run-open' }; }), - runWorker: vi.fn(async (opts: any) => { - calls.runWorker.push(opts); - const total = calls.createRun[0]?.[1]?.totalTasks ?? calls.upsertParticipant[0]?.[3]?.totalTasks ?? 1; - // The platform hands out globally-indexed task ranges; `taskRangeStart` - // lets a test exercise a worker whose range doesn't start at 0. - const start = taskRangeStart; - const assignment = { workerId: 'w1', taskRange: { start, end: start + total - 1, count: total } }; - const records: any[] = []; - for (let ti = start; ti < start + total; ti++) { - // Mirror the real client worker: `measure` merges into the record's - // data alongside whatever the task returns. `step` records options - // so participant-mode option forwarding can be asserted. - const measures: Record = {}; - const steps: any[] = []; - const ctx = { - taskIndex: ti, - assignment, - step: async (_n: string, fn: any, options: any) => { - const value = await fn(); - steps.push({ name: _n, options }); - return value; - }, - measure: (d: Record) => Object.assign(measures, d), - log: () => {}, - }; - const returned = await opts.task(ctx); - calls.taskData.push(returned); - const data = { ...measures, ...(returned ?? {}) }; - const rec = { taskIndex: ti, status: 'success', data, steps }; - opts.onResult?.(rec); - records.push(rec); - } - return { assignment, records }; - }), }; + runWorker.mockImplementation(async (_client: any, opts: any) => { + calls.runWorker.push(opts); + const total = calls.createRun[0]?.[1]?.totalTasks ?? calls.upsertParticipant[0]?.[3]?.totalTasks ?? 1; + // The platform hands out globally-indexed task ranges; `taskRangeStart` + // lets a test exercise a worker whose range doesn't start at 0. + const start = taskRangeStart; + const assignment = { workerId: 'w1', taskRange: { start, end: start + total - 1, count: total } }; + const records: any[] = []; + for (let ti = start; ti < start + total; ti++) { + // Mirror the real client worker: `measure` merges into the record's + // data alongside whatever the task returns. `step` records options + // so participant-mode option forwarding can be asserted. + const measures: Record = {}; + const steps: any[] = []; + const ctx = { + taskIndex: ti, + assignment, + step: async (_n: string, fn: any, options: any) => { + const value = await fn(); + steps.push({ name: _n, options }); + return value; + }, + measure: (d: Record) => Object.assign(measures, d), + log: () => {}, + }; + const returned = await opts.task(ctx); + calls.taskData.push(returned); + const data = { ...measures, ...(returned ?? {}) }; + const rec = { taskIndex: ti, status: 'success', data, steps }; + opts.onResult?.(rec); + records.push(rec); + } + return { assignment, records }; + }); createBenchmarkClient.mockReturnValue(fakeClient); }); @@ -546,6 +564,73 @@ describe('runBenchmark', () => { ]); }); + it('participant mode: --iterations runs that many iterations of every phase', async () => { + const seenPhases: (string | undefined)[] = []; + const task = vi.fn(async (ctx: any) => { + seenPhases.push(ctx.phase); + return {}; + }); + + await runBenchmark( + { + benchmarkSlug: 's', + benchmarkName: 'n', + phases: [{ name: '1MB', iterations: 2 }, { name: '16MB', iterations: 2 }], + participants: [participants[0]], + }, + defineTask(task), + ['--iterations', '3'], + ); + + expect(calls.createRun[0][1].totalTasks).toBe(6); + expect(seenPhases).toEqual(['1MB', '1MB', '1MB', '16MB', '16MB', '16MB']); + }); + + it('participant mode: a failing task keeps its phase tag and TaskError data on the record', async () => { + // Mirrors the real worker's failure path: measures survive a thrown task, + // the task's return value does not. + runWorker.mockImplementation(async (_client: any, opts: any) => { + const assignment = { workerId: 'w1', taskRange: { start: 0, end: 0, count: 1 } }; + const measures: Record = {}; + const ctx = { + taskIndex: 0, + assignment, + step: async (_n: string, fn: any) => fn(), + measure: (d: Record) => Object.assign(measures, d), + log: () => {}, + }; + let status = 'success'; + try { + await opts.task(ctx); + } catch { + status = 'error'; + } + const rec = { taskIndex: 0, status, data: { ...measures } }; + opts.onResult?.(rec); + return { assignment, records: [rec] }; + }); + + const task = defineTask(async () => { + throw new TaskError('boom', { code: 'storage_error', data: { file_size: '1MB' } }); + }); + + const outcome = await runBenchmark( + { + benchmarkSlug: 's', + benchmarkName: 'n', + phases: [{ name: '1MB', iterations: 1 }], + participants: [participants[0]], + }, + task, + [], + ); + + expect(outcome.participants[0].records[0]).toMatchObject({ + status: 'error', + data: { phase: '1MB', file_size: '1MB' }, + }); + }); + it('groupBy round: claims one reporter per participant, interleaves rounds, finishes each', async () => { const recorded: Record = { e2b: [], modal: [] }; const finished: Record = {}; @@ -588,7 +673,7 @@ describe('runBenchmark', () => { expect(finished.modal).toBe(false); // runWorker is NOT used in round mode; the single worker per participant is // planned for every task in the schedule, not just one. - expect(fakeClient.runWorker).not.toHaveBeenCalled(); + expect(runWorker).not.toHaveBeenCalled(); expect(fakeClient.planWorkers).toHaveBeenCalledTimes(2); expect(calls.planWorkers[0][3]).toMatchObject({ workerCount: 1, targetConcurrency: 2 }); }); @@ -1018,7 +1103,7 @@ describe('runBenchmark', () => { expect(createBenchmarkClient).not.toHaveBeenCalled(); expect(fakeClient.upsertBenchmark).not.toHaveBeenCalled(); expect(fakeClient.createRun).not.toHaveBeenCalled(); - expect(fakeClient.runWorker).not.toHaveBeenCalled(); + expect(runWorker).not.toHaveBeenCalled(); expect(fakeClient.submitRunSummary).not.toHaveBeenCalled(); expect(outcome.runId).toBe('no-ingest'); expect(outcome.dashboardUrl).toBe(''); diff --git a/packages/benchsdk-runner/src/__tests__/scoring.test.ts b/packages/benchsdk-runner/src/__tests__/scoring.test.ts index 575433ae..614710e1 100644 --- a/packages/benchsdk-runner/src/__tests__/scoring.test.ts +++ b/packages/benchsdk-runner/src/__tests__/scoring.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from 'vitest'; -import { validateScoringSpec, ScoringSpecError, lowerIsBetter, higherIsBetter } from '../scoring'; +import { score, validateScoringSpec, ScoringSpecError, lowerIsBetter, higherIsBetter, scoringConfigToSpec } from '../scoring'; import type { ScoringSpec } from '../scoring'; +import type { BenchmarkRunOutcome } from '../bench-config'; +import type { TaskResultRecord } from '@benchsdk/api'; describe('validateScoringSpec', () => { it('does not throw when declared weights sum to 1.0 across all metrics', () => { @@ -59,3 +61,125 @@ describe('validateScoringSpec', () => { expect(() => validateScoringSpec(spec)).toThrow('(no metrics declared)'); }); }); + +describe('score with groupBy', () => { + const spec: ScoringSpec = { + groupBy: 'file_size', + metrics: [lowerIsBetter('uploadMs', { unit: 'ms', ceiling: 1000, weights: { median: 1, p95: 0, p99: 0 } })], + }; + + function record(taskIndex: number, status: string, data: Record): TaskResultRecord { + return { taskIndex, status, data: data as TaskResultRecord['data'] }; + } + + it('emits one row per group value', () => { + const results = score( + { + participants: [ + { + participant: 'aws-s3', + records: [ + record(0, 'success', { file_size: '1MB', uploadMs: 100 }), + record(1, 'success', { file_size: '4MB', uploadMs: 400 }), + ], + }, + ], + } as unknown as BenchmarkRunOutcome, + spec, + ); + + const grouped = results.filter((r) => r.dimensions?.file_size != null); + expect(grouped.map((r) => r.dimensions.file_size)).toEqual(['1MB', '4MB']); + expect(grouped.every((r) => r.successRate === 1)).toBe(true); + // Run-wide aggregate across every group. + expect(results.find((r) => r.dimensions?.file_size == null)?.successRate).toBe(1); + }); + + it('counts a failure against its own group instead of a separate row', () => { + const results = score( + { + participants: [ + { + participant: 'aws-s3', + records: [ + record(0, 'success', { file_size: '1MB', uploadMs: 100 }), + record(1, 'error', { file_size: '1MB' }), + ], + }, + ], + } as unknown as BenchmarkRunOutcome, + spec, + ); + + const groupRow = results.find((r) => r.dimensions?.file_size === '1MB'); + expect(groupRow).toBeDefined(); + expect(groupRow!.successRate).toBe(0.5); + // Run-wide aggregate spans both records. + expect(results.find((r) => r.dimensions?.file_size == null)?.successRate).toBe(0.5); + }); + + it('keeps a participant with no records as a skipped row', () => { + const results = score( + { participants: [{ participant: 'aws-s3', records: [] }] } as unknown as BenchmarkRunOutcome, + spec, + ); + + expect(results).toHaveLength(1); + expect(results[0]).toMatchObject({ provider: 'aws-s3', skipped: true, successRate: 0 }); + }); +}); + +describe('scoringConfigToSpec success rule', () => { + const spec = scoringConfigToSpec({ + success: { requireData: { verified: true } }, + metrics: [{ key: 'forkMs', unit: 'ms', ceiling: 1000, weights: { median: 1, p95: 0, p99: 0 } }], + }); + + function record(taskIndex: number, status: string, data: Record): TaskResultRecord { + return { taskIndex, status, data: data as TaskResultRecord['data'] }; + } + + it('excludes a record whose required data field does not match', () => { + const results = score( + { + participants: [ + { + participant: 'tigris', + records: [ + record(0, 'success', { verified: true, forkMs: 100 }), + record(1, 'success', { verified: false, forkMs: 900 }), + ], + }, + ], + } as unknown as BenchmarkRunOutcome, + spec, + ); + + expect(results[0].successRate).toBe(0.5); + // Only the verified record's timing is aggregated. + expect(results[0].metrics[0].median).toBe(100); + }); + + it('counts every successful record when no success rule is declared', () => { + const plain = scoringConfigToSpec({ + metrics: [{ key: 'forkMs', unit: 'ms', ceiling: 1000, weights: { median: 1, p95: 0, p99: 0 } }], + }); + const results = score( + { + participants: [ + { + participant: 'tigris', + records: [ + record(0, 'success', { verified: true, forkMs: 100 }), + record(1, 'success', { verified: false, forkMs: 900 }), + ], + }, + ], + } as unknown as BenchmarkRunOutcome, + plain, + ); + + expect(results[0].successRate).toBe(1); + expect(results[0].metrics[0].median).toBe(500); + }); +}); diff --git a/packages/benchsdk-runner/src/bench-config.ts b/packages/benchsdk-runner/src/bench-config.ts index 289a0fb6..3bf0ca4f 100644 --- a/packages/benchsdk-runner/src/bench-config.ts +++ b/packages/benchsdk-runner/src/bench-config.ts @@ -31,13 +31,13 @@ * recorded. */ import type { - BaseParticipant, DefineStepOptions, JsonObject, TaskResultRecord, TaskStepRecord, -} from '@benchsdk/client'; -import type { HigherIsBetter, LowerIsBetter, ScoringSpec } from './scoring.js'; +} from '@benchsdk/api'; +import type { BaseParticipant } from '@benchsdk/worker'; +import type { BenchmarkScoringConfig, HigherIsBetter, LowerIsBetter, ScoringSpec } from './scoring.js'; /** How tasks are ordered across participants. */ export type GroupBy = 'participant' | 'round'; @@ -71,7 +71,7 @@ export interface TaskResult { * Pre-measured steps the task timed itself (e.g. socket phases). * Only honored in `groupBy: 'round'` runs, where the runner builds records * manually. In `groupBy: 'participant'` runs the platform worker - * (`client.runWorker`) owns steps, so `steps` and `latencyMs` are ignored. + * (`runWorker`) owns steps, so `steps` and `latencyMs` are ignored. */ steps?: TaskStepRecord[]; /** Task-owned overall latency; overrides framework wall-clock (round mode only). */ @@ -112,7 +112,7 @@ export interface TaskContext { /** Current phase name, when the benchmark declares `phases`. */ phase?: string; /** - * Runs `fn` as a named platform step. Mirrors `@benchsdk/client`'s + * Runs `fn` as a named platform step. Mirrors `@benchsdk/worker`'s * `RunWorkerContext.step`; supports closures and try/finally. A `concurrency` * greater than 1 invokes `fn` that many times in parallel and returns an array. * `timeoutMs` aborts any invocation that exceeds it with a `step_timeout` TaskError. @@ -156,6 +156,13 @@ export interface ParticipantRecords { /** The orchestration knobs a run actually used, after CLI overrides. */ export interface ResolvedRunConfig { iterations: number; + /** + * Iterations each phase runs, when the benchmark declares `phases` and + * `--iterations` overrode their configured counts. A phase is one arm of a + * comparison (a file size), so the flag scales every arm equally rather than + * dividing a total between them. + */ + phaseIterations?: number; concurrency: number; staggerDelayMs: number; groupBy: GroupBy; @@ -225,6 +232,12 @@ export interface BenchmarkConfig { defaultProviders?: string[]; /** The participants this benchmark can run against. `--provider` selects a subset by name. */ participants: T[]; + /** + * Static run-level dimensions copied into the submitted summary (e.g. + * `{ file_size: '10MB' }`). Useful for distinguishing runs of the same + * benchmark that differ by an external parameter. + */ + dimensions?: Record; /** * Run-level scoring hook, called once with `lowerIsBetter` and `higherIsBetter` * primitives after the outcome is assembled but before `onComplete`. Use it to @@ -237,6 +250,12 @@ export interface BenchmarkConfig { * writers). This is the run-level counterpart to per-step `ctx.measure`. */ onComplete?: (outcome: BenchmarkRunOutcome) => void | Promise; + /** + * Serializable scoring spec uploaded to the platform. When provided without + * `onScore`, the runner computes the run summary from this spec automatically. + * The platform can recompute `compositeScore` from the same spec at read time. + */ + scoring?: BenchmarkScoringConfig; } function assertPositiveInt(value: number | undefined, field: string): void { @@ -246,6 +265,75 @@ function assertPositiveInt(value: number | undefined, field: string): void { } } +function assertFiniteNumber(value: unknown, field: string): number { + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error(`${field} must be a finite number (got ${value})`); + } + return value; +} + +function validateBenchmarkScoringConfig(scoring: BenchmarkScoringConfig): void { + if (!Array.isArray(scoring.metrics) || scoring.metrics.length === 0) { + throw new Error('scoring.metrics must be a non-empty array'); + } + if (scoring.success !== undefined) { + const requireData = scoring.success.requireData; + if (requireData === null || typeof requireData !== 'object' || Array.isArray(requireData)) { + throw new Error('scoring.success.requireData must be a plain object'); + } + if (Object.keys(requireData).length === 0) { + throw new Error('scoring.success.requireData must declare at least one data field'); + } + for (const [key, value] of Object.entries(requireData)) { + const type = typeof value; + if (type !== 'string' && type !== 'number' && type !== 'boolean') { + throw new Error( + `scoring.success.requireData.${key} must be a string, number, or boolean (got ${type})`, + ); + } + } + } + const seen = new Set(); + let totalWeight = 0; + for (let i = 0; i < scoring.metrics.length; i++) { + const metric = scoring.metrics[i]; + if (metric === null || typeof metric !== 'object' || Array.isArray(metric)) { + throw new Error(`scoring.metrics[${i}] must be an object`); + } + const key = metric.key; + if (typeof key !== 'string' || key.trim() === '') { + throw new Error(`scoring.metrics[${i}].key must be a non-empty string`); + } + if (seen.has(key)) { + throw new Error(`duplicate scoring metric key: ${key}`); + } + seen.add(key); + if (typeof metric.unit !== 'string' || metric.unit.trim() === '') { + throw new Error(`scoring.metrics[${i}].unit must be a non-empty string`); + } + assertFiniteNumber(metric.ceiling, `scoring.metrics[${i}].ceiling`); + if (metric.floor !== undefined) { + assertFiniteNumber(metric.floor, `scoring.metrics[${i}].floor`); + } + if (metric.weights === null || typeof metric.weights !== 'object' || Array.isArray(metric.weights)) { + throw new Error(`scoring.metrics[${i}].weights must be an object`); + } + const median = assertFiniteNumber(metric.weights.median, `scoring.metrics[${i}].weights.median`); + const p95 = assertFiniteNumber(metric.weights.p95, `scoring.metrics[${i}].weights.p95`); + const p99 = assertFiniteNumber(metric.weights.p99, `scoring.metrics[${i}].weights.p99`); + if (median < 0 || p95 < 0 || p99 < 0) { + throw new Error(`scoring.metrics[${i}].weights must be non-negative`); + } + totalWeight += median + p95 + p99; + if (metric.trim !== undefined) { + assertFiniteNumber(metric.trim, `scoring.metrics[${i}].trim`); + } + } + if (Math.abs(totalWeight - 1) > 0.01) { + throw new Error(`scoring metric weights must sum to 1.0 (got ${totalWeight.toFixed(3)})`); + } +} + /** Validates `config` at file-evaluation time so mistakes surface immediately. */ export function defineBenchmarkConfig( config: BenchmarkConfig, @@ -296,6 +384,14 @@ export function defineBenchmarkConfig( config: BenchmarkConfig, args: CliArgs, ): ResolvedRunConfig { - const phaseTotal = config.phases?.reduce((sum, p) => sum + p.iterations, 0); - if (phaseTotal !== undefined && args.iterations !== undefined) { - console.warn('--iterations is ignored because this benchmark declares phases.'); + // `--iterations` applies per phase: phases are the arms of one comparison, so + // scaling them equally keeps the arms comparable, where splitting a total + // between them would shrink each arm as arms are added. A benchmark that + // sizes its arms differently (cold probes more than warm) meant that + // difference, so its counts win over the flag. + const phases = config.phases; + const unevenPhases = phases !== undefined && phases.some((p) => p.iterations !== phases[0].iterations); + if (unevenPhases && args.iterations !== undefined) { + console.warn('--iterations is ignored because this benchmark sizes its phases individually.'); } + const phaseIterations = phases !== undefined && !unevenPhases ? args.iterations : undefined; + const phaseTotal = + phases !== undefined + ? (phaseIterations !== undefined + ? phaseIterations * phases.length + : phases.reduce((sum, p) => sum + p.iterations, 0)) + : undefined; const resolved: ResolvedRunConfig = { iterations: phaseTotal ?? args.iterations ?? config.iterations ?? 1, + phaseIterations, concurrency: args.concurrency ?? config.concurrency ?? 1, staggerDelayMs: args.staggerDelayMs ?? config.staggerDelayMs ?? 0, groupBy: args.groupBy ?? config.groupBy ?? 'participant', @@ -306,21 +321,24 @@ interface Slot { /** * Flattens a config into an ordered list of task slots. With `phases`, each - * phase contributes `iterations` slots tagged with its name (framework owns - * the phase boundary — no index arithmetic in the task). Without phases, the - * task is repeated `iterations` times. + * phase contributes its own iterations' worth of slots tagged with its name + * (framework owns the phase boundary — no index arithmetic in the task). + * Without phases, the task is repeated `iterations` times. */ function buildSchedule( config: BenchmarkConfig, - iterations: number, + resolved: ResolvedRunConfig, task: BenchmarkTask, ): Slot[] { if (config.phases?.length) { return config.phases.flatMap((phase) => - Array.from({ length: phase.iterations }, () => ({ phase: phase.name, task })), + Array.from({ length: resolved.phaseIterations ?? phase.iterations }, () => ({ + phase: phase.name, + task, + })), ); } - return Array.from({ length: iterations }, () => ({ phase: undefined, task })); + return Array.from({ length: resolved.iterations }, () => ({ phase: undefined, task })); } type OnResult = (record: TaskResultRecord, meta: { iterations: number; participant: string }) => void; @@ -416,6 +434,32 @@ function resolveParticipants(config: BenchmarkConfig< return available; } +/** Builds a JSON-serializable snapshot of the resolved run execution config. */ +function runConfigToJson( + config: BenchmarkConfig, + resolved: ResolvedRunConfig, + participants: string[], +): JsonObject { + const phases = config.phases?.map((phase) => ({ + name: phase.name, + iterations: resolved.phaseIterations ?? phase.iterations, + })); + const runConfig = { + benchmarkSlug: config.benchmarkSlug, + benchmarkName: config.benchmarkName, + ...(resolved.phaseIterations !== undefined ? { phaseIterations: resolved.phaseIterations } : {}), + ...(phases ? { phases } : {}), + ...(!config.phases ? { iterations: resolved.iterations } : {}), + concurrency: resolved.concurrency, + staggerDelayMs: resolved.staggerDelayMs, + groupBy: resolved.groupBy, + ...(config.dimensions ? { dimensions: config.dimensions } : {}), + ...(config.scoring ? { scoring: config.scoring } : {}), + participants, + }; + return JSON.parse(JSON.stringify(runConfig)) as JsonObject; +} + /** * Runs `config`'s `task` against its participants. Selects participants by * `--provider` (if given), env-gates them, then drives them per the resolved @@ -445,7 +489,7 @@ export async function runBenchmark( client = createBenchmarkClient({ baseUrl, apiKey }); } - const schedule = buildSchedule(config, resolved.iterations, task); + const schedule = buildSchedule(config, resolved, task); const totalTasks = schedule.length; const concurrencyLabel = resolved.groupBy === 'round' ? 'n/a (round mode)' : String(resolved.concurrency); @@ -476,11 +520,19 @@ export async function runBenchmark( dashboardUrl = ''; } else { if (identityIsOurs) { + const benchmarkConfig: JsonObject = config.scoring + ? { scoring: config.scoring as unknown as JsonObject } + : {}; await client!.upsertBenchmark(config.benchmarkSlug, { name: config.benchmarkName, + ...(Object.keys(benchmarkConfig).length > 0 ? { config: benchmarkConfig } : {}), }); } + const runConfig = client + ? runConfigToJson(config, resolved, available.map((p) => p.name)) + : {}; + if (args.runKey) { // Shared run: get-or-created by key, so sibling processes (one per provider) // converge on one run. Opened participant-sized — register only the @@ -488,6 +540,7 @@ export async function runBenchmark( // run lists exactly who's benchmarked and each brings its own task count. const { run, organizationSlug } = await client!.createRun(config.benchmarkSlug, { runKey: args.runKey, + config: runConfig, }); runId = run.id; dashboardUrl = dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run.id); @@ -501,6 +554,7 @@ export async function runBenchmark( totalTasks, workerCount: 1, participants: available.map((p) => p.name), + config: runConfig, }); runId = run.id; dashboardUrl = dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run.id); @@ -525,9 +579,11 @@ export async function runBenchmark( participants: participantRecords, config: resolved, }; - if (client && config.onScore) { + if (client && (config.onScore || config.scoring)) { try { - const spec = await config.onScore(lowerIsBetter, higherIsBetter); + const spec = config.onScore + ? await config.onScore(lowerIsBetter, higherIsBetter) + : scoringConfigToSpec(config.scoring!, config.dimensions); const scored = score(outcome, spec); const run = { gitSha: process.env.GITHUB_SHA ?? getGitSha(), @@ -537,9 +593,13 @@ export async function runBenchmark( platform: os.platform(), arch: os.arch(), }; - await client.submitRunSummary(config.benchmarkSlug, runId, { run, results: scored }); + await client.submitRunSummary(config.benchmarkSlug, runId, { + run, + results: scored, + ...(config.scoring ? { scoring: config.scoring as unknown as JsonObject } : {}), + }); } catch (err) { - // A ScoringSpecError means onScore itself is misconfigured (e.g. metric + // A ScoringSpecError means the scoring spec is misconfigured (e.g. metric // weights don't sum to 1.0) — an authoring bug, not a transient submit // failure, so it must fail the run rather than degrade to a warning. if (err instanceof ScoringSpecError) throw err; @@ -571,7 +631,7 @@ function getGitRef(): string | undefined { } /** - * 'participant' ordering: one `client.runWorker` per participant, in turn. + * 'participant' ordering: one `runWorker` call per participant, in turn. * `staggerDelayMs` here launches task N at `workerStart + N * staggerDelayMs` * (vs. round mode's fixed delay between rounds — intentionally different). * `TaskResult.steps`/`latencyMs` are ignored in this path: the platform @@ -633,7 +693,7 @@ async function runGroupedByParticipant( let rampStartMs: number | undefined; await client.planWorkers(config.benchmarkSlug, runId, participant.name); - const result = await client.runWorker({ + const result = await runWorker(client, { benchmarkSlug: config.benchmarkSlug, runId: runId, participantSlug: participant.name, @@ -652,16 +712,26 @@ async function runGroupedByParticipant( // upload; the runner just threads them onto the task context. The runner // wraps `ctx.step` so per-step `timeoutMs` and `concurrency` work in // participant mode as well. - const taskResult = await slot.task({ - participant, - taskIndex: scheduleIndex, - phase: slot.phase, - step: (name, fn, options) => runStepWithClient(ctx.step, name, fn, options), - measure: ctx.measure, - log: ctx.log, - }); + // Tagged before the task runs, not after: measures survive a thrown + // task, so a failed record still carries its phase and can be grouped + // (and filtered) alongside the successful records of that phase. if (slot.phase) ctx.measure({ phase: slot.phase }); - return taskResult?.data; + try { + const taskResult = await slot.task({ + participant, + taskIndex: scheduleIndex, + phase: slot.phase, + step: (name, fn, options) => runStepWithClient(ctx.step, name, fn, options), + measure: ctx.measure, + log: ctx.log, + }); + return taskResult?.data; + } catch (error) { + // Mirrors the 'round' path: a TaskError's domain data is preserved on + // the failure record instead of being dropped for the error message. + if (error instanceof TaskError && error.data) ctx.measure(error.data); + throw error; + } }, onResult: (record) => onResult(record, { iterations: schedule.length, participant: participant.name }), }); @@ -683,7 +753,7 @@ async function runGroupedByParticipant( /** * 'round' ordering: claim one `BenchmarkReporter` per participant up front, * then loop rounds, running one task per participant per round and streaming - * each result to its reporter. Steps are built manually (no `client.runWorker` + * each result to its reporter. Steps are built manually (no `runWorker` * to own them) via a shim that mirrors the platform's record shape. */ async function runGroupedByRound( diff --git a/packages/benchsdk-runner/src/scoring.ts b/packages/benchsdk-runner/src/scoring.ts index 97de23f0..398febe4 100644 --- a/packages/benchsdk-runner/src/scoring.ts +++ b/packages/benchsdk-runner/src/scoring.ts @@ -1,4 +1,4 @@ -import type { JsonObject, TaskResultRecord } from '@benchsdk/client'; +import type { JsonObject, TaskResultRecord } from '@benchsdk/api'; import type { BenchmarkRunOutcome } from './bench-config.js'; export type MetricValue = string | ((record: TaskResultRecord) => number | number[] | undefined); @@ -14,8 +14,47 @@ export interface MetricScoring { trim?: number; } +export interface BenchmarkScoringWeights { + median: number; + p95: number; + p99: number; +} + +export interface BenchmarkScoringMetric { + key: string; + label?: string; + unit: string; + ceiling: number; + floor?: number; + higherIsBetter?: boolean; + weights: BenchmarkScoringWeights; + trim?: number; +} + +/** + * Serializable counterpart to a `success` predicate: a record counts as + * successful only if it succeeded *and* every listed data field equals the + * given value (e.g. `{ verified: true }`, `{ actionsCompleted: 24 }`). Kept to + * equality on scalar data fields so the platform can express the same rule as + * a query predicate rather than replaying benchmark code. + */ +export interface BenchmarkScoringSuccess { + requireData: Record; +} + +/** Serializable scoring spec declared in a `*.bench.ts` file and uploaded to the platform. */ +export interface BenchmarkScoringConfig { + /** Optional data key to group records by when computing summary rows (e.g. 'file_size'). */ + groupBy?: string; + /** Extra conditions a record must meet to count as successful. Default: `status === 'success'`. */ + success?: BenchmarkScoringSuccess; + metrics: BenchmarkScoringMetric[]; +} + export interface ScoringSpec { dimensions?: Record; + /** Optional data key that groups task records into separate summary rows. */ + groupBy?: string; success?: (record: TaskResultRecord) => boolean; metrics: MetricScoring[]; } @@ -165,48 +204,120 @@ export function validateScoringSpec(spec: ScoringSpec): void { } } +function groupRecordsByKey( + records: TaskResultRecord[], + key: string, +): { value: unknown; records: TaskResultRecord[] }[] { + const groups = new Map(); + for (const record of records) { + const raw = record.data?.[key]; + const value = raw === undefined ? undefined : raw; + const mapKey = value === undefined ? '__undefined__' : JSON.stringify(value); + let group = groups.get(mapKey); + if (!group) { + group = { value, records: [] }; + groups.set(mapKey, group); + } + group.records.push(record); + } + return Array.from(groups.values()); +} + +function scoreGroup( + records: TaskResultRecord[], + spec: ScoringSpec, + baseDimensions: JsonObject, + groupKey: string | undefined, + groupValue: unknown, + provider: string, +): BenchmarkScoreResult { + const successFilter = spec.success ?? ((r: TaskResultRecord) => r.status === 'success'); + const passing = records.filter(successFilter); + const successRate = records.length === 0 ? 0 : passing.length / records.length; + const skipped = records.length === 0; + + let metricScoresSum = 0; + const metrics: BenchmarkScoreResult['metrics'] = []; + + for (const metric of spec.metrics) { + const samples = collectSamples(metric, passing); + // A metric with no data points should not contribute to the composite + // score; otherwise an empty lower-is-better metric would be scored as 100. + if (samples.length === 0) { + continue; + } + const { median, p95, p99 } = computeStats(samples, metric.trim ?? 0.05); + const metricScore = + metric.weights.median * scoreStat(median, metric) + + metric.weights.p95 * scoreStat(p95, metric) + + metric.weights.p99 * scoreStat(p99, metric); + metricScoresSum += metricScore; + + metrics.push({ name: metric.name, unit: metric.unit, median, p95, p99 }); + } + + const compositeScore = successRate === 0 ? 0 : Math.round(metricScoresSum * successRate * 100) / 100; + + const dimensions = toJsonObject({ + ...baseDimensions, + ...(groupKey !== undefined && groupValue !== undefined ? { [groupKey]: groupValue } : {}), + }); + + return { + provider, + dimensions, + metrics, + compositeScore, + successRate, + skipped, + }; +} + export function score(outcome: BenchmarkRunOutcome, spec: ScoringSpec): BenchmarkScoreResult[] { validateScoringSpec(spec); - const successFilter = spec.success ?? ((r: TaskResultRecord) => r.status === 'success'); - const dimensions = toJsonObject(spec.dimensions ?? {}); + const baseDimensions = toJsonObject(spec.dimensions ?? {}); const results: BenchmarkScoreResult[] = []; for (const { participant, records } of outcome.participants) { - const passing = records.filter(successFilter); - const successRate = records.length === 0 ? 0 : passing.length / records.length; - const skipped = records.length === 0; - - let metricScoresSum = 0; - const metrics: BenchmarkScoreResult['metrics'] = []; - - for (const metric of spec.metrics) { - const samples = collectSamples(metric, passing); - // A metric with no data points should not contribute to the composite - // score; otherwise an empty lower-is-better metric would be scored as 100. - if (samples.length === 0) { - continue; - } - const { median, p95, p99 } = computeStats(samples, metric.trim ?? 0.05); - const metricScore = - metric.weights.median * scoreStat(median, metric) + - metric.weights.p95 * scoreStat(p95, metric) + - metric.weights.p99 * scoreStat(p99, metric); - metricScoresSum += metricScore; - - metrics.push({ name: metric.name, unit: metric.unit, median, p95, p99 }); - } - - const compositeScore = successRate === 0 ? 0 : Math.round(metricScoresSum * successRate * 100) / 100; + // A participant that recorded nothing has no group to belong to, but it + // still has to appear in the summary as skipped rather than vanish from it. + const groups = spec.groupBy && records.length > 0 + ? [{ value: undefined, records }, ...groupRecordsByKey(records, spec.groupBy)] + : [{ value: undefined, records }]; - results.push({ - provider: participant, - dimensions, - metrics, - compositeScore, - successRate, - skipped, - }); + for (const group of groups) { + results.push(scoreGroup(group.records, spec, baseDimensions, spec.groupBy, group.value, participant)); + } } return results; } + +/** Builds a runtime {@link ScoringSpec} from a serializable {@link BenchmarkScoringConfig}. */ +export function scoringConfigToSpec( + config: BenchmarkScoringConfig, + dimensions?: Record, +): ScoringSpec { + const success = config.success; + return { + ...(dimensions ? { dimensions: toJsonObject(dimensions) } : {}), + ...(config.groupBy ? { groupBy: config.groupBy } : {}), + ...(success + ? { + success: (record: TaskResultRecord) => + record.status === 'success' && + Object.entries(success.requireData).every(([key, value]) => record.data?.[key] === value), + } + : {}), + metrics: config.metrics.map((metric) => ({ + name: metric.key, + value: metric.key, + unit: metric.unit, + ceiling: metric.ceiling, + floor: metric.floor, + higherIsBetter: metric.higherIsBetter, + weights: metric.weights, + trim: metric.trim, + })), + }; +} diff --git a/packages/create-bench/src/__tests__/cli.test.ts b/packages/create-bench/src/__tests__/cli.test.ts index 06aebfe3..9b4a57c7 100644 --- a/packages/create-bench/src/__tests__/cli.test.ts +++ b/packages/create-bench/src/__tests__/cli.test.ts @@ -35,6 +35,6 @@ describe('create-bench CLI', () => { ); expect(pkg.name).toBe(path.basename(tempDir)); - expect(pkg.dependencies['@benchsdk/runner']).toBe('^0.1.0'); + expect(pkg.dependencies['@benchsdk/runner']).toBe('^0.2.0'); }); }); diff --git a/packages/create-bench/src/index.ts b/packages/create-bench/src/index.ts index 7cfdf535..d1138fb2 100644 --- a/packages/create-bench/src/index.ts +++ b/packages/create-bench/src/index.ts @@ -19,7 +19,7 @@ function scaffold(targetDir: string, projectName: string): void { typecheck: 'tsc --noEmit', }, dependencies: { - '@benchsdk/runner': '^0.1.0', + '@benchsdk/runner': '^0.2.0', }, devDependencies: { tsx: '^4.22.4', @@ -64,18 +64,24 @@ export const config = defineBenchmarkConfig({ iterations: 10, concurrency: 1, participants: [{ name: 'local', requiredEnvVars: [] }], + scoring: { + metrics: [ + { key: 'durationMs', unit: 'ms', ceiling: 1000, weights: { median: 0.7, p95: 0.2, p99: 0.1 } }, + ], + }, }); export const task = defineTask(async ({ taskIndex, step, measure, log }) => { log(\`running task \${taskIndex}\`); // Declare named steps with \`step(...)\`; values flow between them via // closures and each step is recorded on the platform with its own timing. + const start = Date.now(); await step('work', async () => { // Replace with your benchmark logic. await new Promise((resolve) => setTimeout(resolve, 100)); }); // \`measure(...)\` attaches metrics to the current step (or the task). - measure({ ok: true }); + measure({ durationMs: Date.now() - start }); }); `; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c9e6ba6e..8a814010 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -283,9 +283,12 @@ importers: packages/benchsdk-runner: dependencies: - '@benchsdk/client': + '@benchsdk/api': + specifier: workspace:* + version: link:../benchsdk-api + '@benchsdk/worker': specifier: workspace:* - version: link:../benchsdk + version: link:../benchsdk-worker devDependencies: '@types/node': specifier: ^20.0.0