refactor(benchsdk-runner): migrate from @benchsdk/client umbrella to @benchsdk/api + @benchsdk/worker

.github/workflows/storage-benchmarks.yml

@@ -13,15 +13,15 @@ on:
   workflow_dispatch:
     inputs:
       iterations:
-        description: 'Iterations per provider'
+        description: 'Iterations per provider, per file size'
         required: false
         default: '100'
       storage_concurrency:
         description: 'Parallel storage iterations per job'
         required: false
         default: '1'
       file_size:
-        description: 'File size to test (leave empty to run all)'
+        description: 'File size to test (leave empty to run all four in one run)'
         required: false
         default: ''
         type: choice
@@ -48,10 +48,11 @@ permissions:
 
 jobs:
   bench:
-    name: Bench ${{ matrix.provider }} ${{ matrix.file_size }}
+    name: Bench ${{ matrix.provider }}
     runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe
     # runs-on: self-hosted
-    timeout-minutes: 90
+    # One job now covers all four sizes, so it does ~4x the work a per-size job did.
+    timeout-minutes: 300
     strategy:
       fail-fast: false
       matrix:
@@ -65,7 +66,6 @@ jobs:
           - tensorlake
           - archil
           - neon
-        file_size: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.file_size != '' && fromJson(format('["{0}"]', github.event.inputs.file_size))) || (github.event_name == 'push' && fromJson('["1MB"]')) || fromJson('["1MB","4MB","10MB","16MB"]') }}
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
@@ -87,11 +87,11 @@ jobs:
         run: |
           . benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|TENSORLAKE_API_KEY|TENSORLAKE_API_URL|TENSORLAKE_ORGANIZATION_ID|TENSORLAKE_PROJECT_ID|TENSORLAKE_FILESYSTEM|ARCHIL_S3_ACCESS_KEY_ID|ARCHIL_S3_SECRET_ACCESS_KEY|ARCHIL_BUCKET|ARCHIL_REGION|ARCHIL_BRANCH|NEON_BUCKET|NEON_ENDPOINT|NEON_ACCESS_KEY_ID|NEON_SECRET_ACCESS_KEY|NEON_REGION|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)'
 
-          FILE_SIZE="${{ matrix.file_size }}"
-          # Matrix jobs for the same file size share one platform run (the benchmark
-          # slug is size-independent, so the size is part of the key). The attempt
-          # number is included so re-runs do not rejoin completed runs.
-          RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${FILE_SIZE}"
+          # Every size runs in one job, so a run covers all of them and the
+          # dashboard can compare sizes within it. The attempt number is included
+          # so re-runs do not rejoin completed runs.
+          FILE_SIZE="${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.file_size != '' && github.event.inputs.file_size) || (github.event_name == 'push' && '1MB') || '1MB,4MB,10MB,16MB' }}"
+          RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
           npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/storage.bench.ts \
             --provider ${{ matrix.provider }} \
             --file-size $FILE_SIZE \
@@ -103,7 +103,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: storage-results-${{ matrix.provider }}-${{ matrix.file_size }}
+          name: storage-results-${{ matrix.provider }}
           path: results/storage/
           if-no-files-found: ignore
           retention-days: 7
@@ -162,16 +162,19 @@ jobs:
             const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
             // Only render sizes that actually ran in this workflow. The checkout
             // carries committed latest.json files for every size (from the weekly
-            // run), so we derive the size list from this run's downloaded
-            // artifacts (storage-results-<provider>-<size>) instead of hardcoding.
+            // run), so we derive the size list from the size directories inside
+            // this run's downloaded artifacts instead of hardcoding.
             const sizeOrder = ['1mb', '4mb', '10mb', '16mb'];
             const ranSizes = new Set();
-            if (fs.existsSync('artifacts')) {
-              for (const name of fs.readdirSync('artifacts')) {
-                if (!name.startsWith('storage-results-')) continue;
-                ranSizes.add(name.split('-').pop().toLowerCase());
+            function collectSizes(dir) {
+              for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+                if (!entry.isDirectory()) continue;
+                const name = entry.name.toLowerCase();
+                if (sizeOrder.includes(name)) ranSizes.add(name);
+                else collectSizes(path.join(dir, entry.name));
               }
             }
+            if (fs.existsSync('artifacts')) collectSizes('artifacts');
             const sizes = sizeOrder.filter(s => ranSizes.has(s));
             let body = '## Storage Benchmark Results\n\n';
             let hasResults = false;

benchmarks/ai-gateway/ai-gateway-gemini.bench.ts

@@ -35,29 +35,20 @@ export const config = defineBenchmarkConfig({
   phases,
   groupBy: 'round',
   participants: providers,
-  onScore: (lowerIsBetter, higherIsBetter) => ({
+  scoring: {
     metrics: [
-      lowerIsBetter('coldE2eMs', {
-        unit: 'ms',
-        ceiling: 20000,
-        value: (record) => ((record.data?.phase as string | undefined) === 'cold' && typeof record.latencyMs === 'number' ? record.latencyMs : undefined),
-        weights: { median: 0.30, p95: 0.15, p99: 0 },
-      }),
-      lowerIsBetter('warmTtftMs', {
-        unit: 'ms',
-        ceiling: 20000,
-        value: (record) => ((record.data?.phase as string | undefined) === 'warm' && typeof record.latencyMs === 'number' ? record.latencyMs : undefined),
-        weights: { median: 0.30, p95: 0.15, p99: 0 },
-      }),
-      higherIsBetter('outputTokensPerSec', {
+      { key: 'coldE2eMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } },
+      { key: 'warmTtftMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } },
+      {
+        key: 'outputTokensPerSec',
         unit: 'tokens/sec',
         floor: 5,
         ceiling: 200,
-        value: (record) => (typeof record.data?.outputTokensPerSec === 'number' ? record.data.outputTokensPerSec : undefined),
+        higherIsBetter: true,
         weights: { median: 0.10, p95: 0, p99: 0 },
-      }),
+      },
     ],
-  }),
+  },
   onComplete: (outcome) =>
     writeAIGatewayLegacyResults(outcome.participants, {
       resultsDir: path.resolve(__dirname, '../../results/ai-gateway-latency/gemini'),

benchmarks/ai-gateway/ai-gateway-kimi.bench.ts

@@ -52,29 +52,20 @@ export const config = defineBenchmarkConfig({
   phases,
   groupBy: 'round',
   participants: providers,
-  onScore: (lowerIsBetter, higherIsBetter) => ({
+  scoring: {
     metrics: [
-      lowerIsBetter('coldE2eMs', {
-        unit: 'ms',
-        ceiling: 20000,
-        value: (record) => ((record.data?.phase as string | undefined) === 'cold' && typeof record.latencyMs === 'number' ? record.latencyMs : undefined),
-        weights: { median: 0.30, p95: 0.15, p99: 0 },
-      }),
-      lowerIsBetter('warmTtftMs', {
-        unit: 'ms',
-        ceiling: 20000,
-        value: (record) => ((record.data?.phase as string | undefined) === 'warm' && typeof record.latencyMs === 'number' ? record.latencyMs : undefined),
-        weights: { median: 0.30, p95: 0.15, p99: 0 },
-      }),
-      higherIsBetter('outputTokensPerSec', {
+      { key: 'coldE2eMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } },
+      { key: 'warmTtftMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } },
+      {
+        key: 'outputTokensPerSec',
         unit: 'tokens/sec',
         floor: 5,
         ceiling: 200,
-        value: (record) => (typeof record.data?.outputTokensPerSec === 'number' ? record.data.outputTokensPerSec : undefined),
+        higherIsBetter: true,
         weights: { median: 0.10, p95: 0, p99: 0 },
-      }),
+      },
     ],
-  }),
+  },
   onComplete: (outcome) =>
     writeAIGatewayLegacyResults(outcome.participants, {
       resultsDir: path.resolve(__dirname, '../../results/ai-gateway-latency/kimi'),

benchmarks/ai-gateway/ai-gateway-openai.bench.ts

@@ -35,29 +35,20 @@ export const config = defineBenchmarkConfig({
   phases,
   groupBy: 'round',
   participants: providers,
-  onScore: (lowerIsBetter, higherIsBetter) => ({
+  scoring: {
     metrics: [
-      lowerIsBetter('coldE2eMs', {
-        unit: 'ms',
-        ceiling: 20000,
-        value: (record) => ((record.data?.phase as string | undefined) === 'cold' && typeof record.latencyMs === 'number' ? record.latencyMs : undefined),
-        weights: { median: 0.30, p95: 0.15, p99: 0 },
-      }),
-      lowerIsBetter('warmTtftMs', {
-        unit: 'ms',
-        ceiling: 20000,
-        value: (record) => ((record.data?.phase as string | undefined) === 'warm' && typeof record.latencyMs === 'number' ? record.latencyMs : undefined),
-        weights: { median: 0.30, p95: 0.15, p99: 0 },
-      }),
-      higherIsBetter('outputTokensPerSec', {
+      { key: 'coldE2eMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } },
+      { key: 'warmTtftMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } },
+      {
+        key: 'outputTokensPerSec',
         unit: 'tokens/sec',
         floor: 5,
         ceiling: 200,
-        value: (record) => (typeof record.data?.outputTokensPerSec === 'number' ? record.data.outputTokensPerSec : undefined),
+        higherIsBetter: true,
         weights: { median: 0.10, p95: 0, p99: 0 },
-      }),
+      },
     ],
-  }),
+  },
   onComplete: (outcome) =>
     writeAIGatewayLegacyResults(outcome.participants, {
       resultsDir: path.resolve(__dirname, '../../results/ai-gateway-latency/openai'),

benchmarks/ai-gateway/ai-gateway.bench.ts

@@ -52,29 +52,20 @@ export const config = defineBenchmarkConfig({
   phases,
   groupBy: 'round',
   participants: providers,
-  onScore: (lowerIsBetter, higherIsBetter) => ({
+  scoring: {
     metrics: [
-      lowerIsBetter('coldE2eMs', {
-        unit: 'ms',
-        ceiling: 20000,
-        value: (record) => ((record.data?.phase as string | undefined) === 'cold' && typeof record.latencyMs === 'number' ? record.latencyMs : undefined),
-        weights: { median: 0.30, p95: 0.15, p99: 0 },
-      }),
-      lowerIsBetter('warmTtftMs', {
-        unit: 'ms',
-        ceiling: 20000,
-        value: (record) => ((record.data?.phase as string | undefined) === 'warm' && typeof record.latencyMs === 'number' ? record.latencyMs : undefined),
-        weights: { median: 0.30, p95: 0.15, p99: 0 },
-      }),
-      higherIsBetter('outputTokensPerSec', {
+      { key: 'coldE2eMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } },
+      { key: 'warmTtftMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } },
+      {
+        key: 'outputTokensPerSec',
         unit: 'tokens/sec',
         floor: 5,
         ceiling: 200,
-        value: (record) => (typeof record.data?.outputTokensPerSec === 'number' ? record.data.outputTokensPerSec : undefined),
+        higherIsBetter: true,
         weights: { median: 0.10, p95: 0, p99: 0 },
-      }),
+      },
     ],
-  }),
+  },
   onComplete: (outcome) =>
     writeAIGatewayLegacyResults(outcome.participants, {
       resultsDir: path.resolve(__dirname, '../../results/ai-gateway-latency/anthropic'),

benchmarks/ai-gateway/shared-task.ts

@@ -3,7 +3,7 @@
  * OpenAI, Gemini, Kimi). Deliberately holds only what's identical across
  * every family — the probe task itself, phase/step shaping, and CLI
  * iteration-flag parsing — not the family's own identity (`benchmarkSlug`,
- * `providers`, `onScore`, `onComplete`), which each `*.bench.ts` file
+ * `providers`, `scoring`, `onComplete`), which each `*.bench.ts` file
  * declares directly via `defineBenchmarkConfig`, mirroring
  * `ai-gateway.bench.ts` (the Anthropic family, the original of the four).
  * Config/scoring intentionally isn't abstracted here even though it's
@@ -85,8 +85,15 @@ function phaseSteps(result: PhaseProbeResult): TaskStepRecord[] {
 }
 
 function probeData(result: PhaseProbeResult): JsonObject {
+  // The scored latency is reported under a phase-specific key (`coldE2eMs` /
+  // `warmTtftMs`) so cold and warm are separate metrics without a scoring-time
+  // phase filter.
+  const scoredLatencyMs = result.mode === 'cold' ? result.coldE2eMs ?? result.ttftMs : result.ttftMs;
   return {
     mode: result.mode,
+    ...(typeof scoredLatencyMs === 'number' && Number.isFinite(scoredLatencyMs)
+      ? { [result.mode === 'cold' ? 'coldE2eMs' : 'warmTtftMs']: scoredLatencyMs }
+      : {}),
     ...(result.outputTokens !== undefined ? { outputTokens: result.outputTokens } : {}),
     ...(result.outputTokensPerSec !== undefined ? { outputTokensPerSec: result.outputTokensPerSec } : {}),
     ...(result.resolvedProvider !== undefined ? { resolvedProvider: result.resolvedProvider } : {}),

benchmarks/browser/browser-throughput.bench.ts

@@ -35,31 +35,24 @@ export const config = defineBenchmarkConfig({
   iterations: 2,
   groupBy: 'round',
   participants: throughputProviders,
-  onScore: (lowerIsBetter, higherIsBetter) => ({
-    success: (record) => record.status === 'success' && typeof record.data?.actionsCompleted === 'number' && record.data.actionsCompleted === ACTIONS_PER_SESSION,
+  scoring: {
+    // A session that dropped actions isn't comparable to one that completed
+    // them all, so it counts against the success rate and its timings are
+    // excluded from the metrics.
+    success: { requireData: { actionsCompleted: ACTIONS_PER_SESSION } },
     metrics: [
-      higherIsBetter('actionsPerSecond', {
+      {
+        key: 'actionsPerSecond',
         unit: 'actions/sec',
         floor: 0,
         ceiling: 10,
+        higherIsBetter: true,
         weights: { median: 0.40, p95: 0, p99: 0 },
-      }),
-      lowerIsBetter('taskMs', {
-        unit: 'ms',
-        ceiling: 30000,
-        weights: { median: 0.25, p95: 0.20, p99: 0 },
-      }),
-      lowerIsBetter('screenshot', {
-        unit: 'ms',
-        ceiling: 30000,
-        value: (record) => {
-          const actions = Array.isArray(record.data?.actions) ? (record.data!.actions as any[]) : [];
-          return actions.filter((a) => a.type === 'screenshot' && a.success).map((a) => a.durationMs);
-        },
-        weights: { median: 0.15, p95: 0, p99: 0 },
-      }),
+      },
+      { key: 'taskMs', unit: 'ms', ceiling: 30000, weights: { median: 0.25, p95: 0.20, p99: 0 } },
+      { key: 'screenshotMs', unit: 'ms', ceiling: 30000, weights: { median: 0.15, p95: 0, p99: 0 } },
     ],
-  }),
+  },
   onComplete: (outcome) =>
     writeThroughputLegacyResults(outcome.participants, {
       resultsDir: path.resolve(__dirname, '../../results/browser-throughput'),
@@ -231,6 +224,17 @@ async function runActionLoop(page: Page, results: ActionResult[], navigateUrl: s
   }
 }
 
+/** Median duration of the session's successful screenshots, or undefined if it took none. */
+function medianScreenshotMs(actions: ActionResult[]): number | undefined {
+  const durations = actions
+    .filter((a) => a.type === 'screenshot' && a.success)
+    .map((a) => a.durationMs)
+    .sort((a, b) => a - b);
+  if (durations.length === 0) return undefined;
+  const mid = Math.floor(durations.length / 2);
+  return durations.length % 2 === 0 ? (durations[mid - 1] + durations[mid]) / 2 : durations[mid];
+}
+
 /** One provider instance per participant, reused across that provider's sessions. */
 const providerCache = new Map<string, any>();
 
@@ -254,6 +258,7 @@ export const task = defineTask<ThroughputProviderConfig>(async (ctx) => {
   let taskMs = 0;
   let actionsCompleted = 0;
   let actionsPerSecond = 0;
+  let screenshotMs: number | undefined;
 
   let session: { sessionId: string; connectUrl: string } | undefined;
   let browser: Browser | undefined;
@@ -292,7 +297,8 @@ export const task = defineTask<ThroughputProviderConfig>(async (ctx) => {
       taskMs = actions.reduce((sum, a) => sum + a.durationMs, 0);
       actionsCompleted = actions.filter((a) => a.success).length;
       actionsPerSecond = taskMs > 0 ? actionsCompleted / (taskMs / 1000) : 0;
-      measure({ actionsPerSecond, actionsCompleted });
+      screenshotMs = medianScreenshotMs(actions);
+      measure({ actionsPerSecond, actionsCompleted, ...(screenshotMs !== undefined ? { screenshotMs } : {}) });
     });
   } catch (err) {
     iterationError = err instanceof Error ? err.message : String(err);
@@ -329,6 +335,7 @@ export const task = defineTask<ThroughputProviderConfig>(async (ctx) => {
     taskMs,
     actionsCompleted,
     actionsPerSecond,
+    ...(screenshotMs !== undefined ? { screenshotMs } : {}),
     actions: actions as unknown as JsonValue,
     ...(iterationError ? { errorMessage: iterationError } : {}),
   };

benchmarks/browser/browser.bench.ts

@@ -26,12 +26,12 @@ export const config = defineBenchmarkConfig({
   iterations: 2,
   concurrency: 1,
   participants: browserProviders,
-  onScore: (lowerIsBetter) => ({
+  scoring: {
     metrics: [
-      lowerIsBetter('totalMs', { unit: 'ms', ceiling: 10000, weights: { median: 0.40, p95: 0.20, p99: 0.10 } }),
-      lowerIsBetter('createMs', { unit: 'ms', ceiling: 10000, weights: { median: 0.30, p95: 0, p99: 0 } }),
+      { key: 'totalMs', unit: 'ms', ceiling: 10000, weights: { median: 0.40, p95: 0.20, p99: 0.10 } },
+      { key: 'createMs', unit: 'ms', ceiling: 10000, weights: { median: 0.30, p95: 0, p99: 0 } },
     ],
-  }),
+  },
   onComplete: (outcome) =>
     writeBrowserLegacyResults(outcome.participants, {
       resultsDir: path.resolve(__dirname, '../../results/browser'),

benchmarks/sandbox/tti.bench.ts

@@ -39,15 +39,11 @@ export const config = defineBenchmarkConfig({
   iterations: 2,
   concurrency: 1,
   participants: providers,
-  onScore: (lowerIsBetter) => ({
+  scoring: {
     metrics: [
-      lowerIsBetter('ttiMs', {
-        unit: 'ms',
-        ceiling: 10000,
-        weights: { median: 0.60, p95: 0.25, p99: 0.15 },
-      }),
+      { key: 'ttiMs', unit: 'ms', ceiling: 10000, weights: { median: 0.60, p95: 0.25, p99: 0.15 } },
     ],
-  }),
+  },
   // Legacy JSON labels a burst run 'concurrent' (see merge-results /
   // generate-svg) — that's the shape carrying the wall-clock/ramp fields. The
   // `results/` directory name predates this file and is kept verbatim so the