diff --git a/.changeset/benchsdk-client-run-key.md b/.changeset/benchsdk-client-run-key.md new file mode 100644 index 00000000..90795488 --- /dev/null +++ b/.changeset/benchsdk-client-run-key.md @@ -0,0 +1,5 @@ +--- +"@benchsdk/client": minor +--- + +`createRun` accepts an optional `runKey`: callers passing the same key (per org + benchmark) get-or-create one shared run instead of each opening its own. `BenchmarkRun.runKey` reports the key a run was created with. diff --git a/.changeset/benchsdk-runner-shapes-run-key.md b/.changeset/benchsdk-runner-shapes-run-key.md new file mode 100644 index 00000000..bf198f3c --- /dev/null +++ b/.changeset/benchsdk-runner-shapes-run-key.md @@ -0,0 +1,11 @@ +--- +"@benchsdk/runner": minor +--- + +Verbs-only CLI: `bench run ` is the one mutating command. The benchmark is declared in the file and materialized (upserted) as a side effect of running it, and a run is opened as a side effect too — there are no imperative `bench create benchmark` / `bench create run` commands. + +`bench run` gains `--shape `: a bench file can declare named `shapes`, each swapping in its own platform identity (`slug`/`name`, optional `kind`) and a stable knob (`staggerDelayMs`) while reusing the same task and participants. This collapses the per-shape slug/name/knob triple that was duplicated across package scripts and CI. + +`bench run` gains `--run-key `: sibling processes passing the same key (per org + benchmark) get-or-create one shared run instead of each opening its own, so provider jobs running in parallel land in a single, directly-comparable run. Each process registers only the participants it runs. The key binding is permanent, so callers that need a fresh run (e.g. a CI re-run) vary the key (e.g. include `GITHUB_RUN_ATTEMPT`). + +`--slug` remains a working alias for `--benchmark`. diff --git a/.changeset/benchsdk-runner-slug-flag.md b/.changeset/benchsdk-runner-slug-flag.md deleted file mode 100644 index f100a1c7..00000000 --- a/.changeset/benchsdk-runner-slug-flag.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@benchsdk/runner": minor ---- - -Add `--slug` and `--name` CLI overrides for `benchmarkSlug`/`benchmarkName`, so one `*.bench.ts` can report under several platform benchmarks (e.g. the sandbox TTI entrypoint reporting sequential/staggered/burst runs of the same workload). diff --git a/.changeset/participant-sized-runs.md b/.changeset/participant-sized-runs.md new file mode 100644 index 00000000..6d8b8121 --- /dev/null +++ b/.changeset/participant-sized-runs.md @@ -0,0 +1,5 @@ +--- +"@benchsdk/client": minor +--- + +`createRun` no longer requires `totalTasks`: omit it to open a participant-sized run, whose total is the sum of what its participants declare when they register. `BenchmarkRun.participantSized` reports which kind a run is. diff --git a/.github/workflows/git-benchmarks.yml b/.github/workflows/git-benchmarks.yml new file mode 100644 index 00000000..d8088b8d --- /dev/null +++ b/.github/workflows/git-benchmarks.yml @@ -0,0 +1,126 @@ +name: Git Benchmark + +on: + push: + branches: [master] + paths: + - 'benchmarks/git/**' + - 'benchmarks/src/util/**' + - 'benchmarks/src/merge-results.ts' + - 'package.json' + - '.github/workflows/git-benchmarks.yml' + schedule: + - cron: '0 0 * * 1' # Weekly on Monday at midnight UTC + workflow_dispatch: + inputs: + iterations: + description: 'Iterations per provider' + required: false + default: '10' + provider: + description: 'Provider to run (leave empty for all)' + required: false + default: '' + dry_run: + description: 'Run without committing results' + required: false + default: false + type: boolean + +concurrency: + group: git-benchmarks + cancel-in-progress: true + +jobs: + bench: + permissions: + contents: read + name: Bench ${{ matrix.provider }} + runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + provider: ${{ (github.event.inputs.provider != '' && fromJson(format('["{0}"]', github.event.inputs.provider))) || fromJson('["github","gitlab","bitbucket","tensorlake"]') }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: 'pnpm' + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Clear stale results from checkout + run: rm -rf results/git/ + - name: Run git benchmark + run: | + . benchmarks/scripts/load-vault-secrets.sh '^(GITHUB_GIT_REPO_URL|GITHUB_TOKEN|GITLAB_GIT_REPO_URL|GITLAB_TOKEN|BITBUCKET_GIT_REPO_URL|BITBUCKET_TOKEN|TENSORLAKE_GIT_REPO_URL|TENSORLAKE_API_KEY|BENCHMARKS_PLATFORM_API_KEY)$' + + ITERATIONS="${{ (github.event_name == 'schedule' && '50') || github.event.inputs.iterations || '10' }}" + + # Every provider job passes the same --run-key, so all matrix jobs + # get-or-create one shared platform run and each registers only its + # own participant. The attempt is in the key so workflow re-runs open + # fresh runs instead of rejoining the completed ones. + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + + npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts \ + --provider ${{ matrix.provider }} \ + --iterations $ITERATIONS \ + --run-key "$RUN_KEY" + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: git-results-${{ matrix.provider }} + path: results/git/ + if-no-files-found: ignore + retention-days: 7 + + collect: + name: Collect Git Results + runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + needs: [bench] + if: always() + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: 'pnpm' + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts/ + pattern: git-results-* + - name: Merge results + run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode git + - name: Commit and push results + if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add results/git/ + git diff --cached --quiet && echo "No changes to commit" && exit 0 + git commit -m "chore: update git benchmark results [skip ci]" + + branch="${GITHUB_REF#refs/heads/}" + for attempt in 1 2 3 4 5; do + git fetch origin "${branch}" + git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } + if git push origin "HEAD:${branch}"; then + echo "Pushed on attempt ${attempt}" + exit 0 + fi + echo "Push rejected (attempt ${attempt}); will rebase and retry" + done + echo "Failed to push after multiple attempts" >&2 + exit 1 diff --git a/.github/workflows/sandbox-dax-benchmarks.yml b/.github/workflows/sandbox-dax-benchmarks.yml index f05f49b6..c8a46001 100644 --- a/.github/workflows/sandbox-dax-benchmarks.yml +++ b/.github/workflows/sandbox-dax-benchmarks.yml @@ -8,6 +8,8 @@ on: - 'benchmarks/sandbox/providers.ts' - 'benchmarks/src/merge-results.ts' - 'benchmarks/scripts/dax-benchmark.sh' + - 'benchmarks/sandbox/generate-dax-svg.ts' + - 'dax.svg' - 'package.json' - '.github/workflows/sandbox-dax-benchmarks.yml' schedule: @@ -149,6 +151,13 @@ jobs: pattern: results-* - name: Merge results run: npx tsx benchmarks/src/merge-results.ts --input artifacts + - name: Generate DAX SVG + run: | + if [ -f results/sandbox-dax/latest.json ]; then + pnpm run generate-dax-svg + else + echo "No DAX results found; skipping chart" + fi - name: Ingest results to platform if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' continue-on-error: true @@ -248,7 +257,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git add results/sandbox-dax/ + git add results/sandbox-dax/ dax.svg git diff --cached --quiet && echo "No changes to commit" && exit 0 git commit -m "chore: update dax benchmark results [skip ci]" diff --git a/.github/workflows/sandbox-tti-benchmarks.yml b/.github/workflows/sandbox-tti-benchmarks.yml index f7d94031..fcebf0ab 100644 --- a/.github/workflows/sandbox-tti-benchmarks.yml +++ b/.github/workflows/sandbox-tti-benchmarks.yml @@ -118,19 +118,26 @@ jobs: run: | . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|ARKER_API_KEY|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' - # One entrypoint, three launch shapes: the knobs come from the CLI - # and --slug/--name pick which platform benchmark each reports to. + # One entrypoint, three launch shapes selected with --shape; the scale + # knobs come from the CLI. Every provider passes the same --run-key, so + # each shape's providers get-or-create one shared run and rank against + # each other (each still registers itself and claims its own worker). + # The shapes are distinct benchmark slugs, so one key value yields one + # run per shape; the run attempt is in the key so a workflow re-run + # opens fresh runs instead of rejoining the completed ones. BENCH=(npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts) - COMMON=(--provider ${{ matrix.provider }} \ - --iterations ${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }}) + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + COMMON=( + --provider ${{ matrix.provider }} + --iterations ${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }} + --run-key "$RUN_KEY" + ) CONCURRENCY=${{ github.event.inputs.concurrency || (github.event_name == 'push' && '10') || '100' }} # Sequential is one-at-a-time by definition, so it never gets # --concurrency (the config default pins it to 1). SEQUENTIAL=("${BENCH[@]}" "${COMMON[@]}") - STAGGERED=("${BENCH[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" --stagger-delay-ms 200 \ - --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)') - BURST=("${BENCH[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" \ - --slug sandbox-burst-local --name 'Sandbox burst TTI (local)') + STAGGERED=("${BENCH[@]}" "${COMMON[@]}" --shape staggered --concurrency "$CONCURRENCY") + BURST=("${BENCH[@]}" "${COMMON[@]}" --shape burst --concurrency "$CONCURRENCY") case "${{ github.event.inputs.mode }}" in sequential) "${SEQUENTIAL[@]}" ;; staggered) "${STAGGERED[@]}" ;; diff --git a/README.md b/README.md index bef4c0a3..7699c9ac 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,10 @@ Our partners support our independent benchmarks. ![Object Storage Snapshot & Fork](./snapshot_fork_small.svg) +### [DAX Sandbox Builds](#dax-sandbox-builds) + +![DAX Sandbox Builds](./dax.svg) + [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](./LICENSE) **TTI (Time to Interactive)** = API call to first command execution. Lower is better. diff --git a/benchmarks/git/git.bench.ts b/benchmarks/git/git.bench.ts new file mode 100644 index 00000000..410236a2 --- /dev/null +++ b/benchmarks/git/git.bench.ts @@ -0,0 +1,177 @@ +/** + * Git workflow benchmark. Measures shallow clone, commit+push, and pull over + * HTTPS for git hosting providers by shelling out to `git`. + * Declarative — exports `config` + `task`; `bench run` owns the entrypoint. + * + * Every participant requires both a writable `*_GIT_REPO_URL` env var and a + * matching `*_TOKEN` env var; the runner skips providers without credentials. + * + * bench run benchmarks/git/git.bench.ts + * bench run benchmarks/git/git.bench.ts --provider tensorlake --iterations 5 + */ +import '../src/env.js'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { defineBenchmarkConfig, defineTask, TaskError } from '@benchsdk/runner'; +import { withTimeout } from '../src/util/timeout.js'; +import { formatError } from '../src/util/error.js'; +import { providers } from './providers.js'; +import { writeGitLegacyResults } from './legacy-results.js'; +import type { GitProviderConfig } from './types.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +const execFileAsync = promisify(execFile); +const CLONE_TIMEOUT_MS = 60_000; +const COMMITTER_NAME = 'ComputeSDK Benchmark'; +const COMMITTER_EMAIL = 'bench@example.com'; + +function resolveRepoConfig(config: GitProviderConfig): { repoUrl: string } { + const repoUrl = config.repoUrlEnvVar ? sanitizeRepoUrl(process.env[config.repoUrlEnvVar] ?? '') : ''; + return { repoUrl }; +} + +function sanitizeRepoUrl(repoUrl: string): string { + try { + const url = new URL(repoUrl); + url.username = ''; + url.password = ''; + return url.toString(); + } catch { + return repoUrl; + } +} + +function escapeShell(s: string): string { + return s.replace(/'/g, `'\\''`); +} + +function buildGitEnv(askpassPath: string): NodeJS.ProcessEnv { + return { ...process.env, GIT_ASKPASS: askpassPath, GIT_TERMINAL_PROMPT: '0' }; +} + +async function writeAskpassScript(askpassPath: string, username: string, token: string): Promise { + const script = `#!/bin/sh +case "$1" in + *Password*) printf '%s\\n' '${escapeShell(token)}' ;; + *Username*) printf '%s\\n' '${escapeShell(username)}' ;; + *) printf '%s\\n' '${escapeShell(username)}' ;; +esac +`; + await fs.promises.writeFile(askpassPath, script, { mode: 0o700 }); +} + +export const config = defineBenchmarkConfig({ + benchmarkSlug: 'git-workflow-local', + benchmarkName: 'Git workflow (local)', + benchmarkKind: 'git', + iterations: 3, + concurrency: 1, + participants: providers, + onComplete: (outcome) => + writeGitLegacyResults(outcome.participants, path.resolve(__dirname, '../../results/git')), +}); + +export const task = defineTask(async (ctx) => { + const { participant, step, measure, taskIndex } = ctx; + const timeout = participant.timeout ?? CLONE_TIMEOUT_MS; + + const repoUrl = resolveRepoConfig(participant).repoUrl; + const token = process.env[participant.tokenEnvVar] ?? ''; + const username = participant.tokenUsername; + + const branch = `${participant.name}-${taskIndex}-${Date.now()}`; + const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'bench-git-')); + const workDir = path.join(tempDir, 'repo'); + const pullDir = path.join(tempDir, 'pull'); + const askpassPath = path.join(tempDir, 'askpass.sh'); + + await writeAskpassScript(askpassPath, username, token); + + const env = buildGitEnv(askpassPath); + function runGit( + args: string[], + cwd: string, + execTimeout = timeout, + ): Promise<{ stdout: string; stderr: string }> { + return execFileAsync('git', args, { cwd, env, timeout: execTimeout }); + } + + let cloneMs = 0; + let pushMs = 0; + let pullMs = 0; + let pushAttempted = false; + + try { + const cloneStart = performance.now(); + await step('clone', () => + withTimeout( + runGit(['clone', '--depth', '1', '--single-branch', repoUrl, workDir], process.cwd()), + timeout, + 'Git clone timed out', + ), + ); + cloneMs = performance.now() - cloneStart; + + const defaultBranch = await runGit(['branch', '--show-current'], workDir) + .then((r) => r.stdout.trim()) + .catch(() => participant.defaultBranch ?? 'main'); + + // Prepare, commit, and push the test branch. + await runGit(['checkout', '-b', branch], workDir); + await fs.promises.writeFile(path.join(workDir, 'bench.txt'), `benchmark ${branch}\n`); + await runGit(['add', 'bench.txt'], workDir); + const commitResult = await runGit( + ['-c', `user.name=${COMMITTER_NAME}`, '-c', `user.email=${COMMITTER_EMAIL}`, 'commit', '-m', `bench: ${branch}`], + workDir, + ); + const commitSha = commitResult.stdout.match(/\[.+?\s+([a-f0-9]+)\]/)?.[1] ?? ''; + + pushAttempted = true; + const pushStart = performance.now(); + await step('push', () => + withTimeout( + runGit(['push', '-u', 'origin', branch], workDir), + timeout, + 'Git push timed out', + ), + ); + pushMs = performance.now() - pushStart; + + // Prepare a separate shallow clone so the pull actually fetches the new + // branch from the remote instead of finding all objects already local. + await fs.promises.mkdir(pullDir, { recursive: true }); + await runGit(['init'], pullDir); + await runGit(['remote', 'add', 'origin', repoUrl], pullDir); + await runGit(['fetch', '--depth', '1', 'origin', defaultBranch], pullDir); + await runGit(['checkout', '-b', defaultBranch, 'FETCH_HEAD'], pullDir); + + const pullStart = performance.now(); + await step('pull', () => + withTimeout( + runGit(['pull', '--ff-only', 'origin', branch], pullDir), + timeout, + 'Git pull timed out', + ), + ); + pullMs = performance.now() - pullStart; + + const authData: Record = { cloneMs, pushMs, pullMs, branch, commitSha }; + measure(authData as any); + return { data: authData as any }; + } catch (err) { + throw new TaskError(formatError(err), { + code: 'GIT_WORKFLOW_ERROR', + data: { branch, cloneMs, pushMs, pullMs }, + }); + } finally { + if (pushAttempted) { + await runGit(['push', 'origin', '--delete', branch], workDir, timeout).catch(() => {}); + } + await fs.promises.rm(tempDir, { recursive: true, force: true }).catch(() => {}); + } +}); diff --git a/benchmarks/git/legacy-results.ts b/benchmarks/git/legacy-results.ts new file mode 100644 index 00000000..bbadd664 --- /dev/null +++ b/benchmarks/git/legacy-results.ts @@ -0,0 +1,107 @@ +import { mkdirSync, copyFileSync } from 'node:fs'; +import path from 'node:path'; +import type { ParticipantRecords } from '@benchsdk/runner'; +import type { JsonObject } from '@benchsdk/client'; +import { byTaskIndex } from '../src/util/records.js'; +import { computeStats } from '../src/util/stats.js'; +import type { GitBenchmarkResult, GitTimingResult } from './types.js'; + +function num(x: unknown): number { + return typeof x === 'number' ? x : 0; +} + +/** Map CLI participant records to legacy GitBenchmarkResult[]. */ +export function recordsToGitResults(participants: ParticipantRecords[]): GitBenchmarkResult[] { + return participants.map((participant) => { + const iterations: GitTimingResult[] = byTaskIndex(participant.records).map((r) => { + const d = (r.data ?? {}) as JsonObject; + const base: GitTimingResult = { + cloneMs: num(d.cloneMs), + pushMs: num(d.pushMs), + pullMs: num(d.pullMs), + branch: typeof d.branch === 'string' ? d.branch : '', + commitSha: typeof d.commitSha === 'string' ? d.commitSha : undefined, + }; + return r.status === 'error' ? { ...base, error: r.errorCode ?? 'error' } : base; + }); + + const successful = iterations.filter((i) => !i.error); + const summary = { + cloneMs: computeStats(successful.map((i) => i.cloneMs)), + pushMs: computeStats(successful.map((i) => i.pushMs)), + pullMs: computeStats(successful.map((i) => i.pullMs)), + }; + + return { + provider: participant.participant, + mode: 'git' as const, + iterations, + summary, + successRate: iterations.length ? successful.length / iterations.length : 0, + }; + }); +} + +export async function writeGitResultsJson(results: GitBenchmarkResult[], outPath: string): Promise { + const fs = await import('node:fs'); + const os = await import('node:os'); + + const clean = results.map((r) => ({ + provider: r.provider, + mode: r.mode, + iterations: r.iterations.map((i) => { + const entry: Record = { + cloneMs: i.cloneMs, + pushMs: i.pushMs, + pullMs: i.pullMs, + branch: i.branch, + }; + if (i.commitSha !== undefined) entry.commitSha = i.commitSha; + if (i.error !== undefined) entry.error = i.error; + return entry; + }), + summary: r.summary, + ...(r.successRate !== undefined ? { successRate: r.successRate } : {}), + ...(r.compositeScore !== undefined ? { compositeScore: r.compositeScore } : {}), + ...(r.skipped ? { skipped: r.skipped, skipReason: r.skipReason } : {}), + })); + + const output = { + version: '1.0', + timestamp: new Date().toISOString(), + environment: { + node: process.version, + platform: os.platform(), + arch: os.arch(), + }, + config: { + iterations: results[0]?.iterations.length || 0, + }, + results: clean, + }; + + fs.writeFileSync(outPath, JSON.stringify(output, null, 2)); + console.log(`Results written to ${outPath}`); +} + +/** + * Map records -> GitBenchmarkResult[], compute success rates, and write both + * `.json` and `latest.json` into resultsDir. + * TEMPORARY BRIDGE until the platform read API exposes per-iteration data. + */ +export async function writeGitLegacyResults( + participants: ParticipantRecords[], + resultsDir: string, +): Promise { + const results = recordsToGitResults(participants); + + mkdirSync(resultsDir, { recursive: true }); + + const timestamp = new Date().toISOString().slice(0, 10); + const outPath = path.join(resultsDir, `${timestamp}.json`); + await writeGitResultsJson(results, outPath); + + const latestPath = path.join(resultsDir, 'latest.json'); + copyFileSync(outPath, latestPath); + console.log(`Copied latest: ${latestPath}`); +} diff --git a/benchmarks/git/providers.ts b/benchmarks/git/providers.ts new file mode 100644 index 00000000..82bedc1a --- /dev/null +++ b/benchmarks/git/providers.ts @@ -0,0 +1,41 @@ +import type { GitProviderConfig } from './types.js'; + +/** + * Git hosting provider benchmark configurations. + * + * Every participant requires both a writable `*_GIT_REPO_URL` env var and a + * matching `*_TOKEN` env var. The runner skips providers whose credentials are + * missing, so there is no read-only fallback. + */ +export const providers: GitProviderConfig[] = [ + { + name: 'github', + requiredEnvVars: ['GITHUB_GIT_REPO_URL', 'GITHUB_TOKEN'], + repoUrlEnvVar: 'GITHUB_GIT_REPO_URL', + tokenEnvVar: 'GITHUB_TOKEN', + tokenUsername: 'token', + }, + { + name: 'gitlab', + requiredEnvVars: ['GITLAB_GIT_REPO_URL', 'GITLAB_TOKEN'], + repoUrlEnvVar: 'GITLAB_GIT_REPO_URL', + tokenEnvVar: 'GITLAB_TOKEN', + tokenUsername: 'oauth2', + }, + { + name: 'bitbucket', + requiredEnvVars: ['BITBUCKET_GIT_REPO_URL', 'BITBUCKET_TOKEN'], + repoUrlEnvVar: 'BITBUCKET_GIT_REPO_URL', + tokenEnvVar: 'BITBUCKET_TOKEN', + tokenUsername: 'x-token-auth', + }, + { + name: 'tensorlake', + requiredEnvVars: ['TENSORLAKE_GIT_REPO_URL', 'TENSORLAKE_API_KEY'], + repoUrlEnvVar: 'TENSORLAKE_GIT_REPO_URL', + tokenEnvVar: 'TENSORLAKE_API_KEY', + tokenUsername: 't', + }, + // + // add git providers above +]; diff --git a/benchmarks/git/types.ts b/benchmarks/git/types.ts new file mode 100644 index 00000000..4d888538 --- /dev/null +++ b/benchmarks/git/types.ts @@ -0,0 +1,48 @@ +import type { BaseParticipant } from '@benchsdk/client'; + +export interface GitProviderConfig extends BaseParticipant { + /** Env var holding the writable HTTPS repo URL. */ + repoUrlEnvVar: string; + /** Env var holding the HTTPS auth token for push/pull. */ + tokenEnvVar: string; + /** Username supplied to `git` via GIT_ASKPASS. */ + tokenUsername: string; + /** Default branch to pull back into after pushing a test branch. */ + defaultBranch?: string; + /** Per-operation timeout in ms (default: 60000). */ + timeout?: number; +} + +export interface GitTimingResult { + /** Time to shallow clone the repo in ms. */ + cloneMs: number; + /** Time to commit and push the test branch in ms. */ + pushMs: number; + /** Time to pull the test branch in ms. */ + pullMs: number; + /** Test branch that was pushed/pulled. */ + branch: string; + /** Commit SHA produced by the benchmark push, when available. */ + commitSha?: string; + /** Error message if this iteration failed. */ + error?: string; +} + +export interface GitStats { + cloneMs: { median: number; p95: number; p99: number }; + pushMs: { median: number; p95: number; p99: number }; + pullMs: { median: number; p95: number; p99: number }; +} + +export interface GitBenchmarkResult { + provider: string; + mode: 'git'; + iterations: GitTimingResult[]; + summary: GitStats; + /** Composite weighted score (0-100, higher = better). Computed post-benchmark. */ + compositeScore?: number; + /** Success rate as a fraction (0 to 1). Computed post-benchmark. */ + successRate?: number; + skipped?: boolean; + skipReason?: string; +} diff --git a/benchmarks/sandbox/generate-dax-svg.ts b/benchmarks/sandbox/generate-dax-svg.ts new file mode 100644 index 00000000..5c7b87f9 --- /dev/null +++ b/benchmarks/sandbox/generate-dax-svg.ts @@ -0,0 +1,136 @@ +#!/usr/bin/env node + +import fs from 'node:fs'; +import path from 'node:path'; +import type { DaxBenchmarkResult } from './dax.js'; + +const root = path.resolve(import.meta.dirname, '../..'); +const inputPath = path.join(root, 'results', 'sandbox-dax', 'latest.json'); +const outputPath = path.join(root, 'dax.svg'); + +interface DaxResultFile { + timestamp: string; + results: DaxBenchmarkResult[]; +} + +const data = JSON.parse(fs.readFileSync(inputPath, 'utf8')) as DaxResultFile; + +const escape = (value: string | number): string => String(value) + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"'); +const format = (ms: number | undefined): string => (ms && ms > 0 ? `${(ms / 1000).toFixed(2)}s` : '—'); +const displayName = (name: string): string => name.toLowerCase() === 'e2b' + ? 'E2B' + : name.split('-').map((part) => part.charAt(0).toUpperCase() + part.slice(1)).join(' '); +const results = data.results.filter((result) => !result.skipped).map((result) => { + const successful = result.iterations.filter((iteration) => + !iteration.error && typeof iteration.totalMs === 'number' && iteration.totalMs > 0, + ); + const phaseTotal = Math.max(7, ...result.iterations.map((iteration) => iteration.phasesTotal || 0)); + const phaseCompleted = result.iterations.length + ? Math.max(...result.iterations.map((iteration) => iteration.phasesCompleted || 0)) + : 0; + return { + ...result, + successful, + phaseTotal, + phaseCompleted, + successPercent: result.iterations.length ? Math.round(successful.length / result.iterations.length * 100) : 0, + median: result.summary?.totalMs?.median || 0, + }; +}).sort((a, b) => { + if (a.phaseCompleted !== b.phaseCompleted) return b.phaseCompleted - a.phaseCompleted; + if (Boolean(a.successful.length) !== Boolean(b.successful.length)) return b.successful.length - a.successful.length; + return (a.median || Number.MAX_SAFE_INTEGER) - (b.median || Number.MAX_SAFE_INTEGER); +}); + +const width = 1200; +const padding = 24; +const headerHeight = 110; +const sectionGap = 28; +const tableHeaderHeight = 42; +const tableRowHeight = 42; +const tableTop = headerHeight + sectionGap + 34; +const height = tableTop + tableHeaderHeight + results.length * tableRowHeight + 52; +const timestamp = new Date(data.timestamp).toISOString().slice(0, 10); +const logoPath = 'M1036.26,1002.28h237.87l-.93,19.09c-8.38,110.32-49.81,198.3-123.82,262.07-73.09,63.31-170.84,95.43-290.48,95.43-130.81,0-235.55-44.69-311.43-133.6-74.48-87.98-112.65-209.48-112.65-361.23v-60.51c0-96.83,17.7-183.41,51.68-257.43,34.91-74.95,85.19-133.61,149.89-173.63,64.7-40.04,140.12-60.52,225.3-60.52,117.77,0,214.13,32.12,286.29,95.9,72.62,63.3,114.98,153.61,126.15,267.67l1.86,19.08h-238.34l-.93-15.83c-4.65-59.11-20.95-101.94-47.95-127.08-27-25.6-69.83-38.17-127.08-38.17-61.91,0-107.06,20.95-137.33,65.17-31.65,45.15-47.94,117.77-48.87,215.53v74.48c0,102.41,15.36,177.83,45.62,223.91,28.86,44.22,74.01,65.63,137.79,65.63,58.19,0,101.48-12.57,128.95-38.17,26.99-25.14,43.29-66.1,47.48-121.5l.93-16.3Z'; +const cols = [ + { x: 40, label: 'PROVIDER' }, + { x: 220, label: 'PHASES' }, + { x: 300, label: 'SUCCESS' }, + { x: 390, label: 'TOTAL (MED)' }, + { x: 530, label: 'PREPARE' }, + { x: 640, label: 'BUN DL' }, + { x: 735, label: 'BUN UNPACK' }, + { x: 850, label: 'CLONE' }, + { x: 940, label: 'INSTALL' }, + { x: 1050, label: 'TYPECHECK' }, +]; + +let svg = ` + + + + + + + + + + + + + DAX Sandbox Benchmarks + Build and typecheck workload performance across sandbox providers + Detailed Metrics +`; + +const tableHeaderY = tableTop; +svg += ` + +`; +for (const col of cols) svg += ` ${col.label}\n`; + +results.forEach((result, index) => { + const y = tableHeaderY + tableHeaderHeight + index * tableRowHeight; + const successful = result.successful.length; + const total = result.iterations.length; + const successClass = result.successPercent >= 100 ? 'good' : result.successPercent ? 'warn' : 'bad'; + const summary = result.summary || {}; + const values = [ + summary.totalMs?.median, + summary.prepareMs?.median, + summary.bunDownloadMs?.median, + summary.bunUnpackMs?.median, + summary.cloneMs?.median, + summary.installMs?.median, + summary.typecheckMs?.median, + ]; + svg += ` + ${escape(displayName(result.provider))} + ${result.phaseCompleted}/${result.phaseTotal} + ${result.successPercent}% + ${successful ? format(values[0]) : 'Failed'} +`; + values.slice(1).forEach((value, valueIndex) => { + svg += ` ${successful ? format(value) : '—'}\n`; + }); +}); + +svg += ` DAX phases: prepare, Bun download, Bun unpack, clone, install, and typecheck. Lower duration is better. +\n`; + +fs.writeFileSync(outputPath, svg); +console.log(`Wrote ${outputPath}`); diff --git a/benchmarks/sandbox/tti.bench.ts b/benchmarks/sandbox/tti.bench.ts index bbb7a80c..a1b9aa7f 100644 --- a/benchmarks/sandbox/tti.bench.ts +++ b/benchmarks/sandbox/tti.bench.ts @@ -3,17 +3,23 @@ * first command (`node -v`) succeeding, excluding destroy. Declarative — * exports `config` + `task`; `bench run` owns the entrypoint. * - * One workload, three launch shapes — the only thing that differs is the - * framework's own knobs, so they're CLI flags rather than separate files: - * sequential one at a time (the config default: concurrency 1) - * burst --concurrency N: all slots open at once, so this launches that - * many real sandboxes simultaneously — raise N deliberately - * staggered --concurrency N --stagger-delay-ms D: task i starts at i * D - * Each shape reports under its own platform benchmark via `--slug`/`--name`. + * One workload, three launch shapes. Each shape is its own platform benchmark + * but the same task, so they're declared once in `shapes` and picked with + * `--shape`; the scale knobs (`--iterations`/`--concurrency`) stay on the CLI: + * sequential the base config (concurrency 1) — no `--shape` + * burst --concurrency N: all slots open at once, launching that many + * real sandboxes simultaneously — raise N deliberately + * staggered its 200ms delay is baked into the shape; task i starts at i * D * * bench run benchmarks/sandbox/tti.bench.ts --iterations 5 --provider e2b,modal - * bench run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 10 --concurrency 10 - * bench run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 10 --concurrency 10 --stagger-delay-ms 200 + * bench run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 10 --concurrency 10 + * bench run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 10 --concurrency 10 + * + * To rank providers against each other, run each provider with the same + * `--run-key`: they get-or-create one shared run and each claims its own worker. + * + * bench run benchmarks/sandbox/tti.bench.ts --shape burst --provider e2b --run-key "$GITHUB_RUN_ID" + * bench run benchmarks/sandbox/tti.bench.ts --shape burst --provider modal --run-key "$GITHUB_RUN_ID" */ import '../src/env.js'; import path from 'node:path'; @@ -52,6 +58,13 @@ export const config = defineBenchmarkConfig({ benchmarkKind: 'sandbox', iterations: 2, concurrency: 1, + // The launch shapes: same task, distinct platform identities. `--shape burst` + // just swaps the slug/name — the caller brings `--concurrency`. Staggered + // also carries its defining 200ms delay so no caller has to remember it. + shapes: { + burst: { slug: 'sandbox-burst-local', name: 'Sandbox burst TTI (local)' }, + staggered: { slug: 'sandbox-staggered-local', name: 'Sandbox staggered TTI (local)', staggerDelayMs: 200 }, + }, participants: providers, onComplete: (outcome) => { const { resultsDir, mode } = legacyShape(outcome.config); diff --git a/benchmarks/src/merge-results.ts b/benchmarks/src/merge-results.ts index 793546c5..7da01477 100644 --- a/benchmarks/src/merge-results.ts +++ b/benchmarks/src/merge-results.ts @@ -1,7 +1,7 @@ /** * Merge per-provider benchmark results into combined result files. * - * Usage: tsx src/merge-results.ts --input [--mode storage|snapshot-fork|browser|browser-throughput|ai-gateway] + * Usage: tsx src/merge-results.ts --input [--mode storage|snapshot-fork|browser|browser-throughput|ai-gateway|git] * * By default, merges sandbox benchmark results: reads latest.json files from * the input directory, groups by mode (sequential/staggered/burst), computes @@ -21,6 +21,9 @@ * With --mode ai-gateway, merges AI gateway benchmark results: deduplicates * by provider, computes AI-gateway-specific composite scores, and writes * combined files to results/ai-gateway/latest.json. + * + * With --mode git, merges git benchmark results: deduplicates by provider + * and writes combined files to results/git/latest.json. */ import fs from 'fs'; import path from 'path'; @@ -34,12 +37,14 @@ import { } from '../browser/throughput-scoring.js'; import { computeAIGatewayCompositeScores, sortAIGatewayByCompositeScore } from '../ai-gateway/scoring.js'; import { printResultsTable, writeResultsJson } from '../sandbox/table.js'; +import { writeGitResultsJson } from '../git/legacy-results.js'; import type { BenchmarkResult } from '../sandbox/types.js'; import type { StorageBenchmarkResult } from '../storage/types.js'; import type { SnapshotForkBenchmarkResult } from '../storage/snapshot-fork-types.js'; import type { BrowserBenchmarkResult } from '../browser/types.js'; import type { ThroughputBenchmarkResult } from '../browser/throughput-types.js'; import type { AIGatewayBenchmarkResult } from '../ai-gateway/types.js'; +import type { GitBenchmarkResult } from '../git/types.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(__dirname, '../..'); @@ -734,6 +739,85 @@ async function mainAIGateway() { console.log(`Copied latest: ${latestPath}`); } +function printGitResultsTable(results: GitBenchmarkResult[]): void { + const sorted = [...results].sort((a, b) => (a.provider).localeCompare(b.provider)); + + console.log(`\n${'='.repeat(95)}`); + console.log(' GIT BENCHMARK RESULTS'); + console.log('='.repeat(95)); + console.log( + ['Provider', 'Clone', 'Push', 'Pull', 'Success'] + .map((h, i) => h.padEnd([14, 12, 12, 12, 10][i])) + .join(' | '), + ); + console.log( + [14, 12, 12, 12, 10].map((w) => '-'.repeat(w)).join('-+-'), + ); + + for (const r of sorted) { + if (r.skipped) { + console.log([r.provider.padEnd(14), '--'.padEnd(12), '--'.padEnd(12), '--'.padEnd(12), 'SKIPPED'.padEnd(10)].join(' | ')); + continue; + } + const ok = r.iterations.filter((i) => !i.error).length; + const total = r.iterations.length; + const clone = (r.summary.cloneMs.median / 1000).toFixed(2) + 's'; + const push = (r.summary.pushMs.median / 1000).toFixed(2) + 's'; + const pull = (r.summary.pullMs.median / 1000).toFixed(2) + 's'; + console.log([r.provider.padEnd(14), clone.padEnd(12), push.padEnd(12), pull.padEnd(12), `${ok}/${total}`.padEnd(10)].join(' | ')); + } + console.log('='.repeat(95)); +} + +async function mainGit() { + const jsonFiles: string[] = []; + function walk(dir: string) { + if (!fs.existsSync(dir)) return; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else if (entry.name === 'latest.json') jsonFiles.push(full); + } + } + walk(inputDir!); + + if (jsonFiles.length === 0) { + console.error(`No latest.json files found in ${inputDir}`); + process.exit(1); + } + + console.log(`Found ${jsonFiles.length} result files`); + + const seen = new Map(); + + for (const file of jsonFiles) { + const raw = JSON.parse(fs.readFileSync(file, 'utf-8')) as { results: GitBenchmarkResult[] }; + const fromSingleProvider = raw.results.length === 1; + for (const result of raw.results) { + const existing = seen.get(result.provider); + if (!existing || (fromSingleProvider && !existing.fromSingleProvider)) { + seen.set(result.provider, { result, fromSingleProvider }); + } + } + } + + const deduped = Array.from(seen.values()).map((e) => e.result); + console.log(`\nMerging ${deduped.length} provider results for mode: git`); + + printGitResultsTable(deduped); + + const timestamp = new Date().toISOString().slice(0, 10); + const resultsDir = path.resolve(ROOT, 'results/git'); + fs.mkdirSync(resultsDir, { recursive: true }); + + const outPath = path.join(resultsDir, `${timestamp}.json`); + await writeGitResultsJson(deduped, outPath); + + const latestPath = path.join(resultsDir, 'latest.json'); + fs.copyFileSync(outPath, latestPath); + console.log(`Copied latest: ${latestPath}`); +} + const runner = mergeMode === 'storage' ? mainStorage : mergeMode === 'snapshot-fork' @@ -744,6 +828,8 @@ const runner = mergeMode === 'storage' ? mainBrowserThroughput : mergeMode === 'ai-gateway' ? mainAIGateway + : mergeMode === 'git' + ? mainGit : main; runner().catch(err => { console.error('Merge failed:', err); diff --git a/dax.svg b/dax.svg new file mode 100644 index 00000000..9f169b75 --- /dev/null +++ b/dax.svg @@ -0,0 +1,304 @@ + + + + + + + + + + + + + + DAX Sandbox Benchmarks + Build and typecheck workload performance across sandbox providers + Detailed Metrics + + + PROVIDER + PHASES + SUCCESS + TOTAL (MED) + PREPARE + BUN DL + BUN UNPACK + CLONE + INSTALL + TYPECHECK + + Namespace + 7/7 + 67% + 35.50s + 2.68s + 0.18s + 0.42s + 1.17s + 7.60s + 19.58s + + Lightning + 7/7 + 100% + 41.56s + 4.97s + 0.52s + 0.62s + 2.68s + 11.94s + 16.88s + + Blaxel + 7/7 + 33% + 44.39s + 2.18s + 0.34s + 0.62s + 2.19s + 11.74s + 33.33s + + Createos + 7/7 + 100% + 50.25s + 2.98s + 0.21s + 0.47s + 2.12s + 10.64s + 26.62s + + Upstash + 7/7 + 100% + 51.25s + 3.68s + 0.26s + 0.62s + 1.75s + 10.45s + 25.11s + + Tensorlake + 7/7 + 100% + 56.39s + 12.59s + 0.23s + 0.57s + 1.68s + 10.30s + 27.59s + + Tenki + 7/7 + 100% + 66.26s + 3.89s + 0.68s + 0.62s + 4.17s + 15.55s + 36.86s + + Daytona + 7/7 + 100% + 70.43s + 2.82s + 0.31s + 0.56s + 3.94s + 13.75s + 37.25s + + E2B + 7/7 + 100% + 78.13s + 8.09s + 0.51s + 0.75s + 2.92s + 18.17s + 41.15s + + Vercel + 7/7 + 100% + 80.13s + 17.17s + 0.17s + 0.68s + 2.27s + 11.86s + 42.38s + + Superserve + 7/7 + 100% + 88.19s + 4.51s + 0.30s + 0.57s + 4.41s + 15.64s + 59.94s + + Isorun + 7/7 + 67% + 94.34s + 0.98s + 0.35s + 0.54s + 1.50s + 10.97s + 27.39s + + Modal + 7/7 + 100% + 99.76s + 4.03s + 0.35s + 0.70s + 2.40s + 15.62s + 69.42s + + Declaw + 7/7 + 67% + 179.48s + 5.97s + 0.66s + 0.66s + 4.55s + 15.14s + 139.80s + + Sandbox0 + 7/7 + 100% + 183.00s + 3.81s + 1.12s + 0.68s + 7.62s + 73.26s + 79.58s + + Opencomputer + 6/7 + 0% + Failed + — + — + — + — + — + — + + Archil + 6/7 + 0% + Failed + — + — + — + — + — + — + + Runloop + 2/7 + 100% + 85.04s + — + — + — + — + 19.77s + 37.04s + + Cloud Run + 2/7 + 0% + Failed + — + — + — + — + — + — + + Beam + 0/7 + 0% + Failed + — + — + — + — + — + — + + Cloudflare + 0/7 + 0% + Failed + — + — + — + — + — + — + + Codesandbox + 0/7 + 0% + Failed + — + — + — + — + — + — + + Hopx + 0/7 + 0% + Failed + — + — + — + — + — + — + + Northflank + 0/7 + 0% + Failed + — + — + — + — + — + — + DAX phases: prepare, Bun download, Bun unpack, clone, install, and typecheck. Lower duration is better. + diff --git a/package.json b/package.json index b28c7ece..489cb889 100644 --- a/package.json +++ b/package.json @@ -5,30 +5,30 @@ "type": "module", "scripts": { "typecheck": "tsc --noEmit", - "bench": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3", + "bench": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3", "bench:sequential": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts", - "bench:staggered": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200", - "bench:burst": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3", + "bench:staggered": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3", + "bench:burst": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3", "bench:sandbox:dax": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/dax.bench.ts", - "bench:blaxel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider blaxel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider blaxel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider blaxel", - "bench:codesandbox": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider codesandbox && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider codesandbox && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider codesandbox", - "bench:daytona": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider daytona && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider daytona && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider daytona", - "bench:e2b": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider e2b && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider e2b && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider e2b", - "bench:hopx": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider hopx && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider hopx && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider hopx", - "bench:isorun": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider isorun && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider isorun && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider isorun", - "bench:modal": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider modal && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider modal && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider modal", - "bench:namespace": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider namespace && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider namespace && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider namespace", - "bench:northflank": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider northflank && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider northflank && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider northflank", - "bench:railway": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider railway && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider railway && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider railway", - "bench:render": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider render && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider render && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider render", - "bench:runloop": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider runloop && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider runloop && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider runloop", - "bench:vercel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider vercel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider vercel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider vercel", - "bench:just-bash": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider just-bash && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider just-bash && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider just-bash", - "bench:sprites": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sprites && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider sprites && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider sprites", - "bench:sandbox0": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sandbox0 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider sandbox0 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider sandbox0", - "bench:sail": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sail && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider sail && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider sail", - "bench:opencomputer": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider opencomputer && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider opencomputer && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider opencomputer", - "bench:tensorlake": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider tensorlake && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider tensorlake && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider tensorlake", + "bench:blaxel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider blaxel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider blaxel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider blaxel", + "bench:codesandbox": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider codesandbox && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider codesandbox && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider codesandbox", + "bench:daytona": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider daytona && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider daytona && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider daytona", + "bench:e2b": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider e2b && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider e2b && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider e2b", + "bench:hopx": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider hopx && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider hopx && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider hopx", + "bench:isorun": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider isorun && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider isorun && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider isorun", + "bench:modal": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider modal && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider modal && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider modal", + "bench:namespace": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider namespace && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider namespace && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider namespace", + "bench:northflank": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider northflank && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider northflank && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider northflank", + "bench:railway": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider railway && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider railway && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider railway", + "bench:render": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider render && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider render && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider render", + "bench:runloop": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider runloop && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider runloop && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider runloop", + "bench:vercel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider vercel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider vercel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider vercel", + "bench:just-bash": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider just-bash && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider just-bash && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider just-bash", + "bench:sprites": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sprites && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider sprites && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider sprites", + "bench:sandbox0": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sandbox0 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider sandbox0 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider sandbox0", + "bench:sail": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sail && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider sail && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider sail", + "bench:opencomputer": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider opencomputer && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider opencomputer && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider opencomputer", + "bench:tensorlake": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider tensorlake && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider tensorlake && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider tensorlake", "bench:browser": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts", "bench:browser:browserbase": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts --provider browserbase", "bench:browser:hyperbrowser": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts --provider hyperbrowser", @@ -76,6 +76,7 @@ "generate-svg:sequential": "tsx benchmarks/sandbox/generate-svg.ts --mode sequential", "generate-svg:staggered": "tsx benchmarks/sandbox/generate-svg.ts --mode staggered", "generate-svg:burst": "tsx benchmarks/sandbox/generate-svg.ts --mode burst", + "generate-dax-svg": "tsx benchmarks/sandbox/generate-dax-svg.ts", "generate-storage-svg": "tsx benchmarks/storage/generate-svg.ts", "generate-snapshot-fork-svg": "tsx benchmarks/storage/generate-snapshot-fork-svg.ts", "generate-browser-svg": "tsx benchmarks/browser/generate-svg.ts", @@ -89,7 +90,12 @@ "bench:ai-gateway:pydantic": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway.bench.ts --provider pydantic-ai-gateway", "bench:ai-gateway:concentrate": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway.bench.ts --provider concentrate-ai-gateway", "bench:ai-gateway:anthropic": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway.bench.ts --provider anthropic-direct", - "generate-ai-gateway-svg": "tsx benchmarks/ai-gateway/generate-svg.ts" + "generate-ai-gateway-svg": "tsx benchmarks/ai-gateway/generate-svg.ts", + "bench:git": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts", + "bench:git:github": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts --provider github", + "bench:git:gitlab": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts --provider gitlab", + "bench:git:bitbucket": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts --provider bitbucket", + "bench:git:tensorlake": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts --provider tensorlake" }, "dependencies": { "@aws-sdk/client-s3": "^3.1076.0", diff --git a/packages/benchsdk-runner/src/__tests__/bench-config.test.ts b/packages/benchsdk-runner/src/__tests__/bench-config.test.ts index d390074b..da7df4c4 100644 --- a/packages/benchsdk-runner/src/__tests__/bench-config.test.ts +++ b/packages/benchsdk-runner/src/__tests__/bench-config.test.ts @@ -85,6 +85,31 @@ describe('defineBenchmarkConfig', () => { defineBenchmarkConfig({ benchmarkSlug: 's', benchmarkName: 'n', participants, phases: [{ name: 'cold', iterations: 1 }, { name: 'cold', iterations: 1 }] }), ).toThrow('duplicate phase name'); }); + + it('accepts a valid shapes map', () => { + const config = defineBenchmarkConfig({ + benchmarkSlug: 's', + benchmarkName: 'n', + participants, + shapes: { + burst: { slug: 'sandbox-burst-local', name: 'Burst' }, + staggered: { slug: 'sandbox-staggered-local', staggerDelayMs: 200 }, + }, + }); + expect(config.shapes?.burst.slug).toBe('sandbox-burst-local'); + }); + + it('rejects a shape without a lowercase slug', () => { + expect(() => + defineBenchmarkConfig({ benchmarkSlug: 's', benchmarkName: 'n', participants, shapes: { burst: { slug: 'Burst' } } }), + ).toThrow('burst'); + }); + + it('rejects a shape with a negative staggerDelayMs', () => { + expect(() => + defineBenchmarkConfig({ benchmarkSlug: 's', benchmarkName: 'n', participants, shapes: { s: { slug: 'ok', staggerDelayMs: -1 } } }), + ).toThrow('staggerDelayMs'); + }); }); describe('defineTask', () => { diff --git a/packages/benchsdk-runner/src/__tests__/cli.test.ts b/packages/benchsdk-runner/src/__tests__/cli.test.ts index 85faabab..922a9fcf 100644 --- a/packages/benchsdk-runner/src/__tests__/cli.test.ts +++ b/packages/benchsdk-runner/src/__tests__/cli.test.ts @@ -5,13 +5,19 @@ import { NoAvailableParticipantsError } from '../no-available-participants.js'; const fixture = (name: string) => `src/__tests__/fixtures/${name}`; describe('runBenchmarkFile', () => { + it('rejects retired imperative commands (there is no `create`)', async () => { + await expect(runBenchmarkFile(['create', 'benchmark', 'sandbox'])).rejects.toThrow(/Usage:/); + await expect(runBenchmarkFile(['create', 'run'])).rejects.toThrow(/Usage:/); + }); + it('rejects when the command is not `run`', async () => { - await expect(runBenchmarkFile([])).rejects.toThrow(/Usage: bench run/); - await expect(runBenchmarkFile(['nope', fixture('good.bench.ts')])).rejects.toThrow(/Usage: bench run/); + await expect(runBenchmarkFile([])).rejects.toThrow(/Usage:/); + await expect(runBenchmarkFile(['nope', fixture('good.bench.ts')])).rejects.toThrow(/Usage:/); }); - it('rejects when no file is given', async () => { - await expect(runBenchmarkFile(['run'])).rejects.toThrow(/Usage: bench run/); + it('rejects when no file is given, or a flag stands where the file should', async () => { + await expect(runBenchmarkFile(['run'])).rejects.toThrow(/Usage:/); + await expect(runBenchmarkFile(['run', '--shape', 'burst'])).rejects.toThrow(/Usage:/); }); it('rejects a module that does not export a config', async () => { diff --git a/packages/benchsdk-runner/src/__tests__/runner.test.ts b/packages/benchsdk-runner/src/__tests__/runner.test.ts index d6c0db22..7edffce3 100644 --- a/packages/benchsdk-runner/src/__tests__/runner.test.ts +++ b/packages/benchsdk-runner/src/__tests__/runner.test.ts @@ -49,15 +49,23 @@ describe('parseCliArgs', () => { expect(parseCliArgs(['--group-by=participant'])).toEqual({ groupBy: 'participant' }); }); - it('parses --slug and --name', () => { - expect(parseCliArgs(['--slug', 'sandbox-burst-local'])).toEqual({ slug: 'sandbox-burst-local' }); - expect(parseCliArgs(['--slug=sandbox-tti-local'])).toEqual({ slug: 'sandbox-tti-local' }); + it('parses --benchmark (and its --slug alias) and --name', () => { + expect(parseCliArgs(['--benchmark', 'sandbox-burst-local'])).toEqual({ benchmark: 'sandbox-burst-local' }); + expect(parseCliArgs(['--benchmark=sandbox-tti-local'])).toEqual({ benchmark: 'sandbox-tti-local' }); + expect(parseCliArgs(['--slug', 'sandbox-burst-local'])).toEqual({ benchmark: 'sandbox-burst-local' }); expect(parseCliArgs(['--name', 'Sandbox burst TTI'])).toEqual({ name: 'Sandbox burst TTI' }); expect(() => parseCliArgs(['--name', ' '])).toThrow('--name'); }); - it('throws on a non-slug --slug', () => { - expect(() => parseCliArgs(['--slug', 'Sandbox TTI'])).toThrow('--slug'); + it('parses --shape and --run-key', () => { + expect(parseCliArgs(['--shape', 'burst'])).toEqual({ shape: 'burst' }); + expect(parseCliArgs(['--run-key=ci-123'])).toEqual({ runKey: 'ci-123' }); + expect(() => parseCliArgs(['--shape', ''])).toThrow('--shape'); + expect(() => parseCliArgs(['--run-key', ''])).toThrow('--run-key'); + }); + + it('throws on a non-slug --benchmark', () => { + expect(() => parseCliArgs(['--benchmark', 'Sandbox TTI'])).toThrow('--benchmark'); expect(() => parseCliArgs(['--slug', ''])).toThrow('--slug'); }); @@ -164,14 +172,19 @@ describe('runBenchmark', () => { process.env.E2B_API_KEY = 'x'; process.env.MODAL_TOKEN = 'y'; process.env.BENCHMARKS_PLATFORM_API_KEY = 'test-key'; - calls = { upsertBenchmark: [], createRun: [], planWorkers: [], runWorker: [], taskData: [] }; + calls = { upsertBenchmark: [], createRun: [], planWorkers: [], upsertParticipant: [], getRun: [], runWorker: [], taskData: [] }; fakeClient = { upsertBenchmark: vi.fn(async (...a: any[]) => { calls.upsertBenchmark.push(a); return {}; }), createRun: vi.fn(async (...a: any[]) => { calls.createRun.push(a); return { run: { id: 'run-1' }, participants: [] }; }), planWorkers: vi.fn(async (...a: any[]) => { calls.planWorkers.push(a); return []; }), + upsertParticipant: vi.fn(async (...a: any[]) => { calls.upsertParticipant.push(a); return {}; }), + getRun: vi.fn(async (slug: string, runId: string) => { + calls.getRun.push([slug, runId]); + return { id: runId, totalTasks: 3, participantSized: runId === 'run-open' }; + }), runWorker: vi.fn(async (opts: any) => { calls.runWorker.push(opts); - const total = calls.createRun[0]?.[1]?.totalTasks ?? 1; + const total = calls.createRun[0]?.[1]?.totalTasks ?? calls.upsertParticipant[0]?.[3]?.totalTasks ?? 1; // The platform hands out globally-indexed task ranges; `taskRangeStart` // lets a test exercise a worker whose range doesn't start at 0. const start = taskRangeStart; @@ -294,6 +307,84 @@ describe('runBenchmark', () => { expect(calls.createRun[0][0]).toBe('sandbox-burst-local'); }); + it('shares a run by --run-key: get-or-creates it keyed, then registers only its own participant', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + iterations: 2, + participants: [participants[0]], + }; + + const outcome = await runBenchmark(config, defineTask(async () => ({})), ['--run-key', 'ci-123', '--provider', 'e2b']); + + // The benchmark is still materialized from the file identity. + expect(calls.upsertBenchmark[0][0]).toBe('sandbox-tti-local'); + // One keyed get-or-create, carrying the key and no size/participant list. + expect(calls.createRun).toHaveLength(1); + expect(calls.createRun[0][1]).toMatchObject({ runKey: 'ci-123' }); + expect(calls.createRun[0][1].totalTasks).toBeUndefined(); + expect(calls.createRun[0][1].participants).toBeUndefined(); + // Registers only the provider it runs, sized to its own iteration count. + expect(calls.upsertParticipant[0].slice(0, 3)).toEqual(['sandbox-tti-local', 'run-1', 'e2b']); + expect(calls.upsertParticipant[0][3]).toMatchObject({ totalTasks: 2 }); + expect(calls.runWorker[0]).toMatchObject({ runId: 'run-1' }); + expect(outcome.runId).toBe('run-1'); + expect(outcome.participants[0].records).toHaveLength(2); + }); + + it('does not rename a benchmark it was merely retargeted at', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + iterations: 1, + participants: [participants[0]], + }; + + await runBenchmark(config, defineTask(async () => ({})), ['--benchmark', 'sandbox-burst-local']); + expect(calls.upsertBenchmark).toEqual([]); + + await runBenchmark(config, defineTask(async () => ({})), [ + '--benchmark', + 'sandbox-burst-local', + '--name', + 'Sandbox burst TTI', + ]); + expect(calls.upsertBenchmark[0]).toEqual(['sandbox-burst-local', { name: 'Sandbox burst TTI' }]); + }); + + it('selects a declared shape by --shape, reporting under its slug and name', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + iterations: 1, + participants: [participants[0]], + shapes: { + staggered: { slug: 'sandbox-staggered-local', name: 'Sandbox staggered TTI', staggerDelayMs: 200 }, + }, + }; + + const outcome = await runBenchmark(config, defineTask(async () => ({})), ['--shape', 'staggered']); + + expect(calls.upsertBenchmark[0][0]).toBe('sandbox-staggered-local'); + expect(calls.upsertBenchmark[0][1]).toMatchObject({ name: 'Sandbox staggered TTI' }); + expect(calls.createRun[0][0]).toBe('sandbox-staggered-local'); + // The shape's stable knob applies; scale knobs stay defaulted/overridable. + expect(outcome.config.staggerDelayMs).toBe(200); + }); + + it('rejects an unknown --shape, listing the declared ones', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + participants: [participants[0]], + shapes: { burst: { slug: 'sandbox-burst-local' } }, + }; + + await expect( + runBenchmark(config, defineTask(async () => ({})), ['--shape', 'nope']), + ).rejects.toThrow('Known shapes: burst'); + }); + it('throws NoAvailableParticipantsError, listing the skips, when no participant has its env vars set', async () => { delete process.env.E2B_API_KEY; delete process.env.MODAL_TOKEN; diff --git a/packages/benchsdk-runner/src/bench-config.ts b/packages/benchsdk-runner/src/bench-config.ts index b27558ec..c1952ab2 100644 --- a/packages/benchsdk-runner/src/bench-config.ts +++ b/packages/benchsdk-runner/src/bench-config.ts @@ -19,6 +19,10 @@ * burst { iterations: N, concurrency: N } * staggered { iterations: N, concurrency: N, staggerDelayMs: 200 } * + * A benchmark can name these variants up front via `shapes`, so one file backs + * several platform benchmarks (`bench run --shape burst`) without + * restating each one's slug/name in scripts and CI. + * * A task is comprised of steps, declared via `ctx.step` inside a task function * — it supports closures, conditionals and try/finally, so values (a created * sandbox, say) flow naturally between steps. A task that declares no steps is @@ -37,6 +41,25 @@ import type { /** How tasks are ordered across participants. */ export type GroupBy = 'participant' | 'round'; +/** + * A named variant of a benchmark, selected with `--shape `. A shape + * carries only the parts that make it a distinct *benchmark* — its platform + * identity plus any stable distinguishing knob (e.g. staggered's delay). The + * scale knobs that vary per environment (`--iterations`, `--concurrency`) stay + * on the invocation, so a shape never sets a value only to have the CLI + * override it. + */ +export interface BenchmarkShape { + /** Platform slug this shape reports under (e.g. 'sandbox-burst-local'). */ + slug: string; + /** Display name shown on the platform; defaults to the slug. */ + name?: string; + /** Benchmark kind; defaults to the config's `benchmarkKind`. */ + kind?: string; + /** Default stagger delay (ms) for this shape; overridable with `--stagger-delay-ms`. */ + staggerDelayMs?: number; +} + /** * What a task returns: whatever it measured itself. This replaces the * assumption that the framework owns all timing. A plain data payload is @@ -132,6 +155,7 @@ export interface ResolvedRunConfig { */ export interface BenchmarkRunOutcome { runId: string; + /** Link to this run on the platform dashboard. */ dashboardUrl: string; participants: ParticipantRecords[]; config: ResolvedRunConfig; @@ -146,14 +170,22 @@ export interface BenchmarkRunOutcome { export interface BenchmarkConfig { /** * Stable platform slug for this benchmark (e.g. 'sandbox-tti-local'). - * Overridable per run with `--slug`, so one entrypoint can report under - * several benchmarks. + * Selectable per run with `--shape` (or overridable with `--benchmark`), so + * one entrypoint can report under several benchmarks. */ benchmarkSlug: string; /** Human-readable name shown on the platform. Overridable with `--name`. */ benchmarkName: string; /** Optional platform benchmark kind (e.g. 'sandbox'). */ benchmarkKind?: string; + /** + * Named variants of this benchmark, selected with `--shape `. Each + * shape swaps in its own platform identity (and optional stable knob) while + * reusing the same task and participants, so one bench file can back several + * platform benchmarks without duplicating the slug/name triple across + * package scripts and CI. + */ + shapes?: Record; /** * Total tasks to run per participant. Default: 1. Mutually exclusive with * `phases` — when `phases` is set, total iterations = sum of phase iterations. @@ -234,6 +266,19 @@ export function defineBenchmarkConfig= 0 (got ${shape.staggerDelayMs})`); + } + } + } return config; } diff --git a/packages/benchsdk-runner/src/cli.ts b/packages/benchsdk-runner/src/cli.ts index c76b4587..71089261 100644 --- a/packages/benchsdk-runner/src/cli.ts +++ b/packages/benchsdk-runner/src/cli.ts @@ -1,23 +1,31 @@ /** - * `bench run [--flags]` — the author-facing entrypoint. Imports a - * benchmark module, reads its `config` and `task` exports, and drives the run - * via the internal `runBenchmark`. CLI flags override the config's knobs, and - * `config.onComplete` (if any) fires once the run finishes. + * The author-facing entrypoint. `bench` is verbs-only — the benchmark and its + * runs are implicit, never nouns you type: + * + * bench run [--flags] execute a benchmark + * + * `run` imports a benchmark module, reads its `config` and `task` exports and + * drives `runBenchmark`; CLI flags override the config's knobs and + * `config.onComplete` (if any) fires once the run finishes. The benchmark is + * declared in the file (`--shape` picks a named variant) and materialized on + * run; a run is opened as a side effect, shared across sibling processes when + * they pass the same `--run-key`. There are no imperative `create` commands. * * The executable wrapper lives in `bin.ts`; this module has no side effects so * it can be unit-tested by calling `runBenchmarkFile` directly. */ import { resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { runBenchmark } from './runner.js'; +import { parseCliArgs, runBenchmark } from './runner.js'; import { NoAvailableParticipantsError } from './no-available-participants.js'; import type { BaseParticipant } from '@benchsdk/client'; import type { BenchmarkConfig, BenchmarkTask } from './bench-config.js'; const USAGE = - 'Usage: bench run [--iterations N] [--concurrency N] ' + - '[--stagger-delay-ms N] [--group-by participant|round] [--provider a,b] ' + - "[--slug my-benchmark] [--name 'My benchmark']"; + 'Usage:\n' + + ' bench run [--shape name] [--provider a,b] [--run-key key]\n' + + ' [--benchmark slug] [--name "My benchmark"]\n' + + ' [--iterations N] [--concurrency N] [--stagger-delay-ms N] [--group-by participant|round]'; /** A benchmark module is expected to export `config` and `task`. */ interface BenchmarkModule { @@ -33,19 +41,17 @@ function isBenchmarkConfig(value: unknown): value is BenchmarkConfig { } /** - * Loads a benchmark file and runs it. Throws on bad usage / invalid exports and - * lets `NoAvailableParticipantsError` propagate so the caller can map it to a - * clean exit. Does not call `process.exit`. + * Dispatches one CLI invocation. Throws on bad usage / invalid exports and lets + * `NoAvailableParticipantsError` propagate so the caller can map it to a clean + * exit. Does not call `process.exit`. */ export async function runBenchmarkFile(argv: string[]): Promise { - const [command, file, ...rest] = argv; - if (command !== 'run' || !file) { - throw new Error(USAGE); - } + const [command, ...rest] = argv; - const moduleUrl = pathToFileURL(resolve(process.cwd(), file)).href; - const mod = (await import(moduleUrl)) as BenchmarkModule; + const [file, ...flags] = rest; + if (command !== 'run' || !file || file.startsWith('-')) throw new Error(USAGE); + const mod = (await import(pathToFileURL(resolve(process.cwd(), file)).href)) as BenchmarkModule; const config = mod.config; const task = mod.task ?? mod.default; @@ -56,7 +62,7 @@ export async function runBenchmarkFile(argv: string[]): Promise { throw new Error(`${file} must export a \`task\` created with defineTask.`); } - await runBenchmark(config as BenchmarkConfig, task as BenchmarkTask, rest); + await runBenchmark(config as BenchmarkConfig, task as BenchmarkTask, flags); } /** Executable entry: runs the file and maps outcomes to process exit codes. */ diff --git a/packages/benchsdk-runner/src/runner.ts b/packages/benchsdk-runner/src/runner.ts index 3ce25d3a..a4be4ec8 100644 --- a/packages/benchsdk-runner/src/runner.ts +++ b/packages/benchsdk-runner/src/runner.ts @@ -24,7 +24,6 @@ import { NoAvailableParticipantsError } from './no-available-participants.js'; import type { BaseParticipant, BenchmarkClient, - BenchmarkRun, JsonObject, RunWorkerContext, TaskResultRecord, @@ -34,6 +33,7 @@ import { TaskError } from './bench-config.js'; import type { BenchmarkConfig, BenchmarkRunOutcome, + BenchmarkShape, BenchmarkTask, GroupBy, ParticipantRecords, @@ -44,8 +44,16 @@ import type { import { LogBuffer } from './log-buffer.js'; export interface CliArgs { - slug?: string; + /** Which platform benchmark to report as (`--benchmark`, aka the benchmark slug). */ + benchmark?: string; name?: string; + /** Named variant from the bench file's `shapes` (`--shape`), swapping in its identity. */ + shape?: string; + /** + * Idempotency key (`--run-key`): sibling processes passing the same key share + * one run (get-or-created), instead of each opening its own. + */ + runKey?: string; iterations?: number; concurrency?: number; staggerDelayMs?: number; @@ -98,12 +106,14 @@ export function parseCliArgs(argv: string[]): CliArgs { const arg = argv[i]; const name = arg.includes('=') ? arg.slice(0, arg.indexOf('=')) : arg; switch (name) { - case '--slug': { + // `--slug` is the pre-`--benchmark` spelling, kept working for existing scripts. + case '--slug': + case '--benchmark': { const { value, nextIndex } = readValue(arg, i); if (!/^[a-z0-9][a-z0-9-]*$/.test(value)) { - throw new Error(`--slug expects a lowercase slug (got "${value}")`); + throw new Error(`${name} expects a lowercase benchmark slug (got "${value}")`); } - args.slug = value; + args.benchmark = value; i = nextIndex; break; } @@ -114,6 +124,20 @@ export function parseCliArgs(argv: string[]): CliArgs { i = nextIndex; break; } + case '--shape': { + const { value, nextIndex } = readValue(arg, i); + if (value.trim() === '') throw new Error('--shape expects a value'); + args.shape = value; + i = nextIndex; + break; + } + case '--run-key': { + const { value, nextIndex } = readValue(arg, i); + if (value.trim() === '') throw new Error('--run-key expects a value'); + args.runKey = value; + i = nextIndex; + break; + } case '--iterations': { const { value, nextIndex } = readValue(arg, i); args.iterations = intFlag(value, '--iterations'); @@ -234,40 +258,99 @@ function resolvePlatform(): { baseUrl: string; apiKey: string } { } /** - * Runs `config`'s `task` against `participants`. Selects participants by - * `--provider` (if given), env-gates them, then drives them per the resolved - * `groupBy`. `--slug`/`--name` retarget the whole run at a different platform - * benchmark, so one entrypoint can report under several slugs. + * Resolves `--shape ` against the config's declared `shapes`. Throws with + * the known names if the shape is unknown, so a typo fails loudly instead of + * silently running the base benchmark. */ -export async function runBenchmark( +function resolveShape( + config: BenchmarkConfig, + shapeName: string | undefined, +): BenchmarkShape | undefined { + if (!shapeName) return undefined; + const shape = config.shapes?.[shapeName]; + if (!shape) { + const known = Object.keys(config.shapes ?? {}); + throw new Error( + known.length > 0 + ? `Unknown --shape "${shapeName}". Known shapes: ${known.join(', ')}.` + : `Unknown --shape "${shapeName}": this benchmark declares no shapes.`, + ); + } + return shape; +} + +/** + * Swaps a shape's identity (and its stable knob) into the config. Only the + * parts that make it a distinct benchmark move here; scale knobs stay on the + * CLI, so `mergeConfig` still lets `--concurrency`/`--iterations` win. + */ +function applyShape( + config: BenchmarkConfig, + shape: BenchmarkShape | undefined, +): BenchmarkConfig { + if (!shape) return config; + const kind = shape.kind ?? config.benchmarkKind; + return { + ...config, + benchmarkSlug: shape.slug, + benchmarkName: shape.name ?? shape.slug, + ...(kind ? { benchmarkKind: kind } : {}), + ...(shape.staggerDelayMs !== undefined ? { staggerDelayMs: shape.staggerDelayMs } : {}), + }; +} + +/** Applies the `--benchmark`/`--name` overrides, so one entrypoint can report under several benchmarks. */ +function applyIdentityOverrides( fileConfig: BenchmarkConfig, - task: BenchmarkTask, - argv: string[] = [], -): Promise { - const args = parseCliArgs(argv); - const config = { + args: CliArgs, +): BenchmarkConfig { + return { ...fileConfig, - ...(args.slug ? { benchmarkSlug: args.slug } : {}), + ...(args.benchmark ? { benchmarkSlug: args.benchmark } : {}), ...(args.name ? { benchmarkName: args.name } : {}), }; - const resolved = mergeConfig(config, args); - const schedule = buildSchedule(config, resolved.iterations, task); - const totalTasks = schedule.length; +} - const selected = selectParticipants(config.participants, resolved.providers); - const { available, skipped } = filterParticipantsByEnv(selected); +function dashboardUrlFor(baseUrl: string, organizationSlug: string, benchmarkSlug: string, runId: string): string { + return `${baseUrl.replace(/\/api\/v1\/?$/, '')}/${organizationSlug}/benchmarks/${benchmarkSlug}/runs/${runId}`; +} +/** The participants a run covers: `--provider` selection, minus any whose env vars are unset. */ +function resolveParticipants(config: BenchmarkConfig, resolved: ResolvedRunConfig): T[] { + const { available, skipped } = filterParticipantsByEnv(selectParticipants(config.participants, resolved.providers)); for (const s of skipped) { console.log(`Skipping ${s.name}: missing ${s.missing.join(', ')}`); } + if (available.length === 0) throw new NoAvailableParticipantsError(skipped); + return available; +} - if (available.length === 0) { - throw new NoAvailableParticipantsError(skipped); - } +/** + * Runs `config`'s `task` against its participants. Selects participants by + * `--provider` (if given), env-gates them, then drives them per the resolved + * `groupBy`. `--shape` swaps in a declared variant's identity; `--benchmark`/ + * `--name` retarget the run at a different platform benchmark, so one entrypoint + * can report under several slugs. With `--run-key`, sibling processes (e.g. one + * CI job per provider) get-or-create one shared run and each registers only its + * own participants. + */ +export async function runBenchmark( + fileConfig: BenchmarkConfig, + task: BenchmarkTask, + argv: string[] = [], +): Promise { + const args = parseCliArgs(argv); + const shaped = applyShape(fileConfig, resolveShape(fileConfig, args.shape)); + const config = applyIdentityOverrides(shaped, args); + const resolved = mergeConfig(config, args); + const available = resolveParticipants(config, resolved); const { baseUrl, apiKey } = resolvePlatform(); const client = createBenchmarkClient({ baseUrl, apiKey }); + const schedule = buildSchedule(config, resolved.iterations, task); + const totalTasks = schedule.length; + const concurrencyLabel = resolved.groupBy === 'round' ? 'n/a (round mode)' : String(resolved.concurrency); console.log(`${config.benchmarkName} (self-contained)`); console.log(`Date: ${new Date().toISOString()}`); @@ -276,34 +359,65 @@ export async function runBenchmark( `staggerDelayMs=${resolved.staggerDelayMs}, groupBy=${resolved.groupBy}\n`, ); - await client.upsertBenchmark(config.benchmarkSlug, { - name: config.benchmarkName, - ...(config.benchmarkKind ? { kind: config.benchmarkKind } : {}), - }); - - const { run, organizationSlug } = await client.createRun(config.benchmarkSlug, { - name: `${config.benchmarkSlug} — ${totalTasks} iterations, concurrency ${resolved.concurrency}`, - totalTasks, - workerCount: 1, - participants: available.map((p) => p.name), - }); + // Declaratively materialize the benchmark from the file/shape identity, which + // is authoritative (its name lives in the file). A bare `--benchmark X` only + // *retargets* reporting at a benchmark this file doesn't name, so we don't + // upsert it — that would rename it to the file's own name. + const identityIsOurs = + args.shape !== undefined || + args.name !== undefined || + !args.benchmark || + args.benchmark === fileConfig.benchmarkSlug; + if (identityIsOurs) { + await client.upsertBenchmark(config.benchmarkSlug, { + name: config.benchmarkName, + ...(config.benchmarkKind ? { kind: config.benchmarkKind } : {}), + }); + } - const dashboardUrl = `${baseUrl.replace(/\/api\/v1\/?$/, '')}/${organizationSlug}/benchmarks/${config.benchmarkSlug}/runs/${run.id}`; - console.log(`Run created: ${run.id}`); - console.log(`View at: ${dashboardUrl}\n`); + let runId: string; + let dashboardUrl: string; + if (args.runKey) { + // Shared run: get-or-created by key, so sibling processes (one per provider) + // converge on one run. Opened participant-sized — register only the + // providers this process runs and let each sibling register its own, so the + // run lists exactly who's benchmarked and each brings its own task count. + const { run, organizationSlug } = await client.createRun(config.benchmarkSlug, { + name: config.benchmarkName, + runKey: args.runKey, + }); + runId = run.id; + dashboardUrl = dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run.id); + for (const participant of available) { + await client.upsertParticipant(config.benchmarkSlug, runId, participant.name, { totalTasks }); + } + console.log(`Shared run (key "${args.runKey}"): ${runId}`); + console.log(`View at: ${dashboardUrl}\n`); + } else { + const { run, organizationSlug } = await client.createRun(config.benchmarkSlug, { + name: `${config.benchmarkSlug} — ${totalTasks} iterations, concurrency ${resolved.concurrency}`, + totalTasks, + workerCount: 1, + participants: available.map((p) => p.name), + }); + runId = run.id; + dashboardUrl = dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run.id); + console.log(`Run created: ${runId}`); + console.log(`View at: ${dashboardUrl}\n`); + } const onResult = defaultOnResult; let participantRecords: ParticipantRecords[]; if (resolved.groupBy === 'round') { - participantRecords = await runGroupedByRound(config, schedule, available, resolved, client, run, baseUrl, apiKey, onResult); + participantRecords = await runGroupedByRound(config, schedule, available, resolved, client, runId, baseUrl, apiKey, onResult); } else { - participantRecords = await runGroupedByParticipant(config, schedule, available, resolved, client, run, onResult); + participantRecords = await runGroupedByParticipant(config, schedule, available, resolved, client, runId, onResult); } console.log(`All done. View at: ${dashboardUrl}`); const outcome: BenchmarkRunOutcome = { - runId: run.id, + runId, dashboardUrl, participants: participantRecords, config: resolved, @@ -325,7 +439,7 @@ async function runGroupedByParticipant( available: T[], resolved: ResolvedRunConfig, client: BenchmarkClient, - run: BenchmarkRun, + runId: string, onResult: OnResult, ): Promise { const participantRecords: ParticipantRecords[] = []; @@ -338,11 +452,11 @@ async function runGroupedByParticipant( // count can't inflate launch offsets: a task whose slot frees after its // scheduled launch time starts immediately instead of sleeping index*delay. let rampStartMs: number | undefined; - await client.planWorkers(config.benchmarkSlug, run.id, participant.name); + await client.planWorkers(config.benchmarkSlug, runId, participant.name); const result = await client.runWorker({ benchmarkSlug: config.benchmarkSlug, - runId: run.id, + runId: runId, participantSlug: participant.name, concurrency: resolved.concurrency, task: async (ctx: RunWorkerContext) => { @@ -372,7 +486,7 @@ async function runGroupedByParticipant( }); if (!result.assignment) { - console.error(` No pending worker to claim for run ${run.id} — it may already be fully claimed.`); + console.error(` No pending worker to claim for run ${runId} — it may already be fully claimed.`); participantRecords.push({ participant: participant.name, records: result.records ?? [] }); continue; } @@ -397,7 +511,7 @@ async function runGroupedByRound( available: T[], resolved: ResolvedRunConfig, client: BenchmarkClient, - run: BenchmarkRun, + runId: string, baseUrl: string, apiKey: string, onResult: OnResult, @@ -414,7 +528,7 @@ async function runGroupedByRound( // reads `targetConcurrency` as tasks-per-worker, so it must be the full // schedule length — otherwise only one task is planned and every record // past the first falls outside the worker's task range. - await client.planWorkers(config.benchmarkSlug, run.id, participant.name, { + await client.planWorkers(config.benchmarkSlug, runId, participant.name, { workerCount: 1, targetConcurrency: schedule.length, }); @@ -424,7 +538,7 @@ async function runGroupedByRound( baseUrl, apiKey, benchmarkSlug: config.benchmarkSlug, - runId: run.id, + runId: runId, participantSlug: participant.name, processKind: 'process', processKey: process.env.HOSTNAME ?? 'local', diff --git a/packages/benchsdk/src/types.ts b/packages/benchsdk/src/types.ts index 644400c5..df9b639c 100644 --- a/packages/benchsdk/src/types.ts +++ b/packages/benchsdk/src/types.ts @@ -28,7 +28,11 @@ export interface BenchmarkRun { benchmarkId: string; name?: string | null; status: BenchmarkRunStatus | string; + /** Idempotency key: runs created with the same key (per org + benchmark) are the same run. */ + runKey?: string | null; totalTasks: number; + /** The run declared no size: `totalTasks` is the sum of what its participants declare. */ + participantSized?: boolean; workerCount: number; config?: JsonObject; createdAt?: string; @@ -116,8 +120,14 @@ export interface UpdateBenchmarkInput { export interface CreateRunInput { name?: string; - totalTasks: number; - workerCount: number; + /** + * Idempotency key for get-or-create: sibling callers passing the same key + * (per org + benchmark) converge on one run instead of each opening its own. + */ + runKey?: string; + /** Omit to open a participant-sized run: each participant declares its own size when it registers. */ + totalTasks?: number; + workerCount?: number; participants?: string[]; config?: JsonObject; } diff --git a/results/git/.gitkeep b/results/git/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tsconfig.json b/tsconfig.json index 6f11950b..c9a9904e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,6 +18,7 @@ "benchmarks/browser/**/*.ts", "benchmarks/storage/**/*.ts", "benchmarks/ai-gateway/**/*.ts", + "benchmarks/git/**/*.ts", "benchmarks/scale/**/*.ts" ], "exclude": ["node_modules", "benchmarks/dist", "packages", "results", ".git"]