From 7d368097c15d23440f0a5ce9adfea2991e0f0af1 Mon Sep 17 00:00:00 2001 From: garrison Date: Wed, 5 Aug 2026 14:24:02 +0000 Subject: [PATCH 01/15] feat(git): add git clone benchmark using isomorphic-git Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- benchmarks/git/git.bench.ts | 77 +++++++++++++++++++++++++++++ benchmarks/git/providers.ts | 34 +++++++++++++ benchmarks/git/types.ts | 39 +++++++++++++++ package.json | 7 ++- pnpm-lock.yaml | 96 +++++++++++++++++++++++++++++++++---- results/git/.gitkeep | 0 tsconfig.json | 1 + 7 files changed, 245 insertions(+), 9 deletions(-) create mode 100644 benchmarks/git/git.bench.ts create mode 100644 benchmarks/git/providers.ts create mode 100644 benchmarks/git/types.ts create mode 100644 results/git/.gitkeep diff --git a/benchmarks/git/git.bench.ts b/benchmarks/git/git.bench.ts new file mode 100644 index 00000000..d269309f --- /dev/null +++ b/benchmarks/git/git.bench.ts @@ -0,0 +1,77 @@ +/** + * Git clone benchmark. Measures shallow clone latency over HTTPS for git + * hosting providers using isomorphic-git as the harness. Declarative — + * exports `config` + `task`; `bench run` owns the entrypoint. + * + * bench run benchmarks/git/git.bench.ts + * bench run benchmarks/git/git.bench.ts --provider github,gitlab --iterations 5 + */ +import '../src/env.js'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import git from 'isomorphic-git'; +import http from 'isomorphic-git/http/node'; +import { defineBenchmarkConfig, defineTask, TaskError } from '@benchsdk/runner'; +import type { AuthCallback } from 'isomorphic-git'; +import { withTimeout } from '../src/util/timeout.js'; +import { formatError } from '../src/util/error.js'; +import { providers } from './providers.js'; +import type { GitProviderConfig } from './types.js'; + +const CLONE_TIMEOUT_MS = 60_000; + +function buildAuth(config: GitProviderConfig): AuthCallback | undefined { + if (!config.tokenEnvVar) return undefined; + const token = process.env[config.tokenEnvVar]; + if (!token) return undefined; + const username = config.tokenUsername ?? 'token'; + return () => ({ username, password: token }); +} + +export const config = defineBenchmarkConfig({ + benchmarkSlug: 'git-clone-local', + benchmarkName: 'Git clone (local)', + benchmarkKind: 'git', + iterations: 3, + concurrency: 1, + participants: providers, +}); + +export const task = defineTask(async (ctx) => { + const { participant, step, measure } = ctx; + const timeout = participant.timeout ?? CLONE_TIMEOUT_MS; + + const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'bench-git-')); + const cloneDir = path.join(tempDir, 'repo'); + + try { + const start = performance.now(); + await step('clone', () => + withTimeout( + git.clone({ + fs: fs as unknown as import('isomorphic-git').FsClient, + http, + dir: cloneDir, + url: participant.url, + singleBranch: true, + depth: 1, + onAuth: buildAuth(participant), + }), + timeout, + 'Git clone timed out', + ), + ); + const cloneMs = performance.now() - start; + + measure({ cloneMs, repoUrl: participant.url }); + return { data: { cloneMs, repoUrl: participant.url } }; + } catch (err) { + throw new TaskError(formatError(err), { + code: 'GIT_CLONE_ERROR', + data: { repoUrl: participant.url, cloneMs: 0 }, + }); + } finally { + await fs.promises.rm(tempDir, { recursive: true, force: true }).catch(() => {}); + } +}); diff --git a/benchmarks/git/providers.ts b/benchmarks/git/providers.ts new file mode 100644 index 00000000..d4142381 --- /dev/null +++ b/benchmarks/git/providers.ts @@ -0,0 +1,34 @@ +import type { GitProviderConfig } from './types.js'; + +/** + * Git hosting provider benchmark configurations. + * + * Each participant points at a small public repo and uses isomorphic-git over + * HTTPS. Tokens are optional: when `tokenEnvVar` is set and present, it is + * passed via `onAuth`; otherwise the clone is anonymous. + */ +export const providers: GitProviderConfig[] = [ + { + name: 'github', + requiredEnvVars: [], + url: 'https://github.com/octocat/Spoon-Knife.git', + tokenEnvVar: 'GITHUB_TOKEN', + tokenUsername: 'token', + }, + { + name: 'gitlab', + requiredEnvVars: [], + url: 'https://gitlab.com/gitlab-org/gitlab-test.git', + tokenEnvVar: 'GITLAB_TOKEN', + tokenUsername: 'oauth2', + }, + { + name: 'bitbucket', + requiredEnvVars: [], + url: 'https://bitbucket.org/atlassian/hello-world.git', + tokenEnvVar: 'BITBUCKET_TOKEN', + tokenUsername: 'x-token-auth', + }, + // + // add git providers above +]; diff --git a/benchmarks/git/types.ts b/benchmarks/git/types.ts new file mode 100644 index 00000000..b686e3eb --- /dev/null +++ b/benchmarks/git/types.ts @@ -0,0 +1,39 @@ +import type { BaseParticipant } from '@benchsdk/client'; + +export interface GitProviderConfig extends BaseParticipant { + /** HTTPS URL of a public repository to clone. */ + url: string; + /** Optional environment variable holding an HTTPS auth token. */ + tokenEnvVar?: string; + /** Username passed to isomorphic-git's onAuth callback when a token is set. */ + tokenUsername?: string; + /** Per-provider timeout for the clone step in ms (default: 60000). */ + timeout?: number; +} + +export interface GitTimingResult { + /** Time to shallow clone the repo in ms. */ + cloneMs: number; + /** Repository URL cloned. */ + repoUrl: string; + /** Error message if this iteration failed. */ + error?: string; +} + +export interface GitStats { + cloneMs: { median: number; p95: number; p99: number }; +} + +export interface GitBenchmarkResult { + provider: string; + mode: 'git'; + repoUrl: string; + iterations: GitTimingResult[]; + summary: GitStats; + /** Composite weighted score (0-100, higher = better). Computed post-benchmark. */ + compositeScore?: number; + /** Success rate as a fraction (0 to 1). Computed post-benchmark. */ + successRate?: number; + skipped?: boolean; + skipReason?: string; +} diff --git a/package.json b/package.json index b28c7ece..37f03eef 100644 --- a/package.json +++ b/package.json @@ -89,7 +89,11 @@ "bench:ai-gateway:pydantic": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway.bench.ts --provider pydantic-ai-gateway", "bench:ai-gateway:concentrate": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway.bench.ts --provider concentrate-ai-gateway", "bench:ai-gateway:anthropic": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway.bench.ts --provider anthropic-direct", - "generate-ai-gateway-svg": "tsx benchmarks/ai-gateway/generate-svg.ts" + "generate-ai-gateway-svg": "tsx benchmarks/ai-gateway/generate-svg.ts", + "bench:git": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts", + "bench:git:github": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts --provider github", + "bench:git:gitlab": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts --provider gitlab", + "bench:git:bitbucket": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts --provider bitbucket" }, "dependencies": { "@aws-sdk/client-s3": "^3.1076.0", @@ -150,6 +154,7 @@ "computesdk": "^4.1.4", "dotenv": "^17.4.2", "e2b": "^2.35.0", + "isomorphic-git": "1.40.0", "p-limit": "^7.3.0", "pg": "^8.22.0", "playwright-core": "^1.61.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b7fb6560..43d9878b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -182,6 +182,9 @@ importers: e2b: specifier: ^2.35.0 version: 2.35.0 + isomorphic-git: + specifier: 1.40.0 + version: 1.40.0 p-limit: specifier: ^7.3.0 version: 7.3.1 @@ -2522,6 +2525,9 @@ packages: assertion-error@1.1.0: resolution: {integrity: sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==} + async-lock@1.4.1: + resolution: {integrity: sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==} + async-retry@1.3.3: resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} @@ -2787,6 +2793,9 @@ packages: cjs-module-lexer@2.2.0: resolution: {integrity: sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==} + clean-git-ref@2.0.1: + resolution: {integrity: sha512-bLSptAy2P0s6hU4PzuIMKmMJJSE6gLXGH1cntDu7bWJUksvuM+7ReOK61mozULErYvP6a15rnYl0zFDef+pyPw==} + cli-boxes@4.0.1: resolution: {integrity: sha512-5IOn+jcCEHEraYolBPs/sT4BxYCe2nHg374OPiItB1O96KZFseS2gthU4twyYzeDcFew4DaUM/xwc5BQf08JJw==} engines: {node: '>=18.20 <19 || >=20.10'} @@ -2999,6 +3008,9 @@ packages: resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + diff3@0.0.3: + resolution: {integrity: sha512-iSq8ngPOt0K53A6eVr4d5Kn6GNrM2nQZtC740pzIriHtn4pOQ2lyzEXQMBeVcWERN0ye7fhBsk9PbLLQOnUx/g==} + diff@8.0.4: resolution: {integrity: sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==} engines: {node: '>=0.3.1'} @@ -3754,6 +3766,9 @@ packages: isarray@1.0.0: resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + isarray@2.0.5: + resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==} + isbinaryfile@5.0.7: resolution: {integrity: sha512-gnWD14Jh3FzS3CPhF0AxNOJ8CxqeblPTADzI38r0wt8ZyQl5edpy75myt08EG2oKvpyiqSqsx+Wkz9vtkbTqYQ==} engines: {node: '>= 18.0.0'} @@ -3761,6 +3776,11 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + isomorphic-git@1.40.0: + resolution: {integrity: sha512-/CbnxwZqIm17y3c/z0INbkgEKSvFerXtO/NGgaRxZ8nvL3eoMtbjuAS7f4Pj7lZzj8HaultvDD1ClJTBVDl89g==} + engines: {node: '>=14.17'} + hasBin: true + isomorphic-ws@5.0.0: resolution: {integrity: sha512-muId7Zzn9ywDsyXgTIafTry2sV3nySZeUDe6YedVd1Hvuuep5AsIlqK+XefWpYTyJG5e503F2xIuT2lcU6rCSw==} peerDependencies: @@ -4060,6 +4080,9 @@ packages: minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + minimisted@2.0.1: + resolution: {integrity: sha512-1oPjfuLQa2caorJUM8HV8lGgWCc0qqAO1MNv/k05G4qslmsndV/5WdNZrqCiyqiz3wohia2Ij2B7w2Dr7/IyrA==} + minipass@7.1.3: resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==} engines: {node: '>=16 || 14 >=14.17'} @@ -4289,6 +4312,9 @@ packages: package-manager-detector@0.2.11: resolution: {integrity: sha512-BEnLolu+yuz22S56CU1SUKq3XC3PkwD5wv4ikR4MfGvnRVcmzXR9DwSlW2fEamyTPyXHomBJRzgapeuBvRNzJQ==} + pako@1.0.11: + resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==} + papaparse@5.5.4: resolution: {integrity: sha512-SwzWD9gl/ElwYLCI0nUja1mFJzjq2D8ziShfNBa7zCHzkOozeOGDwHWQ+tvCzEZcewecWZ5U7kUopDnG+DFYEQ==} @@ -4758,6 +4784,11 @@ packages: setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + sha.js@2.4.12: + resolution: {integrity: sha512-8LzC5+bvI45BjpfXU8V5fdU2mfeKiQe1D1gIMn7XUlF3OTUrpdJpPPH4EMAnF0DsHHdSZqCdSss5qCmJKuiO3w==} + engines: {node: '>= 0.10'} + hasBin: true + shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} @@ -5073,6 +5104,10 @@ packages: resolution: {integrity: sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A==} engines: {node: '>=14.0.0'} + to-buffer@1.2.2: + resolution: {integrity: sha512-db0E3UJjcFhpDhAF4tLo03oli3pwl3dbnzXOUIlRKrp+ldk/VUxzpWYZENsw2SZiuBjHAk7DfB0VU7NKdpb6sw==} + engines: {node: '>= 0.4'} + to-regex-range@5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} @@ -5159,6 +5194,10 @@ packages: resolution: {integrity: sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==} engines: {node: '>= 18'} + typed-array-buffer@1.0.3: + resolution: {integrity: sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==} + engines: {node: '>= 0.4'} + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -8287,6 +8326,8 @@ snapshots: assertion-error@1.1.0: {} + async-lock@1.4.1: {} + async-retry@1.3.3: dependencies: retry: 0.13.1 @@ -8577,6 +8618,8 @@ snapshots: cjs-module-lexer@2.2.0: {} + clean-git-ref@2.0.1: {} + cli-boxes@4.0.1: {} cli-cursor@4.0.0: @@ -8698,7 +8741,6 @@ snapshots: decompress-response@6.0.0: dependencies: mimic-response: 3.1.0 - optional: true deep-eql@4.1.4: dependencies: @@ -8741,6 +8783,8 @@ snapshots: diff-sequences@29.6.3: {} + diff3@0.0.3: {} + diff@8.0.4: {} dir-glob@3.0.1: @@ -9665,10 +9709,26 @@ snapshots: isarray@1.0.0: {} + isarray@2.0.5: {} + isbinaryfile@5.0.7: {} isexe@2.0.0: {} + isomorphic-git@1.40.0: + dependencies: + async-lock: 1.4.1 + clean-git-ref: 2.0.1 + crc-32: 1.2.2 + diff3: 0.0.3 + ignore: 5.3.2 + minimisted: 2.0.1 + pako: 1.0.11 + pify: 4.0.1 + readable-stream: 4.7.0 + sha.js: 2.4.12 + simple-get: 4.0.1 + isomorphic-ws@5.0.0(ws@8.21.1(bufferutil@4.1.0)(utf-8-validate@6.0.6)): dependencies: ws: 8.21.1(bufferutil@4.1.0)(utf-8-validate@6.0.6) @@ -9926,8 +9986,7 @@ snapshots: mimic-function@5.0.1: {} - mimic-response@3.1.0: - optional: true + mimic-response@3.1.0: {} minimatch@10.2.5: dependencies: @@ -9949,8 +10008,11 @@ snapshots: dependencies: brace-expansion: 2.1.2 - minimist@1.2.8: - optional: true + minimist@1.2.8: {} + + minimisted@2.0.1: + dependencies: + minimist: 1.2.8 minipass@7.1.3: {} @@ -10181,6 +10243,8 @@ snapshots: dependencies: quansync: 0.2.11 + pako@1.0.11: {} + papaparse@5.5.4: {} parent-module@1.0.1: @@ -10707,6 +10771,12 @@ snapshots: setprototypeof@1.2.0: {} + sha.js@2.4.12: + dependencies: + inherits: 2.0.4 + safe-buffer: 5.2.1 + to-buffer: 1.2.2 + shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 @@ -10754,15 +10824,13 @@ snapshots: signal-exit@4.1.0: {} - simple-concat@1.0.1: - optional: true + simple-concat@1.0.1: {} simple-get@4.0.1: dependencies: decompress-response: 6.0.0 once: 1.4.0 simple-concat: 1.0.1 - optional: true slash@3.0.0: {} @@ -11069,6 +11137,12 @@ snapshots: tinyspy@2.2.1: {} + to-buffer@1.2.2: + dependencies: + isarray: 2.0.5 + safe-buffer: 5.2.1 + typed-array-buffer: 1.0.3 + to-regex-range@5.0.1: dependencies: is-number: 7.0.0 @@ -11158,6 +11232,12 @@ snapshots: media-typer: 1.1.0 mime-types: 3.0.2 + typed-array-buffer@1.0.3: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + is-typed-array: 1.1.15 + typescript@5.9.3: {} typescript@6.0.3: {} diff --git a/results/git/.gitkeep b/results/git/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tsconfig.json b/tsconfig.json index 6f11950b..c9a9904e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,6 +18,7 @@ "benchmarks/browser/**/*.ts", "benchmarks/storage/**/*.ts", "benchmarks/ai-gateway/**/*.ts", + "benchmarks/git/**/*.ts", "benchmarks/scale/**/*.ts" ], "exclude": ["node_modules", "benchmarks/dist", "packages", "results", ".git"] From 35ce964849fec29438b952502c03a01587b8a9d6 Mon Sep 17 00:00:00 2001 From: garrison Date: Wed, 5 Aug 2026 15:05:09 +0000 Subject: [PATCH 02/15] feat(git): add push/pull phases and tensorlake participant Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- benchmarks/git/git.bench.ts | 121 +++++++++++++++++++++++++++++++----- benchmarks/git/providers.ts | 19 +++++- benchmarks/git/types.ts | 22 +++++-- 3 files changed, 137 insertions(+), 25 deletions(-) diff --git a/benchmarks/git/git.bench.ts b/benchmarks/git/git.bench.ts index d269309f..3275175b 100644 --- a/benchmarks/git/git.bench.ts +++ b/benchmarks/git/git.bench.ts @@ -1,10 +1,14 @@ /** - * Git clone benchmark. Measures shallow clone latency over HTTPS for git - * hosting providers using isomorphic-git as the harness. Declarative — - * exports `config` + `task`; `bench run` owns the entrypoint. + * Git workflow benchmark. Measures shallow clone, commit+push, and pull over + * HTTPS for git hosting providers using isomorphic-git as the harness. + * Declarative — exports `config` + `task`; `bench run` owns the entrypoint. + * + * The push/pull workflow runs only when the participant's token env var is set. + * For the read-only public fixtures (GitHub/GitLab/Bitbucket defaults), only the + * `clone` step is exercised unless `*_GIT_REPO_URL` and `*_TOKEN` are provided. * * bench run benchmarks/git/git.bench.ts - * bench run benchmarks/git/git.bench.ts --provider github,gitlab --iterations 5 + * bench run benchmarks/git/git.bench.ts --provider tensorlake --iterations 5 */ import '../src/env.js'; import fs from 'node:fs'; @@ -20,6 +24,15 @@ import { providers } from './providers.js'; import type { GitProviderConfig } from './types.js'; const CLONE_TIMEOUT_MS = 60_000; +const COMMITTER = { name: 'ComputeSDK Benchmark', email: 'bench@example.com' }; + +function resolveRepoUrl(config: GitProviderConfig): string { + if (config.repoUrlEnvVar) { + const override = process.env[config.repoUrlEnvVar]; + if (override) return override; + } + return config.url; +} function buildAuth(config: GitProviderConfig): AuthCallback | undefined { if (!config.tokenEnvVar) return undefined; @@ -30,8 +43,8 @@ function buildAuth(config: GitProviderConfig): AuthCallback | undefined { } export const config = defineBenchmarkConfig({ - benchmarkSlug: 'git-clone-local', - benchmarkName: 'Git clone (local)', + benchmarkSlug: 'git-workflow-local', + benchmarkName: 'Git workflow (local)', benchmarkKind: 'git', iterations: 3, concurrency: 1, @@ -39,37 +52,111 @@ export const config = defineBenchmarkConfig({ }); export const task = defineTask(async (ctx) => { - const { participant, step, measure } = ctx; + const { participant, step, measure, taskIndex } = ctx; const timeout = participant.timeout ?? CLONE_TIMEOUT_MS; + const repoUrl = resolveRepoUrl(participant); + const onAuth = buildAuth(participant); + const branch = `${participant.name}-${taskIndex}-${Date.now()}`; + const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'bench-git-')); - const cloneDir = path.join(tempDir, 'repo'); + const workDir = path.join(tempDir, 'repo'); + + let cloneMs = 0; + let pushMs = 0; + let pullMs = 0; try { - const start = performance.now(); + const cloneStart = performance.now(); await step('clone', () => withTimeout( git.clone({ fs: fs as unknown as import('isomorphic-git').FsClient, http, - dir: cloneDir, - url: participant.url, + dir: workDir, + url: repoUrl, singleBranch: true, depth: 1, - onAuth: buildAuth(participant), + onAuth, }), timeout, 'Git clone timed out', ), ); - const cloneMs = performance.now() - start; + cloneMs = performance.now() - cloneStart; + + // Without auth we can only benchmark clone; skip the write path. + if (!onAuth) { + measure({ cloneMs, repoUrl, branch, pushSkipped: true, pullSkipped: true, commitSha: '' }); + return { data: { cloneMs, pushMs: 0, pullMs: 0, repoUrl, branch, commitSha: '' } }; + } + + const defaultBranch = + (await git.currentBranch({ fs, dir: workDir, fullname: false })) ?? + participant.defaultBranch ?? + 'main'; + + await git.branch({ fs, dir: workDir, ref: branch, checkout: true }); + await fs.promises.writeFile( + path.join(workDir, 'bench.txt'), + `benchmark ${branch}\n`, + ); + await git.add({ fs, dir: workDir, filepath: 'bench.txt' }); + const commitSha = await git.commit({ + fs, + dir: workDir, + message: `bench: ${branch}`, + author: COMMITTER, + committer: COMMITTER, + }); + + const pushStart = performance.now(); + await step('push', () => + withTimeout( + git.push({ + fs: fs as unknown as import('isomorphic-git').FsClient, + http, + dir: workDir, + remote: 'origin', + ref: branch, + remoteRef: branch, + onAuth, + }), + timeout, + 'Git push timed out', + ), + ); + pushMs = performance.now() - pushStart; + + await git.checkout({ fs, dir: workDir, ref: defaultBranch }); + + const pullStart = performance.now(); + await step('pull', () => + withTimeout( + git.pull({ + fs: fs as unknown as import('isomorphic-git').FsClient, + http, + dir: workDir, + ref: defaultBranch, + remoteRef: branch, + singleBranch: true, + fastForwardOnly: true, + onAuth, + author: COMMITTER, + committer: COMMITTER, + }), + timeout, + 'Git pull timed out', + ), + ); + pullMs = performance.now() - pullStart; - measure({ cloneMs, repoUrl: participant.url }); - return { data: { cloneMs, repoUrl: participant.url } }; + measure({ cloneMs, pushMs, pullMs, repoUrl, branch, commitSha }); + return { data: { cloneMs, pushMs, pullMs, repoUrl, branch, commitSha } }; } catch (err) { throw new TaskError(formatError(err), { - code: 'GIT_CLONE_ERROR', - data: { repoUrl: participant.url, cloneMs: 0 }, + code: 'GIT_WORKFLOW_ERROR', + data: { repoUrl, branch, cloneMs, pushMs, pullMs }, }); } finally { await fs.promises.rm(tempDir, { recursive: true, force: true }).catch(() => {}); diff --git a/benchmarks/git/providers.ts b/benchmarks/git/providers.ts index d4142381..7f76b73a 100644 --- a/benchmarks/git/providers.ts +++ b/benchmarks/git/providers.ts @@ -3,15 +3,18 @@ import type { GitProviderConfig } from './types.js'; /** * Git hosting provider benchmark configurations. * - * Each participant points at a small public repo and uses isomorphic-git over - * HTTPS. Tokens are optional: when `tokenEnvVar` is set and present, it is - * passed via `onAuth`; otherwise the clone is anonymous. + * Each participant points at an HTTPS repo. The `url` field provides a + * read-only default (where a public fixture exists); set the matching + * `*_GIT_REPO_URL` env var to a writable repo and the matching token env var + * to enable the push/pull workflow. Tensorlake is gated by required env vars + * because it has no public fixture. */ export const providers: GitProviderConfig[] = [ { name: 'github', requiredEnvVars: [], url: 'https://github.com/octocat/Spoon-Knife.git', + repoUrlEnvVar: 'GITHUB_GIT_REPO_URL', tokenEnvVar: 'GITHUB_TOKEN', tokenUsername: 'token', }, @@ -19,6 +22,7 @@ export const providers: GitProviderConfig[] = [ name: 'gitlab', requiredEnvVars: [], url: 'https://gitlab.com/gitlab-org/gitlab-test.git', + repoUrlEnvVar: 'GITLAB_GIT_REPO_URL', tokenEnvVar: 'GITLAB_TOKEN', tokenUsername: 'oauth2', }, @@ -26,9 +30,18 @@ export const providers: GitProviderConfig[] = [ name: 'bitbucket', requiredEnvVars: [], url: 'https://bitbucket.org/atlassian/hello-world.git', + repoUrlEnvVar: 'BITBUCKET_GIT_REPO_URL', tokenEnvVar: 'BITBUCKET_TOKEN', tokenUsername: 'x-token-auth', }, + { + name: 'tensorlake', + requiredEnvVars: ['TENSORLAKE_GIT_REPO_URL', 'TENSORLAKE_API_KEY'], + url: process.env.TENSORLAKE_GIT_REPO_URL ?? '', + repoUrlEnvVar: 'TENSORLAKE_GIT_REPO_URL', + tokenEnvVar: 'TENSORLAKE_API_KEY', + tokenUsername: 't', + }, // // add git providers above ]; diff --git a/benchmarks/git/types.ts b/benchmarks/git/types.ts index b686e3eb..3bdae997 100644 --- a/benchmarks/git/types.ts +++ b/benchmarks/git/types.ts @@ -1,27 +1,39 @@ import type { BaseParticipant } from '@benchsdk/client'; export interface GitProviderConfig extends BaseParticipant { - /** HTTPS URL of a public repository to clone. */ + /** HTTPS URL of the repo to clone (used when no env override is set). */ url: string; - /** Optional environment variable holding an HTTPS auth token. */ + /** Optional env var that overrides `url` for the read/write repo. */ + repoUrlEnvVar?: string; + /** Optional env var holding an HTTPS auth token for push/pull. */ tokenEnvVar?: string; - /** Username passed to isomorphic-git's onAuth callback when a token is set. */ + /** Username passed to isomorphic-git's `onAuth` callback when a token is set. */ tokenUsername?: string; - /** Per-provider timeout for the clone step in ms (default: 60000). */ + /** Default branch to pull back into after pushing a test branch. */ + defaultBranch?: string; + /** Per-operation timeout in ms (default: 60000). */ timeout?: number; } export interface GitTimingResult { /** Time to shallow clone the repo in ms. */ cloneMs: number; - /** Repository URL cloned. */ + /** Time to commit and push the test branch in ms. */ + pushMs: number; + /** Time to pull the test branch into the default branch in ms. */ + pullMs: number; + /** Repository URL used. */ repoUrl: string; + /** Test branch that was pushed/pulled. */ + branch: string; /** Error message if this iteration failed. */ error?: string; } export interface GitStats { cloneMs: { median: number; p95: number; p99: number }; + pushMs: { median: number; p95: number; p99: number }; + pullMs: { median: number; p95: number; p99: number }; } export interface GitBenchmarkResult { From 950f5d8757fd1aef9ee3a01ba44179ce2f1ad8fc Mon Sep 17 00:00:00 2001 From: garrison Date: Wed, 5 Aug 2026 15:05:44 +0000 Subject: [PATCH 03/15] chore(git): add tensorlake benchmark script Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 37f03eef..77732abd 100644 --- a/package.json +++ b/package.json @@ -93,7 +93,8 @@ "bench:git": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts", "bench:git:github": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts --provider github", "bench:git:gitlab": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts --provider gitlab", - "bench:git:bitbucket": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts --provider bitbucket" + "bench:git:bitbucket": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts --provider bitbucket", + "bench:git:tensorlake": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts --provider tensorlake" }, "dependencies": { "@aws-sdk/client-s3": "^3.1076.0", From 341a651b44da8f6f4a762b5999cec2dffc3b044b Mon Sep 17 00:00:00 2001 From: garrison Date: Wed, 5 Aug 2026 15:54:06 +0000 Subject: [PATCH 04/15] refactor(git): use git CLI instead of isomorphic-git Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- benchmarks/git/git.bench.ts | 108 +++++++++++++++--------------------- package.json | 1 - pnpm-lock.yaml | 96 +++----------------------------- 3 files changed, 53 insertions(+), 152 deletions(-) diff --git a/benchmarks/git/git.bench.ts b/benchmarks/git/git.bench.ts index 3275175b..ef5989e5 100644 --- a/benchmarks/git/git.bench.ts +++ b/benchmarks/git/git.bench.ts @@ -1,6 +1,6 @@ /** * Git workflow benchmark. Measures shallow clone, commit+push, and pull over - * HTTPS for git hosting providers using isomorphic-git as the harness. + * HTTPS for git hosting providers by shelling out to `git`. * Declarative — exports `config` + `task`; `bench run` owns the entrypoint. * * The push/pull workflow runs only when the participant's token env var is set. @@ -11,20 +11,21 @@ * bench run benchmarks/git/git.bench.ts --provider tensorlake --iterations 5 */ import '../src/env.js'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import git from 'isomorphic-git'; -import http from 'isomorphic-git/http/node'; import { defineBenchmarkConfig, defineTask, TaskError } from '@benchsdk/runner'; -import type { AuthCallback } from 'isomorphic-git'; import { withTimeout } from '../src/util/timeout.js'; import { formatError } from '../src/util/error.js'; import { providers } from './providers.js'; import type { GitProviderConfig } from './types.js'; +const execFileAsync = promisify(execFile); const CLONE_TIMEOUT_MS = 60_000; -const COMMITTER = { name: 'ComputeSDK Benchmark', email: 'bench@example.com' }; +const COMMITTER_NAME = 'ComputeSDK Benchmark'; +const COMMITTER_EMAIL = 'bench@example.com'; function resolveRepoUrl(config: GitProviderConfig): string { if (config.repoUrlEnvVar) { @@ -34,12 +35,23 @@ function resolveRepoUrl(config: GitProviderConfig): string { return config.url; } -function buildAuth(config: GitProviderConfig): AuthCallback | undefined { - if (!config.tokenEnvVar) return undefined; - const token = process.env[config.tokenEnvVar]; - if (!token) return undefined; - const username = config.tokenUsername ?? 'token'; - return () => ({ username, password: token }); +function authRepoUrl(repoUrl: string, username: string, token: string): string { + const url = new URL(repoUrl); + url.username = encodeURIComponent(username); + url.password = encodeURIComponent(token); + return url.toString(); +} + +function gitEnv(): NodeJS.ProcessEnv { + return { ...process.env, GIT_TERMINAL_PROMPT: '0' }; +} + +function runGit( + args: string[], + cwd: string, + timeout: number, +): Promise<{ stdout: string; stderr: string }> { + return execFileAsync('git', args, { cwd, env: gitEnv(), timeout }); } export const config = defineBenchmarkConfig({ @@ -56,9 +68,11 @@ export const task = defineTask(async (ctx) => { const timeout = participant.timeout ?? CLONE_TIMEOUT_MS; const repoUrl = resolveRepoUrl(participant); - const onAuth = buildAuth(participant); - const branch = `${participant.name}-${taskIndex}-${Date.now()}`; + const token = participant.tokenEnvVar ? process.env[participant.tokenEnvVar] : undefined; + const username = participant.tokenUsername ?? 'token'; + const cloneUrl = token ? authRepoUrl(repoUrl, username, token) : repoUrl; + const branch = `${participant.name}-${taskIndex}-${Date.now()}`; const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'bench-git-')); const workDir = path.join(tempDir, 'repo'); @@ -70,81 +84,49 @@ export const task = defineTask(async (ctx) => { const cloneStart = performance.now(); await step('clone', () => withTimeout( - git.clone({ - fs: fs as unknown as import('isomorphic-git').FsClient, - http, - dir: workDir, - url: repoUrl, - singleBranch: true, - depth: 1, - onAuth, - }), + runGit(['clone', '--depth', '1', '--single-branch', cloneUrl, workDir], process.cwd(), timeout), timeout, 'Git clone timed out', ), ); cloneMs = performance.now() - cloneStart; - // Without auth we can only benchmark clone; skip the write path. - if (!onAuth) { + if (!token) { measure({ cloneMs, repoUrl, branch, pushSkipped: true, pullSkipped: true, commitSha: '' }); return { data: { cloneMs, pushMs: 0, pullMs: 0, repoUrl, branch, commitSha: '' } }; } - const defaultBranch = - (await git.currentBranch({ fs, dir: workDir, fullname: false })) ?? - participant.defaultBranch ?? - 'main'; - - await git.branch({ fs, dir: workDir, ref: branch, checkout: true }); - await fs.promises.writeFile( - path.join(workDir, 'bench.txt'), - `benchmark ${branch}\n`, + const defaultBranch = await runGit(['branch', '--show-current'], workDir, timeout) + .then((r) => r.stdout.trim()) + .catch(() => participant.defaultBranch ?? 'main'); + + // Prepare, commit, and push the test branch. + await runGit(['checkout', '-b', branch], workDir, timeout); + await fs.promises.writeFile(path.join(workDir, 'bench.txt'), `benchmark ${branch}\n`); + await runGit(['add', 'bench.txt'], workDir, timeout); + const commitResult = await runGit( + ['-c', `user.name=${COMMITTER_NAME}`, '-c', `user.email=${COMMITTER_EMAIL}`, 'commit', '-m', `bench: ${branch}`], + workDir, + timeout, ); - await git.add({ fs, dir: workDir, filepath: 'bench.txt' }); - const commitSha = await git.commit({ - fs, - dir: workDir, - message: `bench: ${branch}`, - author: COMMITTER, - committer: COMMITTER, - }); + const commitSha = commitResult.stdout.match(/\[.+?\s+([a-f0-9]+)\]/)?.[1] ?? ''; const pushStart = performance.now(); await step('push', () => withTimeout( - git.push({ - fs: fs as unknown as import('isomorphic-git').FsClient, - http, - dir: workDir, - remote: 'origin', - ref: branch, - remoteRef: branch, - onAuth, - }), + runGit(['push', '-u', 'origin', branch], workDir, timeout), timeout, 'Git push timed out', ), ); pushMs = performance.now() - pushStart; - await git.checkout({ fs, dir: workDir, ref: defaultBranch }); + await runGit(['checkout', defaultBranch], workDir, timeout); const pullStart = performance.now(); await step('pull', () => withTimeout( - git.pull({ - fs: fs as unknown as import('isomorphic-git').FsClient, - http, - dir: workDir, - ref: defaultBranch, - remoteRef: branch, - singleBranch: true, - fastForwardOnly: true, - onAuth, - author: COMMITTER, - committer: COMMITTER, - }), + runGit(['pull', '--ff-only', 'origin', branch], workDir, timeout), timeout, 'Git pull timed out', ), diff --git a/package.json b/package.json index 77732abd..7837d7f3 100644 --- a/package.json +++ b/package.json @@ -155,7 +155,6 @@ "computesdk": "^4.1.4", "dotenv": "^17.4.2", "e2b": "^2.35.0", - "isomorphic-git": "1.40.0", "p-limit": "^7.3.0", "pg": "^8.22.0", "playwright-core": "^1.61.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 43d9878b..b7fb6560 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -182,9 +182,6 @@ importers: e2b: specifier: ^2.35.0 version: 2.35.0 - isomorphic-git: - specifier: 1.40.0 - version: 1.40.0 p-limit: specifier: ^7.3.0 version: 7.3.1 @@ -2525,9 +2522,6 @@ packages: assertion-error@1.1.0: resolution: {integrity: sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==} - async-lock@1.4.1: - resolution: {integrity: sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==} - async-retry@1.3.3: resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} @@ -2793,9 +2787,6 @@ packages: cjs-module-lexer@2.2.0: resolution: {integrity: sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==} - clean-git-ref@2.0.1: - resolution: {integrity: sha512-bLSptAy2P0s6hU4PzuIMKmMJJSE6gLXGH1cntDu7bWJUksvuM+7ReOK61mozULErYvP6a15rnYl0zFDef+pyPw==} - cli-boxes@4.0.1: resolution: {integrity: sha512-5IOn+jcCEHEraYolBPs/sT4BxYCe2nHg374OPiItB1O96KZFseS2gthU4twyYzeDcFew4DaUM/xwc5BQf08JJw==} engines: {node: '>=18.20 <19 || >=20.10'} @@ -3008,9 +2999,6 @@ packages: resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} - diff3@0.0.3: - resolution: {integrity: sha512-iSq8ngPOt0K53A6eVr4d5Kn6GNrM2nQZtC740pzIriHtn4pOQ2lyzEXQMBeVcWERN0ye7fhBsk9PbLLQOnUx/g==} - diff@8.0.4: resolution: {integrity: sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==} engines: {node: '>=0.3.1'} @@ -3766,9 +3754,6 @@ packages: isarray@1.0.0: resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} - isarray@2.0.5: - resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==} - isbinaryfile@5.0.7: resolution: {integrity: sha512-gnWD14Jh3FzS3CPhF0AxNOJ8CxqeblPTADzI38r0wt8ZyQl5edpy75myt08EG2oKvpyiqSqsx+Wkz9vtkbTqYQ==} engines: {node: '>= 18.0.0'} @@ -3776,11 +3761,6 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} - isomorphic-git@1.40.0: - resolution: {integrity: sha512-/CbnxwZqIm17y3c/z0INbkgEKSvFerXtO/NGgaRxZ8nvL3eoMtbjuAS7f4Pj7lZzj8HaultvDD1ClJTBVDl89g==} - engines: {node: '>=14.17'} - hasBin: true - isomorphic-ws@5.0.0: resolution: {integrity: sha512-muId7Zzn9ywDsyXgTIafTry2sV3nySZeUDe6YedVd1Hvuuep5AsIlqK+XefWpYTyJG5e503F2xIuT2lcU6rCSw==} peerDependencies: @@ -4080,9 +4060,6 @@ packages: minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} - minimisted@2.0.1: - resolution: {integrity: sha512-1oPjfuLQa2caorJUM8HV8lGgWCc0qqAO1MNv/k05G4qslmsndV/5WdNZrqCiyqiz3wohia2Ij2B7w2Dr7/IyrA==} - minipass@7.1.3: resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==} engines: {node: '>=16 || 14 >=14.17'} @@ -4312,9 +4289,6 @@ packages: package-manager-detector@0.2.11: resolution: {integrity: sha512-BEnLolu+yuz22S56CU1SUKq3XC3PkwD5wv4ikR4MfGvnRVcmzXR9DwSlW2fEamyTPyXHomBJRzgapeuBvRNzJQ==} - pako@1.0.11: - resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==} - papaparse@5.5.4: resolution: {integrity: sha512-SwzWD9gl/ElwYLCI0nUja1mFJzjq2D8ziShfNBa7zCHzkOozeOGDwHWQ+tvCzEZcewecWZ5U7kUopDnG+DFYEQ==} @@ -4784,11 +4758,6 @@ packages: setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} - sha.js@2.4.12: - resolution: {integrity: sha512-8LzC5+bvI45BjpfXU8V5fdU2mfeKiQe1D1gIMn7XUlF3OTUrpdJpPPH4EMAnF0DsHHdSZqCdSss5qCmJKuiO3w==} - engines: {node: '>= 0.10'} - hasBin: true - shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} @@ -5104,10 +5073,6 @@ packages: resolution: {integrity: sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A==} engines: {node: '>=14.0.0'} - to-buffer@1.2.2: - resolution: {integrity: sha512-db0E3UJjcFhpDhAF4tLo03oli3pwl3dbnzXOUIlRKrp+ldk/VUxzpWYZENsw2SZiuBjHAk7DfB0VU7NKdpb6sw==} - engines: {node: '>= 0.4'} - to-regex-range@5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} @@ -5194,10 +5159,6 @@ packages: resolution: {integrity: sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==} engines: {node: '>= 18'} - typed-array-buffer@1.0.3: - resolution: {integrity: sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==} - engines: {node: '>= 0.4'} - typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -8326,8 +8287,6 @@ snapshots: assertion-error@1.1.0: {} - async-lock@1.4.1: {} - async-retry@1.3.3: dependencies: retry: 0.13.1 @@ -8618,8 +8577,6 @@ snapshots: cjs-module-lexer@2.2.0: {} - clean-git-ref@2.0.1: {} - cli-boxes@4.0.1: {} cli-cursor@4.0.0: @@ -8741,6 +8698,7 @@ snapshots: decompress-response@6.0.0: dependencies: mimic-response: 3.1.0 + optional: true deep-eql@4.1.4: dependencies: @@ -8783,8 +8741,6 @@ snapshots: diff-sequences@29.6.3: {} - diff3@0.0.3: {} - diff@8.0.4: {} dir-glob@3.0.1: @@ -9709,26 +9665,10 @@ snapshots: isarray@1.0.0: {} - isarray@2.0.5: {} - isbinaryfile@5.0.7: {} isexe@2.0.0: {} - isomorphic-git@1.40.0: - dependencies: - async-lock: 1.4.1 - clean-git-ref: 2.0.1 - crc-32: 1.2.2 - diff3: 0.0.3 - ignore: 5.3.2 - minimisted: 2.0.1 - pako: 1.0.11 - pify: 4.0.1 - readable-stream: 4.7.0 - sha.js: 2.4.12 - simple-get: 4.0.1 - isomorphic-ws@5.0.0(ws@8.21.1(bufferutil@4.1.0)(utf-8-validate@6.0.6)): dependencies: ws: 8.21.1(bufferutil@4.1.0)(utf-8-validate@6.0.6) @@ -9986,7 +9926,8 @@ snapshots: mimic-function@5.0.1: {} - mimic-response@3.1.0: {} + mimic-response@3.1.0: + optional: true minimatch@10.2.5: dependencies: @@ -10008,11 +9949,8 @@ snapshots: dependencies: brace-expansion: 2.1.2 - minimist@1.2.8: {} - - minimisted@2.0.1: - dependencies: - minimist: 1.2.8 + minimist@1.2.8: + optional: true minipass@7.1.3: {} @@ -10243,8 +10181,6 @@ snapshots: dependencies: quansync: 0.2.11 - pako@1.0.11: {} - papaparse@5.5.4: {} parent-module@1.0.1: @@ -10771,12 +10707,6 @@ snapshots: setprototypeof@1.2.0: {} - sha.js@2.4.12: - dependencies: - inherits: 2.0.4 - safe-buffer: 5.2.1 - to-buffer: 1.2.2 - shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 @@ -10824,13 +10754,15 @@ snapshots: signal-exit@4.1.0: {} - simple-concat@1.0.1: {} + simple-concat@1.0.1: + optional: true simple-get@4.0.1: dependencies: decompress-response: 6.0.0 once: 1.4.0 simple-concat: 1.0.1 + optional: true slash@3.0.0: {} @@ -11137,12 +11069,6 @@ snapshots: tinyspy@2.2.1: {} - to-buffer@1.2.2: - dependencies: - isarray: 2.0.5 - safe-buffer: 5.2.1 - typed-array-buffer: 1.0.3 - to-regex-range@5.0.1: dependencies: is-number: 7.0.0 @@ -11232,12 +11158,6 @@ snapshots: media-typer: 1.1.0 mime-types: 3.0.2 - typed-array-buffer@1.0.3: - dependencies: - call-bound: 1.0.4 - es-errors: 1.3.0 - is-typed-array: 1.1.15 - typescript@5.9.3: {} typescript@6.0.3: {} From e697c9e7ae35019937fc3a1b24e67b4c11c1a220 Mon Sep 17 00:00:00 2001 From: garrison Date: Wed, 5 Aug 2026 18:58:43 +0000 Subject: [PATCH 05/15] fix(git): avoid leaking token, gate push/pull on writable repo, and clean up remote branches Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- benchmarks/git/git.bench.ts | 102 ++++++++++++++++++++++++------------ benchmarks/git/types.ts | 4 +- 2 files changed, 71 insertions(+), 35 deletions(-) diff --git a/benchmarks/git/git.bench.ts b/benchmarks/git/git.bench.ts index ef5989e5..0f46ade2 100644 --- a/benchmarks/git/git.bench.ts +++ b/benchmarks/git/git.bench.ts @@ -3,9 +3,9 @@ * HTTPS for git hosting providers by shelling out to `git`. * Declarative — exports `config` + `task`; `bench run` owns the entrypoint. * - * The push/pull workflow runs only when the participant's token env var is set. - * For the read-only public fixtures (GitHub/GitLab/Bitbucket defaults), only the - * `clone` step is exercised unless `*_GIT_REPO_URL` and `*_TOKEN` are provided. + * The push/pull workflow runs only when BOTH the participant's token env var + * AND the writable repo URL override are set. For the read-only public fixtures + * (GitHub/GitLab/Bitbucket defaults), only the `clone` step is exercised. * * bench run benchmarks/git/git.bench.ts * bench run benchmarks/git/git.bench.ts --provider tensorlake --iterations 5 @@ -27,31 +27,47 @@ const CLONE_TIMEOUT_MS = 60_000; const COMMITTER_NAME = 'ComputeSDK Benchmark'; const COMMITTER_EMAIL = 'bench@example.com'; -function resolveRepoUrl(config: GitProviderConfig): string { - if (config.repoUrlEnvVar) { - const override = process.env[config.repoUrlEnvVar]; - if (override) return override; - } - return config.url; +function resolveRepoConfig(config: GitProviderConfig): { repoUrl: string; writable: boolean } { + const override = config.repoUrlEnvVar ? process.env[config.repoUrlEnvVar] : undefined; + const repoUrl = override ? sanitizeRepoUrl(override) : sanitizeRepoUrl(config.url); + return { repoUrl, writable: !!override }; } -function authRepoUrl(repoUrl: string, username: string, token: string): string { - const url = new URL(repoUrl); - url.username = encodeURIComponent(username); - url.password = encodeURIComponent(token); - return url.toString(); +function sanitizeRepoUrl(repoUrl: string): string { + try { + const url = new URL(repoUrl); + url.username = ''; + url.password = ''; + return url.toString(); + } catch { + return repoUrl; + } } -function gitEnv(): NodeJS.ProcessEnv { - return { ...process.env, GIT_TERMINAL_PROMPT: '0' }; +function buildGitEnv( + useAuth: boolean, + username: string, + token: string, + askpassPath?: string, +): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { ...process.env, GIT_TERMINAL_PROMPT: '0' }; + if (useAuth && askpassPath) { + env.GIT_ASKPASS = askpassPath; + env.GIT_BENCH_USER = username; + env.GIT_BENCH_PASS = token; + } + return env; } -function runGit( - args: string[], - cwd: string, - timeout: number, -): Promise<{ stdout: string; stderr: string }> { - return execFileAsync('git', args, { cwd, env: gitEnv(), timeout }); +async function writeAskpassScript(askpassPath: string): Promise { + const script = `#!/bin/sh +case "$1" in + *Password*) printf '%s\\n' "$GIT_BENCH_PASS" ;; + *Username*) printf '%s\\n' "$GIT_BENCH_USER" ;; + *) printf '%s\\n' "$GIT_BENCH_USER" ;; +esac +`; + await fs.promises.writeFile(askpassPath, script, { mode: 0o755 }); } export const config = defineBenchmarkConfig({ @@ -67,66 +83,81 @@ export const task = defineTask(async (ctx) => { const { participant, step, measure, taskIndex } = ctx; const timeout = participant.timeout ?? CLONE_TIMEOUT_MS; - const repoUrl = resolveRepoUrl(participant); + const { repoUrl, writable } = resolveRepoConfig(participant); const token = participant.tokenEnvVar ? process.env[participant.tokenEnvVar] : undefined; const username = participant.tokenUsername ?? 'token'; - const cloneUrl = token ? authRepoUrl(repoUrl, username, token) : repoUrl; + const useAuth = !!(token && writable); const branch = `${participant.name}-${taskIndex}-${Date.now()}`; const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'bench-git-')); const workDir = path.join(tempDir, 'repo'); + const askpassPath = path.join(tempDir, 'askpass.sh'); + + if (useAuth) { + await writeAskpassScript(askpassPath); + } + + const env = buildGitEnv(useAuth, username, token ?? '', askpassPath); + function runGit( + args: string[], + cwd: string, + execTimeout = timeout, + ): Promise<{ stdout: string; stderr: string }> { + return execFileAsync('git', args, { cwd, env, timeout: execTimeout }); + } let cloneMs = 0; let pushMs = 0; let pullMs = 0; + let pushSucceeded = false; try { const cloneStart = performance.now(); await step('clone', () => withTimeout( - runGit(['clone', '--depth', '1', '--single-branch', cloneUrl, workDir], process.cwd(), timeout), + runGit(['clone', '--depth', '1', '--single-branch', repoUrl, workDir], process.cwd()), timeout, 'Git clone timed out', ), ); cloneMs = performance.now() - cloneStart; - if (!token) { - measure({ cloneMs, repoUrl, branch, pushSkipped: true, pullSkipped: true, commitSha: '' }); + if (!useAuth) { + measure({ cloneMs, repoUrl, branch, pushMs: 0, pullMs: 0, commitSha: '' }); return { data: { cloneMs, pushMs: 0, pullMs: 0, repoUrl, branch, commitSha: '' } }; } - const defaultBranch = await runGit(['branch', '--show-current'], workDir, timeout) + const defaultBranch = await runGit(['branch', '--show-current'], workDir) .then((r) => r.stdout.trim()) .catch(() => participant.defaultBranch ?? 'main'); // Prepare, commit, and push the test branch. - await runGit(['checkout', '-b', branch], workDir, timeout); + await runGit(['checkout', '-b', branch], workDir); await fs.promises.writeFile(path.join(workDir, 'bench.txt'), `benchmark ${branch}\n`); - await runGit(['add', 'bench.txt'], workDir, timeout); + await runGit(['add', 'bench.txt'], workDir); const commitResult = await runGit( ['-c', `user.name=${COMMITTER_NAME}`, '-c', `user.email=${COMMITTER_EMAIL}`, 'commit', '-m', `bench: ${branch}`], workDir, - timeout, ); const commitSha = commitResult.stdout.match(/\[.+?\s+([a-f0-9]+)\]/)?.[1] ?? ''; const pushStart = performance.now(); await step('push', () => withTimeout( - runGit(['push', '-u', 'origin', branch], workDir, timeout), + runGit(['push', '-u', 'origin', branch], workDir), timeout, 'Git push timed out', ), ); pushMs = performance.now() - pushStart; + pushSucceeded = true; - await runGit(['checkout', defaultBranch], workDir, timeout); + await runGit(['checkout', defaultBranch], workDir); const pullStart = performance.now(); await step('pull', () => withTimeout( - runGit(['pull', '--ff-only', 'origin', branch], workDir, timeout), + runGit(['pull', '--ff-only', 'origin', branch], workDir), timeout, 'Git pull timed out', ), @@ -141,6 +172,9 @@ export const task = defineTask(async (ctx) => { data: { repoUrl, branch, cloneMs, pushMs, pullMs }, }); } finally { + if (pushSucceeded && useAuth) { + await runGit(['push', 'origin', '--delete', branch], workDir, timeout).catch(() => {}); + } await fs.promises.rm(tempDir, { recursive: true, force: true }).catch(() => {}); } }); diff --git a/benchmarks/git/types.ts b/benchmarks/git/types.ts index 3bdae997..22cc0432 100644 --- a/benchmarks/git/types.ts +++ b/benchmarks/git/types.ts @@ -7,7 +7,7 @@ export interface GitProviderConfig extends BaseParticipant { repoUrlEnvVar?: string; /** Optional env var holding an HTTPS auth token for push/pull. */ tokenEnvVar?: string; - /** Username passed to isomorphic-git's `onAuth` callback when a token is set. */ + /** Username supplied to `git` via GIT_ASKPASS when a token is set. */ tokenUsername?: string; /** Default branch to pull back into after pushing a test branch. */ defaultBranch?: string; @@ -26,6 +26,8 @@ export interface GitTimingResult { repoUrl: string; /** Test branch that was pushed/pulled. */ branch: string; + /** Commit SHA produced by the benchmark push, when available. */ + commitSha?: string; /** Error message if this iteration failed. */ error?: string; } From 37e5c89b80cf7f613855f8709d3fefa8159846f4 Mon Sep 17 00:00:00 2001 From: garrison Date: Wed, 5 Aug 2026 18:59:31 +0000 Subject: [PATCH 06/15] feat(git): add local JSON results writer via onComplete hook Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- benchmarks/git/git.bench.ts | 6 ++ benchmarks/git/legacy-results.ts | 100 +++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 benchmarks/git/legacy-results.ts diff --git a/benchmarks/git/git.bench.ts b/benchmarks/git/git.bench.ts index 0f46ade2..d1059c54 100644 --- a/benchmarks/git/git.bench.ts +++ b/benchmarks/git/git.bench.ts @@ -16,12 +16,16 @@ import { promisify } from 'node:util'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { defineBenchmarkConfig, defineTask, TaskError } from '@benchsdk/runner'; import { withTimeout } from '../src/util/timeout.js'; import { formatError } from '../src/util/error.js'; import { providers } from './providers.js'; +import { writeGitLegacyResults } from './legacy-results.js'; import type { GitProviderConfig } from './types.js'; +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + const execFileAsync = promisify(execFile); const CLONE_TIMEOUT_MS = 60_000; const COMMITTER_NAME = 'ComputeSDK Benchmark'; @@ -77,6 +81,8 @@ export const config = defineBenchmarkConfig({ iterations: 3, concurrency: 1, participants: providers, + onComplete: (outcome) => + writeGitLegacyResults(outcome.participants, path.resolve(__dirname, '../../results/git')), }); export const task = defineTask(async (ctx) => { diff --git a/benchmarks/git/legacy-results.ts b/benchmarks/git/legacy-results.ts new file mode 100644 index 00000000..1f8386d5 --- /dev/null +++ b/benchmarks/git/legacy-results.ts @@ -0,0 +1,100 @@ +import { mkdirSync, copyFileSync } from 'node:fs'; +import path from 'node:path'; +import type { ParticipantRecords } from '@benchsdk/runner'; +import type { JsonObject } from '@benchsdk/client'; +import { byTaskIndex } from '../src/util/records.js'; +import { computeStats } from '../src/util/stats.js'; +import type { GitBenchmarkResult, GitTimingResult } from './types.js'; + +function num(x: unknown): number { + return typeof x === 'number' ? x : 0; +} + +/** Map CLI participant records to legacy GitBenchmarkResult[]. */ +export function recordsToGitResults(participants: ParticipantRecords[]): GitBenchmarkResult[] { + return participants.map((participant) => { + const iterations: GitTimingResult[] = byTaskIndex(participant.records).map((r) => { + const d = (r.data ?? {}) as JsonObject; + const base: GitTimingResult = { + cloneMs: num(d.cloneMs), + pushMs: num(d.pushMs), + pullMs: num(d.pullMs), + repoUrl: typeof d.repoUrl === 'string' ? d.repoUrl : '', + branch: typeof d.branch === 'string' ? d.branch : '', + commitSha: typeof d.commitSha === 'string' ? d.commitSha : undefined, + }; + return r.status === 'error' ? { ...base, error: r.errorCode ?? 'error' } : base; + }); + + const successful = iterations.filter((i) => !i.error); + const summary = { + cloneMs: computeStats(successful.map((i) => i.cloneMs)), + pushMs: computeStats(successful.map((i) => i.pushMs)), + pullMs: computeStats(successful.map((i) => i.pullMs)), + }; + + return { + provider: participant.participant, + mode: 'git' as const, + repoUrl: iterations[0]?.repoUrl ?? '', + iterations, + summary, + successRate: iterations.length ? successful.length / iterations.length : 0, + }; + }); +} + +async function writeGitResultsJson(results: GitBenchmarkResult[], outPath: string): Promise { + const fs = await import('node:fs'); + const os = await import('node:os'); + + const clean = results.map((r) => ({ + provider: r.provider, + mode: r.mode, + repoUrl: r.repoUrl, + iterations: r.iterations, + summary: r.summary, + ...(r.successRate !== undefined ? { successRate: r.successRate } : {}), + ...(r.compositeScore !== undefined ? { compositeScore: r.compositeScore } : {}), + ...(r.skipped ? { skipped: r.skipped, skipReason: r.skipReason } : {}), + })); + + const output = { + version: '1.0', + timestamp: new Date().toISOString(), + environment: { + node: process.version, + platform: os.platform(), + arch: os.arch(), + }, + config: { + iterations: results[0]?.iterations.length || 0, + }, + results: clean, + }; + + fs.writeFileSync(outPath, JSON.stringify(output, null, 2)); + console.log(`Results written to ${outPath}`); +} + +/** + * Map records -> GitBenchmarkResult[], compute success rates, and write both + * `.json` and `latest.json` into resultsDir. + * TEMPORARY BRIDGE until the platform read API exposes per-iteration data. + */ +export async function writeGitLegacyResults( + participants: ParticipantRecords[], + resultsDir: string, +): Promise { + const results = recordsToGitResults(participants); + + mkdirSync(resultsDir, { recursive: true }); + + const timestamp = new Date().toISOString().slice(0, 10); + const outPath = path.join(resultsDir, `${timestamp}.json`); + await writeGitResultsJson(results, outPath); + + const latestPath = path.join(resultsDir, 'latest.json'); + copyFileSync(outPath, latestPath); + console.log(`Copied latest: ${latestPath}`); +} From b5c62c0691a6dcc0bbb37b9a1bb12b7258f7c050 Mon Sep 17 00:00:00 2001 From: garrison Date: Wed, 5 Aug 2026 19:21:04 +0000 Subject: [PATCH 07/15] refactor(git): remove public fixture fallback and require repo URL + token for all providers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- benchmarks/git/git.bench.ts | 7 +++---- benchmarks/git/providers.ts | 18 ++++++------------ benchmarks/git/types.ts | 6 +++--- 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/benchmarks/git/git.bench.ts b/benchmarks/git/git.bench.ts index d1059c54..921828eb 100644 --- a/benchmarks/git/git.bench.ts +++ b/benchmarks/git/git.bench.ts @@ -3,9 +3,8 @@ * HTTPS for git hosting providers by shelling out to `git`. * Declarative — exports `config` + `task`; `bench run` owns the entrypoint. * - * The push/pull workflow runs only when BOTH the participant's token env var - * AND the writable repo URL override are set. For the read-only public fixtures - * (GitHub/GitLab/Bitbucket defaults), only the `clone` step is exercised. + * Every participant requires both a writable `*_GIT_REPO_URL` env var and a + * matching `*_TOKEN` env var; the runner skips providers without credentials. * * bench run benchmarks/git/git.bench.ts * bench run benchmarks/git/git.bench.ts --provider tensorlake --iterations 5 @@ -33,7 +32,7 @@ const COMMITTER_EMAIL = 'bench@example.com'; function resolveRepoConfig(config: GitProviderConfig): { repoUrl: string; writable: boolean } { const override = config.repoUrlEnvVar ? process.env[config.repoUrlEnvVar] : undefined; - const repoUrl = override ? sanitizeRepoUrl(override) : sanitizeRepoUrl(config.url); + const repoUrl = override ? sanitizeRepoUrl(override) : sanitizeRepoUrl(config.url ?? ''); return { repoUrl, writable: !!override }; } diff --git a/benchmarks/git/providers.ts b/benchmarks/git/providers.ts index 7f76b73a..82bedc1a 100644 --- a/benchmarks/git/providers.ts +++ b/benchmarks/git/providers.ts @@ -3,33 +3,28 @@ import type { GitProviderConfig } from './types.js'; /** * Git hosting provider benchmark configurations. * - * Each participant points at an HTTPS repo. The `url` field provides a - * read-only default (where a public fixture exists); set the matching - * `*_GIT_REPO_URL` env var to a writable repo and the matching token env var - * to enable the push/pull workflow. Tensorlake is gated by required env vars - * because it has no public fixture. + * Every participant requires both a writable `*_GIT_REPO_URL` env var and a + * matching `*_TOKEN` env var. The runner skips providers whose credentials are + * missing, so there is no read-only fallback. */ export const providers: GitProviderConfig[] = [ { name: 'github', - requiredEnvVars: [], - url: 'https://github.com/octocat/Spoon-Knife.git', + requiredEnvVars: ['GITHUB_GIT_REPO_URL', 'GITHUB_TOKEN'], repoUrlEnvVar: 'GITHUB_GIT_REPO_URL', tokenEnvVar: 'GITHUB_TOKEN', tokenUsername: 'token', }, { name: 'gitlab', - requiredEnvVars: [], - url: 'https://gitlab.com/gitlab-org/gitlab-test.git', + requiredEnvVars: ['GITLAB_GIT_REPO_URL', 'GITLAB_TOKEN'], repoUrlEnvVar: 'GITLAB_GIT_REPO_URL', tokenEnvVar: 'GITLAB_TOKEN', tokenUsername: 'oauth2', }, { name: 'bitbucket', - requiredEnvVars: [], - url: 'https://bitbucket.org/atlassian/hello-world.git', + requiredEnvVars: ['BITBUCKET_GIT_REPO_URL', 'BITBUCKET_TOKEN'], repoUrlEnvVar: 'BITBUCKET_GIT_REPO_URL', tokenEnvVar: 'BITBUCKET_TOKEN', tokenUsername: 'x-token-auth', @@ -37,7 +32,6 @@ export const providers: GitProviderConfig[] = [ { name: 'tensorlake', requiredEnvVars: ['TENSORLAKE_GIT_REPO_URL', 'TENSORLAKE_API_KEY'], - url: process.env.TENSORLAKE_GIT_REPO_URL ?? '', repoUrlEnvVar: 'TENSORLAKE_GIT_REPO_URL', tokenEnvVar: 'TENSORLAKE_API_KEY', tokenUsername: 't', diff --git a/benchmarks/git/types.ts b/benchmarks/git/types.ts index 22cc0432..f8cc05c4 100644 --- a/benchmarks/git/types.ts +++ b/benchmarks/git/types.ts @@ -1,9 +1,9 @@ import type { BaseParticipant } from '@benchsdk/client'; export interface GitProviderConfig extends BaseParticipant { - /** HTTPS URL of the repo to clone (used when no env override is set). */ - url: string; - /** Optional env var that overrides `url` for the read/write repo. */ + /** Optional fallback HTTPS URL of the repo to clone. */ + url?: string; + /** Env var that overrides `url` for the read/write repo. */ repoUrlEnvVar?: string; /** Optional env var holding an HTTPS auth token for push/pull. */ tokenEnvVar?: string; From 3632871b82965b0b37ad48009d9810423f0aa658 Mon Sep 17 00:00:00 2001 From: garrison Date: Wed, 5 Aug 2026 19:28:37 +0000 Subject: [PATCH 08/15] ci(git): add weekly GitHub Actions workflow for git benchmark Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/git-benchmarks.yml | 102 +++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 .github/workflows/git-benchmarks.yml diff --git a/.github/workflows/git-benchmarks.yml b/.github/workflows/git-benchmarks.yml new file mode 100644 index 00000000..2b9ed5e5 --- /dev/null +++ b/.github/workflows/git-benchmarks.yml @@ -0,0 +1,102 @@ +name: Git Benchmark + +on: + push: + branches: [master] + paths: + - 'benchmarks/git/**' + - 'benchmarks/src/util/**' + - 'package.json' + - '.github/workflows/git-benchmarks.yml' + schedule: + - cron: '0 0 * * 1' # Weekly on Monday at midnight UTC + workflow_dispatch: + inputs: + iterations: + description: 'Iterations per provider' + required: false + default: '10' + provider: + description: 'Provider to run (leave empty for all)' + required: false + default: '' + dry_run: + description: 'Run without committing results' + required: false + default: false + type: boolean + +concurrency: + group: git-benchmarks + cancel-in-progress: true + +permissions: + contents: write + pull-requests: write + +jobs: + bench: + name: Git Benchmark + runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: 'pnpm' + - name: Install dependencies + run: | + if [ "${{ github.event_name }}" = "schedule" ]; then + pnpm update + else + pnpm install --frozen-lockfile + fi + - name: Clear stale results from checkout + run: rm -rf results/git/ + - name: Run git benchmark + run: | + . benchmarks/scripts/load-vault-secrets.sh '^(GITHUB_GIT_REPO_URL|GITHUB_TOKEN|GITLAB_GIT_REPO_URL|GITLAB_TOKEN|BITBUCKET_GIT_REPO_URL|BITBUCKET_TOKEN|TENSORLAKE_GIT_REPO_URL|TENSORLAKE_API_KEY|BENCHMARKS_PLATFORM_API_KEY)$' + + PROVIDER_ARG="" + if [ -n "${{ github.event.inputs.provider }}" ]; then + PROVIDER_ARG="--provider ${{ github.event.inputs.provider }}" + fi + + ITERATIONS="${{ (github.event_name == 'schedule' && '50') || github.event.inputs.iterations || '10' }}" + + npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts \ + $PROVIDER_ARG \ + --iterations $ITERATIONS + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: git-results + path: results/git/ + if-no-files-found: ignore + retention-days: 7 + - name: Commit and push results + if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add results/git/ + git diff --cached --quiet && echo "No changes to commit" && exit 0 + git commit -m "chore: update git benchmark results [skip ci]" + + branch="${GITHUB_REF#refs/heads/}" + for attempt in 1 2 3 4 5; do + git fetch origin "${branch}" + git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; } + if git push origin "HEAD:${branch}"; then + echo "Pushed on attempt ${attempt}" + exit 0 + fi + echo "Push rejected (attempt ${attempt}); will rebase and retry" + done + echo "Failed to push after multiple attempts" >&2 + exit 1 From 9220990ee1d475b340275a6843576288dd70dd7b Mon Sep 17 00:00:00 2001 From: garrison Date: Wed, 5 Aug 2026 19:33:29 +0000 Subject: [PATCH 09/15] refactor(git): per-provider matrix workflow and merge-results --mode git Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/git-benchmarks.yml | 47 +++++++++++---- benchmarks/git/legacy-results.ts | 2 +- benchmarks/src/merge-results.ts | 88 +++++++++++++++++++++++++++- 3 files changed, 125 insertions(+), 12 deletions(-) diff --git a/.github/workflows/git-benchmarks.yml b/.github/workflows/git-benchmarks.yml index 2b9ed5e5..3b712010 100644 --- a/.github/workflows/git-benchmarks.yml +++ b/.github/workflows/git-benchmarks.yml @@ -6,6 +6,7 @@ on: paths: - 'benchmarks/git/**' - 'benchmarks/src/util/**' + - 'benchmarks/src/merge-results.ts' - 'package.json' - '.github/workflows/git-benchmarks.yml' schedule: @@ -36,13 +37,15 @@ permissions: jobs: bench: - name: Git Benchmark + name: Bench ${{ matrix.provider }} runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + provider: ${{ (github.event.inputs.provider != '' && fromJson(format('["{0}"]', github.event.inputs.provider))) || fromJson('["github","gitlab","bitbucket","tensorlake"]') }} steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 - uses: pnpm/action-setup@v4 - uses: actions/setup-node@v4 with: @@ -61,24 +64,48 @@ jobs: run: | . benchmarks/scripts/load-vault-secrets.sh '^(GITHUB_GIT_REPO_URL|GITHUB_TOKEN|GITLAB_GIT_REPO_URL|GITLAB_TOKEN|BITBUCKET_GIT_REPO_URL|BITBUCKET_TOKEN|TENSORLAKE_GIT_REPO_URL|TENSORLAKE_API_KEY|BENCHMARKS_PLATFORM_API_KEY)$' - PROVIDER_ARG="" - if [ -n "${{ github.event.inputs.provider }}" ]; then - PROVIDER_ARG="--provider ${{ github.event.inputs.provider }}" - fi - ITERATIONS="${{ (github.event_name == 'schedule' && '50') || github.event.inputs.iterations || '10' }}" npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts \ - $PROVIDER_ARG \ + --provider ${{ matrix.provider }} \ --iterations $ITERATIONS - name: Upload results if: always() uses: actions/upload-artifact@v4 with: - name: git-results + name: git-results-${{ matrix.provider }} path: results/git/ if-no-files-found: ignore retention-days: 7 + + collect: + name: Collect Git Results + runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe + needs: [bench] + if: always() + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: 'pnpm' + - name: Install dependencies + run: | + if [ "${{ github.event_name }}" = "schedule" ]; then + pnpm update + else + pnpm install --frozen-lockfile + fi + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts/ + pattern: git-results-* + - name: Merge results + run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode git - name: Commit and push results if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' run: | diff --git a/benchmarks/git/legacy-results.ts b/benchmarks/git/legacy-results.ts index 1f8386d5..fbb4873d 100644 --- a/benchmarks/git/legacy-results.ts +++ b/benchmarks/git/legacy-results.ts @@ -44,7 +44,7 @@ export function recordsToGitResults(participants: ParticipantRecords[]): GitBenc }); } -async function writeGitResultsJson(results: GitBenchmarkResult[], outPath: string): Promise { +export async function writeGitResultsJson(results: GitBenchmarkResult[], outPath: string): Promise { const fs = await import('node:fs'); const os = await import('node:os'); diff --git a/benchmarks/src/merge-results.ts b/benchmarks/src/merge-results.ts index 793546c5..7da01477 100644 --- a/benchmarks/src/merge-results.ts +++ b/benchmarks/src/merge-results.ts @@ -1,7 +1,7 @@ /** * Merge per-provider benchmark results into combined result files. * - * Usage: tsx src/merge-results.ts --input [--mode storage|snapshot-fork|browser|browser-throughput|ai-gateway] + * Usage: tsx src/merge-results.ts --input [--mode storage|snapshot-fork|browser|browser-throughput|ai-gateway|git] * * By default, merges sandbox benchmark results: reads latest.json files from * the input directory, groups by mode (sequential/staggered/burst), computes @@ -21,6 +21,9 @@ * With --mode ai-gateway, merges AI gateway benchmark results: deduplicates * by provider, computes AI-gateway-specific composite scores, and writes * combined files to results/ai-gateway/latest.json. + * + * With --mode git, merges git benchmark results: deduplicates by provider + * and writes combined files to results/git/latest.json. */ import fs from 'fs'; import path from 'path'; @@ -34,12 +37,14 @@ import { } from '../browser/throughput-scoring.js'; import { computeAIGatewayCompositeScores, sortAIGatewayByCompositeScore } from '../ai-gateway/scoring.js'; import { printResultsTable, writeResultsJson } from '../sandbox/table.js'; +import { writeGitResultsJson } from '../git/legacy-results.js'; import type { BenchmarkResult } from '../sandbox/types.js'; import type { StorageBenchmarkResult } from '../storage/types.js'; import type { SnapshotForkBenchmarkResult } from '../storage/snapshot-fork-types.js'; import type { BrowserBenchmarkResult } from '../browser/types.js'; import type { ThroughputBenchmarkResult } from '../browser/throughput-types.js'; import type { AIGatewayBenchmarkResult } from '../ai-gateway/types.js'; +import type { GitBenchmarkResult } from '../git/types.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(__dirname, '../..'); @@ -734,6 +739,85 @@ async function mainAIGateway() { console.log(`Copied latest: ${latestPath}`); } +function printGitResultsTable(results: GitBenchmarkResult[]): void { + const sorted = [...results].sort((a, b) => (a.provider).localeCompare(b.provider)); + + console.log(`\n${'='.repeat(95)}`); + console.log(' GIT BENCHMARK RESULTS'); + console.log('='.repeat(95)); + console.log( + ['Provider', 'Clone', 'Push', 'Pull', 'Success'] + .map((h, i) => h.padEnd([14, 12, 12, 12, 10][i])) + .join(' | '), + ); + console.log( + [14, 12, 12, 12, 10].map((w) => '-'.repeat(w)).join('-+-'), + ); + + for (const r of sorted) { + if (r.skipped) { + console.log([r.provider.padEnd(14), '--'.padEnd(12), '--'.padEnd(12), '--'.padEnd(12), 'SKIPPED'.padEnd(10)].join(' | ')); + continue; + } + const ok = r.iterations.filter((i) => !i.error).length; + const total = r.iterations.length; + const clone = (r.summary.cloneMs.median / 1000).toFixed(2) + 's'; + const push = (r.summary.pushMs.median / 1000).toFixed(2) + 's'; + const pull = (r.summary.pullMs.median / 1000).toFixed(2) + 's'; + console.log([r.provider.padEnd(14), clone.padEnd(12), push.padEnd(12), pull.padEnd(12), `${ok}/${total}`.padEnd(10)].join(' | ')); + } + console.log('='.repeat(95)); +} + +async function mainGit() { + const jsonFiles: string[] = []; + function walk(dir: string) { + if (!fs.existsSync(dir)) return; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else if (entry.name === 'latest.json') jsonFiles.push(full); + } + } + walk(inputDir!); + + if (jsonFiles.length === 0) { + console.error(`No latest.json files found in ${inputDir}`); + process.exit(1); + } + + console.log(`Found ${jsonFiles.length} result files`); + + const seen = new Map(); + + for (const file of jsonFiles) { + const raw = JSON.parse(fs.readFileSync(file, 'utf-8')) as { results: GitBenchmarkResult[] }; + const fromSingleProvider = raw.results.length === 1; + for (const result of raw.results) { + const existing = seen.get(result.provider); + if (!existing || (fromSingleProvider && !existing.fromSingleProvider)) { + seen.set(result.provider, { result, fromSingleProvider }); + } + } + } + + const deduped = Array.from(seen.values()).map((e) => e.result); + console.log(`\nMerging ${deduped.length} provider results for mode: git`); + + printGitResultsTable(deduped); + + const timestamp = new Date().toISOString().slice(0, 10); + const resultsDir = path.resolve(ROOT, 'results/git'); + fs.mkdirSync(resultsDir, { recursive: true }); + + const outPath = path.join(resultsDir, `${timestamp}.json`); + await writeGitResultsJson(deduped, outPath); + + const latestPath = path.join(resultsDir, 'latest.json'); + fs.copyFileSync(outPath, latestPath); + console.log(`Copied latest: ${latestPath}`); +} + const runner = mergeMode === 'storage' ? mainStorage : mergeMode === 'snapshot-fork' @@ -744,6 +828,8 @@ const runner = mergeMode === 'storage' ? mainBrowserThroughput : mergeMode === 'ai-gateway' ? mainAIGateway + : mergeMode === 'git' + ? mainGit : main; runner().catch(err => { console.error('Merge failed:', err); From ac8f21ac9e2fa2320d643741541c74f9b00b56d5 Mon Sep 17 00:00:00 2001 From: Noah Kiser <154091133+kisernl@users.noreply.github.com> Date: Wed, 5 Aug 2026 12:54:32 -0500 Subject: [PATCH 10/15] feat: add DAX benchmark SVG (#281) --- .github/workflows/sandbox-dax-benchmarks.yml | 11 +- README.md | 4 + benchmarks/sandbox/generate-dax-svg.ts | 136 +++++++++ dax.svg | 304 +++++++++++++++++++ package.json | 1 + 5 files changed, 455 insertions(+), 1 deletion(-) create mode 100644 benchmarks/sandbox/generate-dax-svg.ts create mode 100644 dax.svg diff --git a/.github/workflows/sandbox-dax-benchmarks.yml b/.github/workflows/sandbox-dax-benchmarks.yml index f05f49b6..c8a46001 100644 --- a/.github/workflows/sandbox-dax-benchmarks.yml +++ b/.github/workflows/sandbox-dax-benchmarks.yml @@ -8,6 +8,8 @@ on: - 'benchmarks/sandbox/providers.ts' - 'benchmarks/src/merge-results.ts' - 'benchmarks/scripts/dax-benchmark.sh' + - 'benchmarks/sandbox/generate-dax-svg.ts' + - 'dax.svg' - 'package.json' - '.github/workflows/sandbox-dax-benchmarks.yml' schedule: @@ -149,6 +151,13 @@ jobs: pattern: results-* - name: Merge results run: npx tsx benchmarks/src/merge-results.ts --input artifacts + - name: Generate DAX SVG + run: | + if [ -f results/sandbox-dax/latest.json ]; then + pnpm run generate-dax-svg + else + echo "No DAX results found; skipping chart" + fi - name: Ingest results to platform if: github.event_name != 'push' && github.event.inputs.dry_run != 'true' continue-on-error: true @@ -248,7 +257,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git add results/sandbox-dax/ + git add results/sandbox-dax/ dax.svg git diff --cached --quiet && echo "No changes to commit" && exit 0 git commit -m "chore: update dax benchmark results [skip ci]" diff --git a/README.md b/README.md index bef4c0a3..7699c9ac 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,10 @@ Our partners support our independent benchmarks. ![Object Storage Snapshot & Fork](./snapshot_fork_small.svg) +### [DAX Sandbox Builds](#dax-sandbox-builds) + +![DAX Sandbox Builds](./dax.svg) + [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](./LICENSE) **TTI (Time to Interactive)** = API call to first command execution. Lower is better. diff --git a/benchmarks/sandbox/generate-dax-svg.ts b/benchmarks/sandbox/generate-dax-svg.ts new file mode 100644 index 00000000..5c7b87f9 --- /dev/null +++ b/benchmarks/sandbox/generate-dax-svg.ts @@ -0,0 +1,136 @@ +#!/usr/bin/env node + +import fs from 'node:fs'; +import path from 'node:path'; +import type { DaxBenchmarkResult } from './dax.js'; + +const root = path.resolve(import.meta.dirname, '../..'); +const inputPath = path.join(root, 'results', 'sandbox-dax', 'latest.json'); +const outputPath = path.join(root, 'dax.svg'); + +interface DaxResultFile { + timestamp: string; + results: DaxBenchmarkResult[]; +} + +const data = JSON.parse(fs.readFileSync(inputPath, 'utf8')) as DaxResultFile; + +const escape = (value: string | number): string => String(value) + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"'); +const format = (ms: number | undefined): string => (ms && ms > 0 ? `${(ms / 1000).toFixed(2)}s` : '—'); +const displayName = (name: string): string => name.toLowerCase() === 'e2b' + ? 'E2B' + : name.split('-').map((part) => part.charAt(0).toUpperCase() + part.slice(1)).join(' '); +const results = data.results.filter((result) => !result.skipped).map((result) => { + const successful = result.iterations.filter((iteration) => + !iteration.error && typeof iteration.totalMs === 'number' && iteration.totalMs > 0, + ); + const phaseTotal = Math.max(7, ...result.iterations.map((iteration) => iteration.phasesTotal || 0)); + const phaseCompleted = result.iterations.length + ? Math.max(...result.iterations.map((iteration) => iteration.phasesCompleted || 0)) + : 0; + return { + ...result, + successful, + phaseTotal, + phaseCompleted, + successPercent: result.iterations.length ? Math.round(successful.length / result.iterations.length * 100) : 0, + median: result.summary?.totalMs?.median || 0, + }; +}).sort((a, b) => { + if (a.phaseCompleted !== b.phaseCompleted) return b.phaseCompleted - a.phaseCompleted; + if (Boolean(a.successful.length) !== Boolean(b.successful.length)) return b.successful.length - a.successful.length; + return (a.median || Number.MAX_SAFE_INTEGER) - (b.median || Number.MAX_SAFE_INTEGER); +}); + +const width = 1200; +const padding = 24; +const headerHeight = 110; +const sectionGap = 28; +const tableHeaderHeight = 42; +const tableRowHeight = 42; +const tableTop = headerHeight + sectionGap + 34; +const height = tableTop + tableHeaderHeight + results.length * tableRowHeight + 52; +const timestamp = new Date(data.timestamp).toISOString().slice(0, 10); +const logoPath = 'M1036.26,1002.28h237.87l-.93,19.09c-8.38,110.32-49.81,198.3-123.82,262.07-73.09,63.31-170.84,95.43-290.48,95.43-130.81,0-235.55-44.69-311.43-133.6-74.48-87.98-112.65-209.48-112.65-361.23v-60.51c0-96.83,17.7-183.41,51.68-257.43,34.91-74.95,85.19-133.61,149.89-173.63,64.7-40.04,140.12-60.52,225.3-60.52,117.77,0,214.13,32.12,286.29,95.9,72.62,63.3,114.98,153.61,126.15,267.67l1.86,19.08h-238.34l-.93-15.83c-4.65-59.11-20.95-101.94-47.95-127.08-27-25.6-69.83-38.17-127.08-38.17-61.91,0-107.06,20.95-137.33,65.17-31.65,45.15-47.94,117.77-48.87,215.53v74.48c0,102.41,15.36,177.83,45.62,223.91,28.86,44.22,74.01,65.63,137.79,65.63,58.19,0,101.48-12.57,128.95-38.17,26.99-25.14,43.29-66.1,47.48-121.5l.93-16.3Z'; +const cols = [ + { x: 40, label: 'PROVIDER' }, + { x: 220, label: 'PHASES' }, + { x: 300, label: 'SUCCESS' }, + { x: 390, label: 'TOTAL (MED)' }, + { x: 530, label: 'PREPARE' }, + { x: 640, label: 'BUN DL' }, + { x: 735, label: 'BUN UNPACK' }, + { x: 850, label: 'CLONE' }, + { x: 940, label: 'INSTALL' }, + { x: 1050, label: 'TYPECHECK' }, +]; + +let svg = ` + + + + + + + + + + + + + DAX Sandbox Benchmarks + Build and typecheck workload performance across sandbox providers + Detailed Metrics +`; + +const tableHeaderY = tableTop; +svg += ` + +`; +for (const col of cols) svg += ` ${col.label}\n`; + +results.forEach((result, index) => { + const y = tableHeaderY + tableHeaderHeight + index * tableRowHeight; + const successful = result.successful.length; + const total = result.iterations.length; + const successClass = result.successPercent >= 100 ? 'good' : result.successPercent ? 'warn' : 'bad'; + const summary = result.summary || {}; + const values = [ + summary.totalMs?.median, + summary.prepareMs?.median, + summary.bunDownloadMs?.median, + summary.bunUnpackMs?.median, + summary.cloneMs?.median, + summary.installMs?.median, + summary.typecheckMs?.median, + ]; + svg += ` + ${escape(displayName(result.provider))} + ${result.phaseCompleted}/${result.phaseTotal} + ${result.successPercent}% + ${successful ? format(values[0]) : 'Failed'} +`; + values.slice(1).forEach((value, valueIndex) => { + svg += ` ${successful ? format(value) : '—'}\n`; + }); +}); + +svg += ` DAX phases: prepare, Bun download, Bun unpack, clone, install, and typecheck. Lower duration is better. +\n`; + +fs.writeFileSync(outputPath, svg); +console.log(`Wrote ${outputPath}`); diff --git a/dax.svg b/dax.svg new file mode 100644 index 00000000..9f169b75 --- /dev/null +++ b/dax.svg @@ -0,0 +1,304 @@ + + + + + + + + + + + + + + DAX Sandbox Benchmarks + Build and typecheck workload performance across sandbox providers + Detailed Metrics + + + PROVIDER + PHASES + SUCCESS + TOTAL (MED) + PREPARE + BUN DL + BUN UNPACK + CLONE + INSTALL + TYPECHECK + + Namespace + 7/7 + 67% + 35.50s + 2.68s + 0.18s + 0.42s + 1.17s + 7.60s + 19.58s + + Lightning + 7/7 + 100% + 41.56s + 4.97s + 0.52s + 0.62s + 2.68s + 11.94s + 16.88s + + Blaxel + 7/7 + 33% + 44.39s + 2.18s + 0.34s + 0.62s + 2.19s + 11.74s + 33.33s + + Createos + 7/7 + 100% + 50.25s + 2.98s + 0.21s + 0.47s + 2.12s + 10.64s + 26.62s + + Upstash + 7/7 + 100% + 51.25s + 3.68s + 0.26s + 0.62s + 1.75s + 10.45s + 25.11s + + Tensorlake + 7/7 + 100% + 56.39s + 12.59s + 0.23s + 0.57s + 1.68s + 10.30s + 27.59s + + Tenki + 7/7 + 100% + 66.26s + 3.89s + 0.68s + 0.62s + 4.17s + 15.55s + 36.86s + + Daytona + 7/7 + 100% + 70.43s + 2.82s + 0.31s + 0.56s + 3.94s + 13.75s + 37.25s + + E2B + 7/7 + 100% + 78.13s + 8.09s + 0.51s + 0.75s + 2.92s + 18.17s + 41.15s + + Vercel + 7/7 + 100% + 80.13s + 17.17s + 0.17s + 0.68s + 2.27s + 11.86s + 42.38s + + Superserve + 7/7 + 100% + 88.19s + 4.51s + 0.30s + 0.57s + 4.41s + 15.64s + 59.94s + + Isorun + 7/7 + 67% + 94.34s + 0.98s + 0.35s + 0.54s + 1.50s + 10.97s + 27.39s + + Modal + 7/7 + 100% + 99.76s + 4.03s + 0.35s + 0.70s + 2.40s + 15.62s + 69.42s + + Declaw + 7/7 + 67% + 179.48s + 5.97s + 0.66s + 0.66s + 4.55s + 15.14s + 139.80s + + Sandbox0 + 7/7 + 100% + 183.00s + 3.81s + 1.12s + 0.68s + 7.62s + 73.26s + 79.58s + + Opencomputer + 6/7 + 0% + Failed + — + — + — + — + — + — + + Archil + 6/7 + 0% + Failed + — + — + — + — + — + — + + Runloop + 2/7 + 100% + 85.04s + — + — + — + — + 19.77s + 37.04s + + Cloud Run + 2/7 + 0% + Failed + — + — + — + — + — + — + + Beam + 0/7 + 0% + Failed + — + — + — + — + — + — + + Cloudflare + 0/7 + 0% + Failed + — + — + — + — + — + — + + Codesandbox + 0/7 + 0% + Failed + — + — + — + — + — + — + + Hopx + 0/7 + 0% + Failed + — + — + — + — + — + — + + Northflank + 0/7 + 0% + Failed + — + — + — + — + — + — + DAX phases: prepare, Bun download, Bun unpack, clone, install, and typecheck. Lower duration is better. + diff --git a/package.json b/package.json index 7837d7f3..727ab1e9 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,7 @@ "generate-svg:sequential": "tsx benchmarks/sandbox/generate-svg.ts --mode sequential", "generate-svg:staggered": "tsx benchmarks/sandbox/generate-svg.ts --mode staggered", "generate-svg:burst": "tsx benchmarks/sandbox/generate-svg.ts --mode burst", + "generate-dax-svg": "tsx benchmarks/sandbox/generate-dax-svg.ts", "generate-storage-svg": "tsx benchmarks/storage/generate-svg.ts", "generate-snapshot-fork-svg": "tsx benchmarks/storage/generate-snapshot-fork-svg.ts", "generate-browser-svg": "tsx benchmarks/browser/generate-svg.ts", From 9a075078c167058a8586b22ed2b87c1959c92e0f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:16:40 -0500 Subject: [PATCH 11/15] feat(runner): verbs-only `bench run` with --shape and --run-key (#264) * feat(runner): bench create-run + --run-id so every provider reports into one run Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(runner): create the shared run empty and let each joiner register itself Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(ci): drop the no-op --provider from create-run Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(runner): verb-first CLI (bench create benchmark|run) and let the run own its size Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * docs(bench): show the shared-run invocation in the TTI header Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(runner): open runs without a size; joiners bring their own iteration count Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(runner): don't rename a benchmark that --benchmark merely retargets Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(runner): verbs-only bench run with --shape and --run-key Retire the imperative create commands: the benchmark is declared in the .bench.ts file (via optional named shapes) and materialized on run, and a run is opened as a side effect. --shape selects a named variant (swapping platform identity + stable knobs); --run-key get-or-creates a shared run so sibling provider jobs converge on one comparable run. Collapse the per-shape slug/name/knob triple out of package scripts and the sandbox TTI workflow (drop the create-runs job; providers pass a shared run key including GITHUB_RUN_ATTEMPT). Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: garrison Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .changeset/benchsdk-client-run-key.md | 5 + .changeset/benchsdk-runner-shapes-run-key.md | 11 + .changeset/benchsdk-runner-slug-flag.md | 5 - .changeset/participant-sized-runs.md | 5 + .github/workflows/sandbox-tti-benchmarks.yml | 23 +- benchmarks/sandbox/tti.bench.ts | 31 ++- package.json | 44 ++-- .../src/__tests__/bench-config.test.ts | 25 +++ .../benchsdk-runner/src/__tests__/cli.test.ts | 14 +- .../src/__tests__/runner.test.ts | 105 ++++++++- packages/benchsdk-runner/src/bench-config.ts | 49 +++- packages/benchsdk-runner/src/cli.ts | 42 ++-- packages/benchsdk-runner/src/runner.ts | 210 ++++++++++++++---- packages/benchsdk/src/types.ts | 14 +- 14 files changed, 458 insertions(+), 125 deletions(-) create mode 100644 .changeset/benchsdk-client-run-key.md create mode 100644 .changeset/benchsdk-runner-shapes-run-key.md delete mode 100644 .changeset/benchsdk-runner-slug-flag.md create mode 100644 .changeset/participant-sized-runs.md diff --git a/.changeset/benchsdk-client-run-key.md b/.changeset/benchsdk-client-run-key.md new file mode 100644 index 00000000..90795488 --- /dev/null +++ b/.changeset/benchsdk-client-run-key.md @@ -0,0 +1,5 @@ +--- +"@benchsdk/client": minor +--- + +`createRun` accepts an optional `runKey`: callers passing the same key (per org + benchmark) get-or-create one shared run instead of each opening its own. `BenchmarkRun.runKey` reports the key a run was created with. diff --git a/.changeset/benchsdk-runner-shapes-run-key.md b/.changeset/benchsdk-runner-shapes-run-key.md new file mode 100644 index 00000000..bf198f3c --- /dev/null +++ b/.changeset/benchsdk-runner-shapes-run-key.md @@ -0,0 +1,11 @@ +--- +"@benchsdk/runner": minor +--- + +Verbs-only CLI: `bench run ` is the one mutating command. The benchmark is declared in the file and materialized (upserted) as a side effect of running it, and a run is opened as a side effect too — there are no imperative `bench create benchmark` / `bench create run` commands. + +`bench run` gains `--shape `: a bench file can declare named `shapes`, each swapping in its own platform identity (`slug`/`name`, optional `kind`) and a stable knob (`staggerDelayMs`) while reusing the same task and participants. This collapses the per-shape slug/name/knob triple that was duplicated across package scripts and CI. + +`bench run` gains `--run-key `: sibling processes passing the same key (per org + benchmark) get-or-create one shared run instead of each opening its own, so provider jobs running in parallel land in a single, directly-comparable run. Each process registers only the participants it runs. The key binding is permanent, so callers that need a fresh run (e.g. a CI re-run) vary the key (e.g. include `GITHUB_RUN_ATTEMPT`). + +`--slug` remains a working alias for `--benchmark`. diff --git a/.changeset/benchsdk-runner-slug-flag.md b/.changeset/benchsdk-runner-slug-flag.md deleted file mode 100644 index f100a1c7..00000000 --- a/.changeset/benchsdk-runner-slug-flag.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@benchsdk/runner": minor ---- - -Add `--slug` and `--name` CLI overrides for `benchmarkSlug`/`benchmarkName`, so one `*.bench.ts` can report under several platform benchmarks (e.g. the sandbox TTI entrypoint reporting sequential/staggered/burst runs of the same workload). diff --git a/.changeset/participant-sized-runs.md b/.changeset/participant-sized-runs.md new file mode 100644 index 00000000..6d8b8121 --- /dev/null +++ b/.changeset/participant-sized-runs.md @@ -0,0 +1,5 @@ +--- +"@benchsdk/client": minor +--- + +`createRun` no longer requires `totalTasks`: omit it to open a participant-sized run, whose total is the sum of what its participants declare when they register. `BenchmarkRun.participantSized` reports which kind a run is. diff --git a/.github/workflows/sandbox-tti-benchmarks.yml b/.github/workflows/sandbox-tti-benchmarks.yml index f7d94031..fcebf0ab 100644 --- a/.github/workflows/sandbox-tti-benchmarks.yml +++ b/.github/workflows/sandbox-tti-benchmarks.yml @@ -118,19 +118,26 @@ jobs: run: | . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|ARKER_API_KEY|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' - # One entrypoint, three launch shapes: the knobs come from the CLI - # and --slug/--name pick which platform benchmark each reports to. + # One entrypoint, three launch shapes selected with --shape; the scale + # knobs come from the CLI. Every provider passes the same --run-key, so + # each shape's providers get-or-create one shared run and rank against + # each other (each still registers itself and claims its own worker). + # The shapes are distinct benchmark slugs, so one key value yields one + # run per shape; the run attempt is in the key so a workflow re-run + # opens fresh runs instead of rejoining the completed ones. BENCH=(npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts) - COMMON=(--provider ${{ matrix.provider }} \ - --iterations ${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }}) + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + COMMON=( + --provider ${{ matrix.provider }} + --iterations ${{ github.event.inputs.iterations || (github.event_name == 'push' && '10') || '100' }} + --run-key "$RUN_KEY" + ) CONCURRENCY=${{ github.event.inputs.concurrency || (github.event_name == 'push' && '10') || '100' }} # Sequential is one-at-a-time by definition, so it never gets # --concurrency (the config default pins it to 1). SEQUENTIAL=("${BENCH[@]}" "${COMMON[@]}") - STAGGERED=("${BENCH[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" --stagger-delay-ms 200 \ - --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)') - BURST=("${BENCH[@]}" "${COMMON[@]}" --concurrency "$CONCURRENCY" \ - --slug sandbox-burst-local --name 'Sandbox burst TTI (local)') + STAGGERED=("${BENCH[@]}" "${COMMON[@]}" --shape staggered --concurrency "$CONCURRENCY") + BURST=("${BENCH[@]}" "${COMMON[@]}" --shape burst --concurrency "$CONCURRENCY") case "${{ github.event.inputs.mode }}" in sequential) "${SEQUENTIAL[@]}" ;; staggered) "${STAGGERED[@]}" ;; diff --git a/benchmarks/sandbox/tti.bench.ts b/benchmarks/sandbox/tti.bench.ts index bbb7a80c..a1b9aa7f 100644 --- a/benchmarks/sandbox/tti.bench.ts +++ b/benchmarks/sandbox/tti.bench.ts @@ -3,17 +3,23 @@ * first command (`node -v`) succeeding, excluding destroy. Declarative — * exports `config` + `task`; `bench run` owns the entrypoint. * - * One workload, three launch shapes — the only thing that differs is the - * framework's own knobs, so they're CLI flags rather than separate files: - * sequential one at a time (the config default: concurrency 1) - * burst --concurrency N: all slots open at once, so this launches that - * many real sandboxes simultaneously — raise N deliberately - * staggered --concurrency N --stagger-delay-ms D: task i starts at i * D - * Each shape reports under its own platform benchmark via `--slug`/`--name`. + * One workload, three launch shapes. Each shape is its own platform benchmark + * but the same task, so they're declared once in `shapes` and picked with + * `--shape`; the scale knobs (`--iterations`/`--concurrency`) stay on the CLI: + * sequential the base config (concurrency 1) — no `--shape` + * burst --concurrency N: all slots open at once, launching that many + * real sandboxes simultaneously — raise N deliberately + * staggered its 200ms delay is baked into the shape; task i starts at i * D * * bench run benchmarks/sandbox/tti.bench.ts --iterations 5 --provider e2b,modal - * bench run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 10 --concurrency 10 - * bench run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 10 --concurrency 10 --stagger-delay-ms 200 + * bench run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 10 --concurrency 10 + * bench run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 10 --concurrency 10 + * + * To rank providers against each other, run each provider with the same + * `--run-key`: they get-or-create one shared run and each claims its own worker. + * + * bench run benchmarks/sandbox/tti.bench.ts --shape burst --provider e2b --run-key "$GITHUB_RUN_ID" + * bench run benchmarks/sandbox/tti.bench.ts --shape burst --provider modal --run-key "$GITHUB_RUN_ID" */ import '../src/env.js'; import path from 'node:path'; @@ -52,6 +58,13 @@ export const config = defineBenchmarkConfig({ benchmarkKind: 'sandbox', iterations: 2, concurrency: 1, + // The launch shapes: same task, distinct platform identities. `--shape burst` + // just swaps the slug/name — the caller brings `--concurrency`. Staggered + // also carries its defining 200ms delay so no caller has to remember it. + shapes: { + burst: { slug: 'sandbox-burst-local', name: 'Sandbox burst TTI (local)' }, + staggered: { slug: 'sandbox-staggered-local', name: 'Sandbox staggered TTI (local)', staggerDelayMs: 200 }, + }, participants: providers, onComplete: (outcome) => { const { resultsDir, mode } = legacyShape(outcome.config); diff --git a/package.json b/package.json index 727ab1e9..489cb889 100644 --- a/package.json +++ b/package.json @@ -5,30 +5,30 @@ "type": "module", "scripts": { "typecheck": "tsc --noEmit", - "bench": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3", + "bench": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3", "bench:sequential": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts", - "bench:staggered": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200", - "bench:burst": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3", + "bench:staggered": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3", + "bench:burst": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3", "bench:sandbox:dax": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/dax.bench.ts", - "bench:blaxel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider blaxel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider blaxel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider blaxel", - "bench:codesandbox": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider codesandbox && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider codesandbox && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider codesandbox", - "bench:daytona": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider daytona && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider daytona && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider daytona", - "bench:e2b": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider e2b && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider e2b && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider e2b", - "bench:hopx": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider hopx && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider hopx && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider hopx", - "bench:isorun": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider isorun && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider isorun && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider isorun", - "bench:modal": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider modal && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider modal && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider modal", - "bench:namespace": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider namespace && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider namespace && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider namespace", - "bench:northflank": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider northflank && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider northflank && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider northflank", - "bench:railway": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider railway && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider railway && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider railway", - "bench:render": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider render && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider render && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider render", - "bench:runloop": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider runloop && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider runloop && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider runloop", - "bench:vercel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider vercel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider vercel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider vercel", - "bench:just-bash": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider just-bash && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider just-bash && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider just-bash", - "bench:sprites": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sprites && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider sprites && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider sprites", - "bench:sandbox0": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sandbox0 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider sandbox0 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider sandbox0", - "bench:sail": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sail && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider sail && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider sail", - "bench:opencomputer": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider opencomputer && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider opencomputer && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider opencomputer", - "bench:tensorlake": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider tensorlake && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-staggered-local --name 'Sandbox staggered TTI (local)' --iterations 3 --concurrency 3 --stagger-delay-ms 200 --provider tensorlake && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --slug sandbox-burst-local --name 'Sandbox burst TTI (local)' --iterations 3 --concurrency 3 --provider tensorlake", + "bench:blaxel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider blaxel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider blaxel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider blaxel", + "bench:codesandbox": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider codesandbox && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider codesandbox && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider codesandbox", + "bench:daytona": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider daytona && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider daytona && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider daytona", + "bench:e2b": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider e2b && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider e2b && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider e2b", + "bench:hopx": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider hopx && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider hopx && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider hopx", + "bench:isorun": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider isorun && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider isorun && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider isorun", + "bench:modal": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider modal && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider modal && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider modal", + "bench:namespace": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider namespace && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider namespace && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider namespace", + "bench:northflank": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider northflank && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider northflank && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider northflank", + "bench:railway": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider railway && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider railway && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider railway", + "bench:render": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider render && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider render && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider render", + "bench:runloop": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider runloop && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider runloop && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider runloop", + "bench:vercel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider vercel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider vercel && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider vercel", + "bench:just-bash": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider just-bash && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider just-bash && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider just-bash", + "bench:sprites": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sprites && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider sprites && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider sprites", + "bench:sandbox0": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sandbox0 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider sandbox0 && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider sandbox0", + "bench:sail": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider sail && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider sail && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider sail", + "bench:opencomputer": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider opencomputer && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider opencomputer && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider opencomputer", + "bench:tensorlake": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --provider tensorlake && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape staggered --iterations 3 --concurrency 3 --provider tensorlake && tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --shape burst --iterations 3 --concurrency 3 --provider tensorlake", "bench:browser": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts", "bench:browser:browserbase": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts --provider browserbase", "bench:browser:hyperbrowser": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts --provider hyperbrowser", diff --git a/packages/benchsdk-runner/src/__tests__/bench-config.test.ts b/packages/benchsdk-runner/src/__tests__/bench-config.test.ts index d390074b..da7df4c4 100644 --- a/packages/benchsdk-runner/src/__tests__/bench-config.test.ts +++ b/packages/benchsdk-runner/src/__tests__/bench-config.test.ts @@ -85,6 +85,31 @@ describe('defineBenchmarkConfig', () => { defineBenchmarkConfig({ benchmarkSlug: 's', benchmarkName: 'n', participants, phases: [{ name: 'cold', iterations: 1 }, { name: 'cold', iterations: 1 }] }), ).toThrow('duplicate phase name'); }); + + it('accepts a valid shapes map', () => { + const config = defineBenchmarkConfig({ + benchmarkSlug: 's', + benchmarkName: 'n', + participants, + shapes: { + burst: { slug: 'sandbox-burst-local', name: 'Burst' }, + staggered: { slug: 'sandbox-staggered-local', staggerDelayMs: 200 }, + }, + }); + expect(config.shapes?.burst.slug).toBe('sandbox-burst-local'); + }); + + it('rejects a shape without a lowercase slug', () => { + expect(() => + defineBenchmarkConfig({ benchmarkSlug: 's', benchmarkName: 'n', participants, shapes: { burst: { slug: 'Burst' } } }), + ).toThrow('burst'); + }); + + it('rejects a shape with a negative staggerDelayMs', () => { + expect(() => + defineBenchmarkConfig({ benchmarkSlug: 's', benchmarkName: 'n', participants, shapes: { s: { slug: 'ok', staggerDelayMs: -1 } } }), + ).toThrow('staggerDelayMs'); + }); }); describe('defineTask', () => { diff --git a/packages/benchsdk-runner/src/__tests__/cli.test.ts b/packages/benchsdk-runner/src/__tests__/cli.test.ts index 85faabab..922a9fcf 100644 --- a/packages/benchsdk-runner/src/__tests__/cli.test.ts +++ b/packages/benchsdk-runner/src/__tests__/cli.test.ts @@ -5,13 +5,19 @@ import { NoAvailableParticipantsError } from '../no-available-participants.js'; const fixture = (name: string) => `src/__tests__/fixtures/${name}`; describe('runBenchmarkFile', () => { + it('rejects retired imperative commands (there is no `create`)', async () => { + await expect(runBenchmarkFile(['create', 'benchmark', 'sandbox'])).rejects.toThrow(/Usage:/); + await expect(runBenchmarkFile(['create', 'run'])).rejects.toThrow(/Usage:/); + }); + it('rejects when the command is not `run`', async () => { - await expect(runBenchmarkFile([])).rejects.toThrow(/Usage: bench run/); - await expect(runBenchmarkFile(['nope', fixture('good.bench.ts')])).rejects.toThrow(/Usage: bench run/); + await expect(runBenchmarkFile([])).rejects.toThrow(/Usage:/); + await expect(runBenchmarkFile(['nope', fixture('good.bench.ts')])).rejects.toThrow(/Usage:/); }); - it('rejects when no file is given', async () => { - await expect(runBenchmarkFile(['run'])).rejects.toThrow(/Usage: bench run/); + it('rejects when no file is given, or a flag stands where the file should', async () => { + await expect(runBenchmarkFile(['run'])).rejects.toThrow(/Usage:/); + await expect(runBenchmarkFile(['run', '--shape', 'burst'])).rejects.toThrow(/Usage:/); }); it('rejects a module that does not export a config', async () => { diff --git a/packages/benchsdk-runner/src/__tests__/runner.test.ts b/packages/benchsdk-runner/src/__tests__/runner.test.ts index d6c0db22..7edffce3 100644 --- a/packages/benchsdk-runner/src/__tests__/runner.test.ts +++ b/packages/benchsdk-runner/src/__tests__/runner.test.ts @@ -49,15 +49,23 @@ describe('parseCliArgs', () => { expect(parseCliArgs(['--group-by=participant'])).toEqual({ groupBy: 'participant' }); }); - it('parses --slug and --name', () => { - expect(parseCliArgs(['--slug', 'sandbox-burst-local'])).toEqual({ slug: 'sandbox-burst-local' }); - expect(parseCliArgs(['--slug=sandbox-tti-local'])).toEqual({ slug: 'sandbox-tti-local' }); + it('parses --benchmark (and its --slug alias) and --name', () => { + expect(parseCliArgs(['--benchmark', 'sandbox-burst-local'])).toEqual({ benchmark: 'sandbox-burst-local' }); + expect(parseCliArgs(['--benchmark=sandbox-tti-local'])).toEqual({ benchmark: 'sandbox-tti-local' }); + expect(parseCliArgs(['--slug', 'sandbox-burst-local'])).toEqual({ benchmark: 'sandbox-burst-local' }); expect(parseCliArgs(['--name', 'Sandbox burst TTI'])).toEqual({ name: 'Sandbox burst TTI' }); expect(() => parseCliArgs(['--name', ' '])).toThrow('--name'); }); - it('throws on a non-slug --slug', () => { - expect(() => parseCliArgs(['--slug', 'Sandbox TTI'])).toThrow('--slug'); + it('parses --shape and --run-key', () => { + expect(parseCliArgs(['--shape', 'burst'])).toEqual({ shape: 'burst' }); + expect(parseCliArgs(['--run-key=ci-123'])).toEqual({ runKey: 'ci-123' }); + expect(() => parseCliArgs(['--shape', ''])).toThrow('--shape'); + expect(() => parseCliArgs(['--run-key', ''])).toThrow('--run-key'); + }); + + it('throws on a non-slug --benchmark', () => { + expect(() => parseCliArgs(['--benchmark', 'Sandbox TTI'])).toThrow('--benchmark'); expect(() => parseCliArgs(['--slug', ''])).toThrow('--slug'); }); @@ -164,14 +172,19 @@ describe('runBenchmark', () => { process.env.E2B_API_KEY = 'x'; process.env.MODAL_TOKEN = 'y'; process.env.BENCHMARKS_PLATFORM_API_KEY = 'test-key'; - calls = { upsertBenchmark: [], createRun: [], planWorkers: [], runWorker: [], taskData: [] }; + calls = { upsertBenchmark: [], createRun: [], planWorkers: [], upsertParticipant: [], getRun: [], runWorker: [], taskData: [] }; fakeClient = { upsertBenchmark: vi.fn(async (...a: any[]) => { calls.upsertBenchmark.push(a); return {}; }), createRun: vi.fn(async (...a: any[]) => { calls.createRun.push(a); return { run: { id: 'run-1' }, participants: [] }; }), planWorkers: vi.fn(async (...a: any[]) => { calls.planWorkers.push(a); return []; }), + upsertParticipant: vi.fn(async (...a: any[]) => { calls.upsertParticipant.push(a); return {}; }), + getRun: vi.fn(async (slug: string, runId: string) => { + calls.getRun.push([slug, runId]); + return { id: runId, totalTasks: 3, participantSized: runId === 'run-open' }; + }), runWorker: vi.fn(async (opts: any) => { calls.runWorker.push(opts); - const total = calls.createRun[0]?.[1]?.totalTasks ?? 1; + const total = calls.createRun[0]?.[1]?.totalTasks ?? calls.upsertParticipant[0]?.[3]?.totalTasks ?? 1; // The platform hands out globally-indexed task ranges; `taskRangeStart` // lets a test exercise a worker whose range doesn't start at 0. const start = taskRangeStart; @@ -294,6 +307,84 @@ describe('runBenchmark', () => { expect(calls.createRun[0][0]).toBe('sandbox-burst-local'); }); + it('shares a run by --run-key: get-or-creates it keyed, then registers only its own participant', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + iterations: 2, + participants: [participants[0]], + }; + + const outcome = await runBenchmark(config, defineTask(async () => ({})), ['--run-key', 'ci-123', '--provider', 'e2b']); + + // The benchmark is still materialized from the file identity. + expect(calls.upsertBenchmark[0][0]).toBe('sandbox-tti-local'); + // One keyed get-or-create, carrying the key and no size/participant list. + expect(calls.createRun).toHaveLength(1); + expect(calls.createRun[0][1]).toMatchObject({ runKey: 'ci-123' }); + expect(calls.createRun[0][1].totalTasks).toBeUndefined(); + expect(calls.createRun[0][1].participants).toBeUndefined(); + // Registers only the provider it runs, sized to its own iteration count. + expect(calls.upsertParticipant[0].slice(0, 3)).toEqual(['sandbox-tti-local', 'run-1', 'e2b']); + expect(calls.upsertParticipant[0][3]).toMatchObject({ totalTasks: 2 }); + expect(calls.runWorker[0]).toMatchObject({ runId: 'run-1' }); + expect(outcome.runId).toBe('run-1'); + expect(outcome.participants[0].records).toHaveLength(2); + }); + + it('does not rename a benchmark it was merely retargeted at', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + iterations: 1, + participants: [participants[0]], + }; + + await runBenchmark(config, defineTask(async () => ({})), ['--benchmark', 'sandbox-burst-local']); + expect(calls.upsertBenchmark).toEqual([]); + + await runBenchmark(config, defineTask(async () => ({})), [ + '--benchmark', + 'sandbox-burst-local', + '--name', + 'Sandbox burst TTI', + ]); + expect(calls.upsertBenchmark[0]).toEqual(['sandbox-burst-local', { name: 'Sandbox burst TTI' }]); + }); + + it('selects a declared shape by --shape, reporting under its slug and name', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + iterations: 1, + participants: [participants[0]], + shapes: { + staggered: { slug: 'sandbox-staggered-local', name: 'Sandbox staggered TTI', staggerDelayMs: 200 }, + }, + }; + + const outcome = await runBenchmark(config, defineTask(async () => ({})), ['--shape', 'staggered']); + + expect(calls.upsertBenchmark[0][0]).toBe('sandbox-staggered-local'); + expect(calls.upsertBenchmark[0][1]).toMatchObject({ name: 'Sandbox staggered TTI' }); + expect(calls.createRun[0][0]).toBe('sandbox-staggered-local'); + // The shape's stable knob applies; scale knobs stay defaulted/overridable. + expect(outcome.config.staggerDelayMs).toBe(200); + }); + + it('rejects an unknown --shape, listing the declared ones', async () => { + const config: BenchmarkConfig = { + benchmarkSlug: 'sandbox-tti-local', + benchmarkName: 'Sandbox TTI', + participants: [participants[0]], + shapes: { burst: { slug: 'sandbox-burst-local' } }, + }; + + await expect( + runBenchmark(config, defineTask(async () => ({})), ['--shape', 'nope']), + ).rejects.toThrow('Known shapes: burst'); + }); + it('throws NoAvailableParticipantsError, listing the skips, when no participant has its env vars set', async () => { delete process.env.E2B_API_KEY; delete process.env.MODAL_TOKEN; diff --git a/packages/benchsdk-runner/src/bench-config.ts b/packages/benchsdk-runner/src/bench-config.ts index b27558ec..c1952ab2 100644 --- a/packages/benchsdk-runner/src/bench-config.ts +++ b/packages/benchsdk-runner/src/bench-config.ts @@ -19,6 +19,10 @@ * burst { iterations: N, concurrency: N } * staggered { iterations: N, concurrency: N, staggerDelayMs: 200 } * + * A benchmark can name these variants up front via `shapes`, so one file backs + * several platform benchmarks (`bench run --shape burst`) without + * restating each one's slug/name in scripts and CI. + * * A task is comprised of steps, declared via `ctx.step` inside a task function * — it supports closures, conditionals and try/finally, so values (a created * sandbox, say) flow naturally between steps. A task that declares no steps is @@ -37,6 +41,25 @@ import type { /** How tasks are ordered across participants. */ export type GroupBy = 'participant' | 'round'; +/** + * A named variant of a benchmark, selected with `--shape `. A shape + * carries only the parts that make it a distinct *benchmark* — its platform + * identity plus any stable distinguishing knob (e.g. staggered's delay). The + * scale knobs that vary per environment (`--iterations`, `--concurrency`) stay + * on the invocation, so a shape never sets a value only to have the CLI + * override it. + */ +export interface BenchmarkShape { + /** Platform slug this shape reports under (e.g. 'sandbox-burst-local'). */ + slug: string; + /** Display name shown on the platform; defaults to the slug. */ + name?: string; + /** Benchmark kind; defaults to the config's `benchmarkKind`. */ + kind?: string; + /** Default stagger delay (ms) for this shape; overridable with `--stagger-delay-ms`. */ + staggerDelayMs?: number; +} + /** * What a task returns: whatever it measured itself. This replaces the * assumption that the framework owns all timing. A plain data payload is @@ -132,6 +155,7 @@ export interface ResolvedRunConfig { */ export interface BenchmarkRunOutcome { runId: string; + /** Link to this run on the platform dashboard. */ dashboardUrl: string; participants: ParticipantRecords[]; config: ResolvedRunConfig; @@ -146,14 +170,22 @@ export interface BenchmarkRunOutcome { export interface BenchmarkConfig { /** * Stable platform slug for this benchmark (e.g. 'sandbox-tti-local'). - * Overridable per run with `--slug`, so one entrypoint can report under - * several benchmarks. + * Selectable per run with `--shape` (or overridable with `--benchmark`), so + * one entrypoint can report under several benchmarks. */ benchmarkSlug: string; /** Human-readable name shown on the platform. Overridable with `--name`. */ benchmarkName: string; /** Optional platform benchmark kind (e.g. 'sandbox'). */ benchmarkKind?: string; + /** + * Named variants of this benchmark, selected with `--shape `. Each + * shape swaps in its own platform identity (and optional stable knob) while + * reusing the same task and participants, so one bench file can back several + * platform benchmarks without duplicating the slug/name triple across + * package scripts and CI. + */ + shapes?: Record; /** * Total tasks to run per participant. Default: 1. Mutually exclusive with * `phases` — when `phases` is set, total iterations = sum of phase iterations. @@ -234,6 +266,19 @@ export function defineBenchmarkConfig= 0 (got ${shape.staggerDelayMs})`); + } + } + } return config; } diff --git a/packages/benchsdk-runner/src/cli.ts b/packages/benchsdk-runner/src/cli.ts index c76b4587..71089261 100644 --- a/packages/benchsdk-runner/src/cli.ts +++ b/packages/benchsdk-runner/src/cli.ts @@ -1,23 +1,31 @@ /** - * `bench run [--flags]` — the author-facing entrypoint. Imports a - * benchmark module, reads its `config` and `task` exports, and drives the run - * via the internal `runBenchmark`. CLI flags override the config's knobs, and - * `config.onComplete` (if any) fires once the run finishes. + * The author-facing entrypoint. `bench` is verbs-only — the benchmark and its + * runs are implicit, never nouns you type: + * + * bench run [--flags] execute a benchmark + * + * `run` imports a benchmark module, reads its `config` and `task` exports and + * drives `runBenchmark`; CLI flags override the config's knobs and + * `config.onComplete` (if any) fires once the run finishes. The benchmark is + * declared in the file (`--shape` picks a named variant) and materialized on + * run; a run is opened as a side effect, shared across sibling processes when + * they pass the same `--run-key`. There are no imperative `create` commands. * * The executable wrapper lives in `bin.ts`; this module has no side effects so * it can be unit-tested by calling `runBenchmarkFile` directly. */ import { resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { runBenchmark } from './runner.js'; +import { parseCliArgs, runBenchmark } from './runner.js'; import { NoAvailableParticipantsError } from './no-available-participants.js'; import type { BaseParticipant } from '@benchsdk/client'; import type { BenchmarkConfig, BenchmarkTask } from './bench-config.js'; const USAGE = - 'Usage: bench run [--iterations N] [--concurrency N] ' + - '[--stagger-delay-ms N] [--group-by participant|round] [--provider a,b] ' + - "[--slug my-benchmark] [--name 'My benchmark']"; + 'Usage:\n' + + ' bench run [--shape name] [--provider a,b] [--run-key key]\n' + + ' [--benchmark slug] [--name "My benchmark"]\n' + + ' [--iterations N] [--concurrency N] [--stagger-delay-ms N] [--group-by participant|round]'; /** A benchmark module is expected to export `config` and `task`. */ interface BenchmarkModule { @@ -33,19 +41,17 @@ function isBenchmarkConfig(value: unknown): value is BenchmarkConfig { } /** - * Loads a benchmark file and runs it. Throws on bad usage / invalid exports and - * lets `NoAvailableParticipantsError` propagate so the caller can map it to a - * clean exit. Does not call `process.exit`. + * Dispatches one CLI invocation. Throws on bad usage / invalid exports and lets + * `NoAvailableParticipantsError` propagate so the caller can map it to a clean + * exit. Does not call `process.exit`. */ export async function runBenchmarkFile(argv: string[]): Promise { - const [command, file, ...rest] = argv; - if (command !== 'run' || !file) { - throw new Error(USAGE); - } + const [command, ...rest] = argv; - const moduleUrl = pathToFileURL(resolve(process.cwd(), file)).href; - const mod = (await import(moduleUrl)) as BenchmarkModule; + const [file, ...flags] = rest; + if (command !== 'run' || !file || file.startsWith('-')) throw new Error(USAGE); + const mod = (await import(pathToFileURL(resolve(process.cwd(), file)).href)) as BenchmarkModule; const config = mod.config; const task = mod.task ?? mod.default; @@ -56,7 +62,7 @@ export async function runBenchmarkFile(argv: string[]): Promise { throw new Error(`${file} must export a \`task\` created with defineTask.`); } - await runBenchmark(config as BenchmarkConfig, task as BenchmarkTask, rest); + await runBenchmark(config as BenchmarkConfig, task as BenchmarkTask, flags); } /** Executable entry: runs the file and maps outcomes to process exit codes. */ diff --git a/packages/benchsdk-runner/src/runner.ts b/packages/benchsdk-runner/src/runner.ts index 3ce25d3a..a4be4ec8 100644 --- a/packages/benchsdk-runner/src/runner.ts +++ b/packages/benchsdk-runner/src/runner.ts @@ -24,7 +24,6 @@ import { NoAvailableParticipantsError } from './no-available-participants.js'; import type { BaseParticipant, BenchmarkClient, - BenchmarkRun, JsonObject, RunWorkerContext, TaskResultRecord, @@ -34,6 +33,7 @@ import { TaskError } from './bench-config.js'; import type { BenchmarkConfig, BenchmarkRunOutcome, + BenchmarkShape, BenchmarkTask, GroupBy, ParticipantRecords, @@ -44,8 +44,16 @@ import type { import { LogBuffer } from './log-buffer.js'; export interface CliArgs { - slug?: string; + /** Which platform benchmark to report as (`--benchmark`, aka the benchmark slug). */ + benchmark?: string; name?: string; + /** Named variant from the bench file's `shapes` (`--shape`), swapping in its identity. */ + shape?: string; + /** + * Idempotency key (`--run-key`): sibling processes passing the same key share + * one run (get-or-created), instead of each opening its own. + */ + runKey?: string; iterations?: number; concurrency?: number; staggerDelayMs?: number; @@ -98,12 +106,14 @@ export function parseCliArgs(argv: string[]): CliArgs { const arg = argv[i]; const name = arg.includes('=') ? arg.slice(0, arg.indexOf('=')) : arg; switch (name) { - case '--slug': { + // `--slug` is the pre-`--benchmark` spelling, kept working for existing scripts. + case '--slug': + case '--benchmark': { const { value, nextIndex } = readValue(arg, i); if (!/^[a-z0-9][a-z0-9-]*$/.test(value)) { - throw new Error(`--slug expects a lowercase slug (got "${value}")`); + throw new Error(`${name} expects a lowercase benchmark slug (got "${value}")`); } - args.slug = value; + args.benchmark = value; i = nextIndex; break; } @@ -114,6 +124,20 @@ export function parseCliArgs(argv: string[]): CliArgs { i = nextIndex; break; } + case '--shape': { + const { value, nextIndex } = readValue(arg, i); + if (value.trim() === '') throw new Error('--shape expects a value'); + args.shape = value; + i = nextIndex; + break; + } + case '--run-key': { + const { value, nextIndex } = readValue(arg, i); + if (value.trim() === '') throw new Error('--run-key expects a value'); + args.runKey = value; + i = nextIndex; + break; + } case '--iterations': { const { value, nextIndex } = readValue(arg, i); args.iterations = intFlag(value, '--iterations'); @@ -234,40 +258,99 @@ function resolvePlatform(): { baseUrl: string; apiKey: string } { } /** - * Runs `config`'s `task` against `participants`. Selects participants by - * `--provider` (if given), env-gates them, then drives them per the resolved - * `groupBy`. `--slug`/`--name` retarget the whole run at a different platform - * benchmark, so one entrypoint can report under several slugs. + * Resolves `--shape ` against the config's declared `shapes`. Throws with + * the known names if the shape is unknown, so a typo fails loudly instead of + * silently running the base benchmark. */ -export async function runBenchmark( +function resolveShape( + config: BenchmarkConfig, + shapeName: string | undefined, +): BenchmarkShape | undefined { + if (!shapeName) return undefined; + const shape = config.shapes?.[shapeName]; + if (!shape) { + const known = Object.keys(config.shapes ?? {}); + throw new Error( + known.length > 0 + ? `Unknown --shape "${shapeName}". Known shapes: ${known.join(', ')}.` + : `Unknown --shape "${shapeName}": this benchmark declares no shapes.`, + ); + } + return shape; +} + +/** + * Swaps a shape's identity (and its stable knob) into the config. Only the + * parts that make it a distinct benchmark move here; scale knobs stay on the + * CLI, so `mergeConfig` still lets `--concurrency`/`--iterations` win. + */ +function applyShape( + config: BenchmarkConfig, + shape: BenchmarkShape | undefined, +): BenchmarkConfig { + if (!shape) return config; + const kind = shape.kind ?? config.benchmarkKind; + return { + ...config, + benchmarkSlug: shape.slug, + benchmarkName: shape.name ?? shape.slug, + ...(kind ? { benchmarkKind: kind } : {}), + ...(shape.staggerDelayMs !== undefined ? { staggerDelayMs: shape.staggerDelayMs } : {}), + }; +} + +/** Applies the `--benchmark`/`--name` overrides, so one entrypoint can report under several benchmarks. */ +function applyIdentityOverrides( fileConfig: BenchmarkConfig, - task: BenchmarkTask, - argv: string[] = [], -): Promise { - const args = parseCliArgs(argv); - const config = { + args: CliArgs, +): BenchmarkConfig { + return { ...fileConfig, - ...(args.slug ? { benchmarkSlug: args.slug } : {}), + ...(args.benchmark ? { benchmarkSlug: args.benchmark } : {}), ...(args.name ? { benchmarkName: args.name } : {}), }; - const resolved = mergeConfig(config, args); - const schedule = buildSchedule(config, resolved.iterations, task); - const totalTasks = schedule.length; +} - const selected = selectParticipants(config.participants, resolved.providers); - const { available, skipped } = filterParticipantsByEnv(selected); +function dashboardUrlFor(baseUrl: string, organizationSlug: string, benchmarkSlug: string, runId: string): string { + return `${baseUrl.replace(/\/api\/v1\/?$/, '')}/${organizationSlug}/benchmarks/${benchmarkSlug}/runs/${runId}`; +} +/** The participants a run covers: `--provider` selection, minus any whose env vars are unset. */ +function resolveParticipants(config: BenchmarkConfig, resolved: ResolvedRunConfig): T[] { + const { available, skipped } = filterParticipantsByEnv(selectParticipants(config.participants, resolved.providers)); for (const s of skipped) { console.log(`Skipping ${s.name}: missing ${s.missing.join(', ')}`); } + if (available.length === 0) throw new NoAvailableParticipantsError(skipped); + return available; +} - if (available.length === 0) { - throw new NoAvailableParticipantsError(skipped); - } +/** + * Runs `config`'s `task` against its participants. Selects participants by + * `--provider` (if given), env-gates them, then drives them per the resolved + * `groupBy`. `--shape` swaps in a declared variant's identity; `--benchmark`/ + * `--name` retarget the run at a different platform benchmark, so one entrypoint + * can report under several slugs. With `--run-key`, sibling processes (e.g. one + * CI job per provider) get-or-create one shared run and each registers only its + * own participants. + */ +export async function runBenchmark( + fileConfig: BenchmarkConfig, + task: BenchmarkTask, + argv: string[] = [], +): Promise { + const args = parseCliArgs(argv); + const shaped = applyShape(fileConfig, resolveShape(fileConfig, args.shape)); + const config = applyIdentityOverrides(shaped, args); + const resolved = mergeConfig(config, args); + const available = resolveParticipants(config, resolved); const { baseUrl, apiKey } = resolvePlatform(); const client = createBenchmarkClient({ baseUrl, apiKey }); + const schedule = buildSchedule(config, resolved.iterations, task); + const totalTasks = schedule.length; + const concurrencyLabel = resolved.groupBy === 'round' ? 'n/a (round mode)' : String(resolved.concurrency); console.log(`${config.benchmarkName} (self-contained)`); console.log(`Date: ${new Date().toISOString()}`); @@ -276,34 +359,65 @@ export async function runBenchmark( `staggerDelayMs=${resolved.staggerDelayMs}, groupBy=${resolved.groupBy}\n`, ); - await client.upsertBenchmark(config.benchmarkSlug, { - name: config.benchmarkName, - ...(config.benchmarkKind ? { kind: config.benchmarkKind } : {}), - }); - - const { run, organizationSlug } = await client.createRun(config.benchmarkSlug, { - name: `${config.benchmarkSlug} — ${totalTasks} iterations, concurrency ${resolved.concurrency}`, - totalTasks, - workerCount: 1, - participants: available.map((p) => p.name), - }); + // Declaratively materialize the benchmark from the file/shape identity, which + // is authoritative (its name lives in the file). A bare `--benchmark X` only + // *retargets* reporting at a benchmark this file doesn't name, so we don't + // upsert it — that would rename it to the file's own name. + const identityIsOurs = + args.shape !== undefined || + args.name !== undefined || + !args.benchmark || + args.benchmark === fileConfig.benchmarkSlug; + if (identityIsOurs) { + await client.upsertBenchmark(config.benchmarkSlug, { + name: config.benchmarkName, + ...(config.benchmarkKind ? { kind: config.benchmarkKind } : {}), + }); + } - const dashboardUrl = `${baseUrl.replace(/\/api\/v1\/?$/, '')}/${organizationSlug}/benchmarks/${config.benchmarkSlug}/runs/${run.id}`; - console.log(`Run created: ${run.id}`); - console.log(`View at: ${dashboardUrl}\n`); + let runId: string; + let dashboardUrl: string; + if (args.runKey) { + // Shared run: get-or-created by key, so sibling processes (one per provider) + // converge on one run. Opened participant-sized — register only the + // providers this process runs and let each sibling register its own, so the + // run lists exactly who's benchmarked and each brings its own task count. + const { run, organizationSlug } = await client.createRun(config.benchmarkSlug, { + name: config.benchmarkName, + runKey: args.runKey, + }); + runId = run.id; + dashboardUrl = dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run.id); + for (const participant of available) { + await client.upsertParticipant(config.benchmarkSlug, runId, participant.name, { totalTasks }); + } + console.log(`Shared run (key "${args.runKey}"): ${runId}`); + console.log(`View at: ${dashboardUrl}\n`); + } else { + const { run, organizationSlug } = await client.createRun(config.benchmarkSlug, { + name: `${config.benchmarkSlug} — ${totalTasks} iterations, concurrency ${resolved.concurrency}`, + totalTasks, + workerCount: 1, + participants: available.map((p) => p.name), + }); + runId = run.id; + dashboardUrl = dashboardUrlFor(baseUrl, organizationSlug, config.benchmarkSlug, run.id); + console.log(`Run created: ${runId}`); + console.log(`View at: ${dashboardUrl}\n`); + } const onResult = defaultOnResult; let participantRecords: ParticipantRecords[]; if (resolved.groupBy === 'round') { - participantRecords = await runGroupedByRound(config, schedule, available, resolved, client, run, baseUrl, apiKey, onResult); + participantRecords = await runGroupedByRound(config, schedule, available, resolved, client, runId, baseUrl, apiKey, onResult); } else { - participantRecords = await runGroupedByParticipant(config, schedule, available, resolved, client, run, onResult); + participantRecords = await runGroupedByParticipant(config, schedule, available, resolved, client, runId, onResult); } console.log(`All done. View at: ${dashboardUrl}`); const outcome: BenchmarkRunOutcome = { - runId: run.id, + runId, dashboardUrl, participants: participantRecords, config: resolved, @@ -325,7 +439,7 @@ async function runGroupedByParticipant( available: T[], resolved: ResolvedRunConfig, client: BenchmarkClient, - run: BenchmarkRun, + runId: string, onResult: OnResult, ): Promise { const participantRecords: ParticipantRecords[] = []; @@ -338,11 +452,11 @@ async function runGroupedByParticipant( // count can't inflate launch offsets: a task whose slot frees after its // scheduled launch time starts immediately instead of sleeping index*delay. let rampStartMs: number | undefined; - await client.planWorkers(config.benchmarkSlug, run.id, participant.name); + await client.planWorkers(config.benchmarkSlug, runId, participant.name); const result = await client.runWorker({ benchmarkSlug: config.benchmarkSlug, - runId: run.id, + runId: runId, participantSlug: participant.name, concurrency: resolved.concurrency, task: async (ctx: RunWorkerContext) => { @@ -372,7 +486,7 @@ async function runGroupedByParticipant( }); if (!result.assignment) { - console.error(` No pending worker to claim for run ${run.id} — it may already be fully claimed.`); + console.error(` No pending worker to claim for run ${runId} — it may already be fully claimed.`); participantRecords.push({ participant: participant.name, records: result.records ?? [] }); continue; } @@ -397,7 +511,7 @@ async function runGroupedByRound( available: T[], resolved: ResolvedRunConfig, client: BenchmarkClient, - run: BenchmarkRun, + runId: string, baseUrl: string, apiKey: string, onResult: OnResult, @@ -414,7 +528,7 @@ async function runGroupedByRound( // reads `targetConcurrency` as tasks-per-worker, so it must be the full // schedule length — otherwise only one task is planned and every record // past the first falls outside the worker's task range. - await client.planWorkers(config.benchmarkSlug, run.id, participant.name, { + await client.planWorkers(config.benchmarkSlug, runId, participant.name, { workerCount: 1, targetConcurrency: schedule.length, }); @@ -424,7 +538,7 @@ async function runGroupedByRound( baseUrl, apiKey, benchmarkSlug: config.benchmarkSlug, - runId: run.id, + runId: runId, participantSlug: participant.name, processKind: 'process', processKey: process.env.HOSTNAME ?? 'local', diff --git a/packages/benchsdk/src/types.ts b/packages/benchsdk/src/types.ts index 644400c5..df9b639c 100644 --- a/packages/benchsdk/src/types.ts +++ b/packages/benchsdk/src/types.ts @@ -28,7 +28,11 @@ export interface BenchmarkRun { benchmarkId: string; name?: string | null; status: BenchmarkRunStatus | string; + /** Idempotency key: runs created with the same key (per org + benchmark) are the same run. */ + runKey?: string | null; totalTasks: number; + /** The run declared no size: `totalTasks` is the sum of what its participants declare. */ + participantSized?: boolean; workerCount: number; config?: JsonObject; createdAt?: string; @@ -116,8 +120,14 @@ export interface UpdateBenchmarkInput { export interface CreateRunInput { name?: string; - totalTasks: number; - workerCount: number; + /** + * Idempotency key for get-or-create: sibling callers passing the same key + * (per org + benchmark) converge on one run instead of each opening its own. + */ + runKey?: string; + /** Omit to open a participant-sized run: each participant declares its own size when it registers. */ + totalTasks?: number; + workerCount?: number; participants?: string[]; config?: JsonObject; } From ddb38f79bcb1fb5a23fe393f91a11658c8e7a947 Mon Sep 17 00:00:00 2001 From: garrison Date: Wed, 5 Aug 2026 19:49:00 +0000 Subject: [PATCH 12/15] ci(git): share one platform run across provider matrix jobs via --run-key Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/git-benchmarks.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/git-benchmarks.yml b/.github/workflows/git-benchmarks.yml index 3b712010..38777692 100644 --- a/.github/workflows/git-benchmarks.yml +++ b/.github/workflows/git-benchmarks.yml @@ -66,9 +66,16 @@ jobs: ITERATIONS="${{ (github.event_name == 'schedule' && '50') || github.event.inputs.iterations || '10' }}" + # Every provider job passes the same --run-key, so all matrix jobs + # get-or-create one shared platform run and each registers only its + # own participant. The attempt is in the key so workflow re-runs open + # fresh runs instead of rejoining the completed ones. + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/git/git.bench.ts \ --provider ${{ matrix.provider }} \ - --iterations $ITERATIONS + --iterations $ITERATIONS \ + --run-key "$RUN_KEY" - name: Upload results if: always() uses: actions/upload-artifact@v4 From 608d69ad429e518104256392cb9cc1f5c7d52ef6 Mon Sep 17 00:00:00 2001 From: garrison Date: Wed, 5 Aug 2026 20:13:37 +0000 Subject: [PATCH 13/15] fix(git): measure pull in independent clone, emit skip markers, and drop repoUrl from results Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- benchmarks/git/git.bench.ts | 31 +++++++++++++++++------- benchmarks/git/legacy-results.ts | 41 +++++++++++++++++++++++++------- benchmarks/git/types.ts | 15 ++++++------ benchmarks/src/merge-results.ts | 6 +++-- 4 files changed, 68 insertions(+), 25 deletions(-) diff --git a/benchmarks/git/git.bench.ts b/benchmarks/git/git.bench.ts index 921828eb..e86fcd94 100644 --- a/benchmarks/git/git.bench.ts +++ b/benchmarks/git/git.bench.ts @@ -91,11 +91,12 @@ export const task = defineTask(async (ctx) => { const { repoUrl, writable } = resolveRepoConfig(participant); const token = participant.tokenEnvVar ? process.env[participant.tokenEnvVar] : undefined; const username = participant.tokenUsername ?? 'token'; - const useAuth = !!(token && writable); + const useAuth = !!(token && writable && repoUrl); const branch = `${participant.name}-${taskIndex}-${Date.now()}`; const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'bench-git-')); const workDir = path.join(tempDir, 'repo'); + const pullDir = path.join(tempDir, 'pull'); const askpassPath = path.join(tempDir, 'askpass.sh'); if (useAuth) { @@ -128,8 +129,15 @@ export const task = defineTask(async (ctx) => { cloneMs = performance.now() - cloneStart; if (!useAuth) { - measure({ cloneMs, repoUrl, branch, pushMs: 0, pullMs: 0, commitSha: '' }); - return { data: { cloneMs, pushMs: 0, pullMs: 0, repoUrl, branch, commitSha: '' } }; + const skippedData: Record = { + cloneMs, + branch, + commitSha: '', + pushSkipped: true, + pullSkipped: true, + }; + measure(skippedData as any); + return { data: skippedData as any }; } const defaultBranch = await runGit(['branch', '--show-current'], workDir) @@ -157,24 +165,31 @@ export const task = defineTask(async (ctx) => { pushMs = performance.now() - pushStart; pushSucceeded = true; - await runGit(['checkout', defaultBranch], workDir); + // Prepare a separate shallow clone so the pull actually fetches the new + // branch from the remote instead of finding all objects already local. + await fs.promises.mkdir(pullDir, { recursive: true }); + await runGit(['init'], pullDir); + await runGit(['remote', 'add', 'origin', repoUrl], pullDir); + await runGit(['fetch', '--depth', '1', 'origin', defaultBranch], pullDir); + await runGit(['checkout', '-b', defaultBranch, 'FETCH_HEAD'], pullDir); const pullStart = performance.now(); await step('pull', () => withTimeout( - runGit(['pull', '--ff-only', 'origin', branch], workDir), + runGit(['pull', '--ff-only', 'origin', branch], pullDir), timeout, 'Git pull timed out', ), ); pullMs = performance.now() - pullStart; - measure({ cloneMs, pushMs, pullMs, repoUrl, branch, commitSha }); - return { data: { cloneMs, pushMs, pullMs, repoUrl, branch, commitSha } }; + const authData: Record = { cloneMs, pushMs, pullMs, branch, commitSha }; + measure(authData as any); + return { data: authData as any }; } catch (err) { throw new TaskError(formatError(err), { code: 'GIT_WORKFLOW_ERROR', - data: { repoUrl, branch, cloneMs, pushMs, pullMs }, + data: { branch, cloneMs, pushMs, pullMs }, }); } finally { if (pushSucceeded && useAuth) { diff --git a/benchmarks/git/legacy-results.ts b/benchmarks/git/legacy-results.ts index fbb4873d..6830e115 100644 --- a/benchmarks/git/legacy-results.ts +++ b/benchmarks/git/legacy-results.ts @@ -17,26 +17,34 @@ export function recordsToGitResults(participants: ParticipantRecords[]): GitBenc const d = (r.data ?? {}) as JsonObject; const base: GitTimingResult = { cloneMs: num(d.cloneMs), - pushMs: num(d.pushMs), - pullMs: num(d.pullMs), - repoUrl: typeof d.repoUrl === 'string' ? d.repoUrl : '', branch: typeof d.branch === 'string' ? d.branch : '', commitSha: typeof d.commitSha === 'string' ? d.commitSha : undefined, }; + if (d.pushSkipped) { + base.pushSkipped = true; + } else if (typeof d.pushMs === 'number') { + base.pushMs = d.pushMs; + } + if (d.pullSkipped) { + base.pullSkipped = true; + } else if (typeof d.pullMs === 'number') { + base.pullMs = d.pullMs; + } return r.status === 'error' ? { ...base, error: r.errorCode ?? 'error' } : base; }); const successful = iterations.filter((i) => !i.error); + const successfulPush = successful.filter((i) => !i.pushSkipped && typeof i.pushMs === 'number'); + const successfulPull = successful.filter((i) => !i.pullSkipped && typeof i.pullMs === 'number'); const summary = { cloneMs: computeStats(successful.map((i) => i.cloneMs)), - pushMs: computeStats(successful.map((i) => i.pushMs)), - pullMs: computeStats(successful.map((i) => i.pullMs)), + pushMs: computeStats(successfulPush.map((i) => i.pushMs as number)), + pullMs: computeStats(successfulPull.map((i) => i.pullMs as number)), }; return { provider: participant.participant, mode: 'git' as const, - repoUrl: iterations[0]?.repoUrl ?? '', iterations, summary, successRate: iterations.length ? successful.length / iterations.length : 0, @@ -51,8 +59,25 @@ export async function writeGitResultsJson(results: GitBenchmarkResult[], outPath const clean = results.map((r) => ({ provider: r.provider, mode: r.mode, - repoUrl: r.repoUrl, - iterations: r.iterations, + iterations: r.iterations.map((i) => { + const entry: Record = { + cloneMs: i.cloneMs, + branch: i.branch, + }; + if (i.commitSha !== undefined) entry.commitSha = i.commitSha; + if (i.pushSkipped) { + entry.pushSkipped = true; + } else if (i.pushMs !== undefined) { + entry.pushMs = i.pushMs; + } + if (i.pullSkipped) { + entry.pullSkipped = true; + } else if (i.pullMs !== undefined) { + entry.pullMs = i.pullMs; + } + if (i.error !== undefined) entry.error = i.error; + return entry; + }), summary: r.summary, ...(r.successRate !== undefined ? { successRate: r.successRate } : {}), ...(r.compositeScore !== undefined ? { compositeScore: r.compositeScore } : {}), diff --git a/benchmarks/git/types.ts b/benchmarks/git/types.ts index f8cc05c4..d1a1f5e7 100644 --- a/benchmarks/git/types.ts +++ b/benchmarks/git/types.ts @@ -18,12 +18,14 @@ export interface GitProviderConfig extends BaseParticipant { export interface GitTimingResult { /** Time to shallow clone the repo in ms. */ cloneMs: number; - /** Time to commit and push the test branch in ms. */ - pushMs: number; - /** Time to pull the test branch into the default branch in ms. */ - pullMs: number; - /** Repository URL used. */ - repoUrl: string; + /** Time to commit and push the test branch in ms, when measured. */ + pushMs?: number; + /** Time to pull the test branch in ms, when measured. */ + pullMs?: number; + /** True when push was skipped (no writable repo/token). */ + pushSkipped?: boolean; + /** True when pull was skipped (no writable repo/token). */ + pullSkipped?: boolean; /** Test branch that was pushed/pulled. */ branch: string; /** Commit SHA produced by the benchmark push, when available. */ @@ -41,7 +43,6 @@ export interface GitStats { export interface GitBenchmarkResult { provider: string; mode: 'git'; - repoUrl: string; iterations: GitTimingResult[]; summary: GitStats; /** Composite weighted score (0-100, higher = better). Computed post-benchmark. */ diff --git a/benchmarks/src/merge-results.ts b/benchmarks/src/merge-results.ts index 7da01477..dbd1c054 100644 --- a/benchmarks/src/merge-results.ts +++ b/benchmarks/src/merge-results.ts @@ -762,8 +762,10 @@ function printGitResultsTable(results: GitBenchmarkResult[]): void { const ok = r.iterations.filter((i) => !i.error).length; const total = r.iterations.length; const clone = (r.summary.cloneMs.median / 1000).toFixed(2) + 's'; - const push = (r.summary.pushMs.median / 1000).toFixed(2) + 's'; - const pull = (r.summary.pullMs.median / 1000).toFixed(2) + 's'; + const pushAllSkipped = r.iterations.every((i) => i.pushSkipped); + const pullAllSkipped = r.iterations.every((i) => i.pullSkipped); + const push = pushAllSkipped ? '--' : (r.summary.pushMs.median / 1000).toFixed(2) + 's'; + const pull = pullAllSkipped ? '--' : (r.summary.pullMs.median / 1000).toFixed(2) + 's'; console.log([r.provider.padEnd(14), clone.padEnd(12), push.padEnd(12), pull.padEnd(12), `${ok}/${total}`.padEnd(10)].join(' | ')); } console.log('='.repeat(95)); From f202d5fe0764e3a56658c6ca09bc9b9ae04a4b08 Mon Sep 17 00:00:00 2001 From: garrison Date: Wed, 5 Aug 2026 21:32:37 +0000 Subject: [PATCH 14/15] fix(git): remove dead clone-only fallback, harden askpass, cleanup branches on partial push Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- benchmarks/git/git.bench.ts | 67 +++++++++++--------------------- benchmarks/git/legacy-results.ts | 30 +++----------- benchmarks/git/types.ts | 26 +++++-------- benchmarks/src/merge-results.ts | 6 +-- 4 files changed, 40 insertions(+), 89 deletions(-) diff --git a/benchmarks/git/git.bench.ts b/benchmarks/git/git.bench.ts index e86fcd94..410236a2 100644 --- a/benchmarks/git/git.bench.ts +++ b/benchmarks/git/git.bench.ts @@ -30,10 +30,9 @@ const CLONE_TIMEOUT_MS = 60_000; const COMMITTER_NAME = 'ComputeSDK Benchmark'; const COMMITTER_EMAIL = 'bench@example.com'; -function resolveRepoConfig(config: GitProviderConfig): { repoUrl: string; writable: boolean } { - const override = config.repoUrlEnvVar ? process.env[config.repoUrlEnvVar] : undefined; - const repoUrl = override ? sanitizeRepoUrl(override) : sanitizeRepoUrl(config.url ?? ''); - return { repoUrl, writable: !!override }; +function resolveRepoConfig(config: GitProviderConfig): { repoUrl: string } { + const repoUrl = config.repoUrlEnvVar ? sanitizeRepoUrl(process.env[config.repoUrlEnvVar] ?? '') : ''; + return { repoUrl }; } function sanitizeRepoUrl(repoUrl: string): string { @@ -47,30 +46,23 @@ function sanitizeRepoUrl(repoUrl: string): string { } } -function buildGitEnv( - useAuth: boolean, - username: string, - token: string, - askpassPath?: string, -): NodeJS.ProcessEnv { - const env: NodeJS.ProcessEnv = { ...process.env, GIT_TERMINAL_PROMPT: '0' }; - if (useAuth && askpassPath) { - env.GIT_ASKPASS = askpassPath; - env.GIT_BENCH_USER = username; - env.GIT_BENCH_PASS = token; - } - return env; +function escapeShell(s: string): string { + return s.replace(/'/g, `'\\''`); +} + +function buildGitEnv(askpassPath: string): NodeJS.ProcessEnv { + return { ...process.env, GIT_ASKPASS: askpassPath, GIT_TERMINAL_PROMPT: '0' }; } -async function writeAskpassScript(askpassPath: string): Promise { +async function writeAskpassScript(askpassPath: string, username: string, token: string): Promise { const script = `#!/bin/sh case "$1" in - *Password*) printf '%s\\n' "$GIT_BENCH_PASS" ;; - *Username*) printf '%s\\n' "$GIT_BENCH_USER" ;; - *) printf '%s\\n' "$GIT_BENCH_USER" ;; + *Password*) printf '%s\\n' '${escapeShell(token)}' ;; + *Username*) printf '%s\\n' '${escapeShell(username)}' ;; + *) printf '%s\\n' '${escapeShell(username)}' ;; esac `; - await fs.promises.writeFile(askpassPath, script, { mode: 0o755 }); + await fs.promises.writeFile(askpassPath, script, { mode: 0o700 }); } export const config = defineBenchmarkConfig({ @@ -88,10 +80,9 @@ export const task = defineTask(async (ctx) => { const { participant, step, measure, taskIndex } = ctx; const timeout = participant.timeout ?? CLONE_TIMEOUT_MS; - const { repoUrl, writable } = resolveRepoConfig(participant); - const token = participant.tokenEnvVar ? process.env[participant.tokenEnvVar] : undefined; - const username = participant.tokenUsername ?? 'token'; - const useAuth = !!(token && writable && repoUrl); + const repoUrl = resolveRepoConfig(participant).repoUrl; + const token = process.env[participant.tokenEnvVar] ?? ''; + const username = participant.tokenUsername; const branch = `${participant.name}-${taskIndex}-${Date.now()}`; const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'bench-git-')); @@ -99,11 +90,9 @@ export const task = defineTask(async (ctx) => { const pullDir = path.join(tempDir, 'pull'); const askpassPath = path.join(tempDir, 'askpass.sh'); - if (useAuth) { - await writeAskpassScript(askpassPath); - } + await writeAskpassScript(askpassPath, username, token); - const env = buildGitEnv(useAuth, username, token ?? '', askpassPath); + const env = buildGitEnv(askpassPath); function runGit( args: string[], cwd: string, @@ -115,7 +104,7 @@ export const task = defineTask(async (ctx) => { let cloneMs = 0; let pushMs = 0; let pullMs = 0; - let pushSucceeded = false; + let pushAttempted = false; try { const cloneStart = performance.now(); @@ -128,18 +117,6 @@ export const task = defineTask(async (ctx) => { ); cloneMs = performance.now() - cloneStart; - if (!useAuth) { - const skippedData: Record = { - cloneMs, - branch, - commitSha: '', - pushSkipped: true, - pullSkipped: true, - }; - measure(skippedData as any); - return { data: skippedData as any }; - } - const defaultBranch = await runGit(['branch', '--show-current'], workDir) .then((r) => r.stdout.trim()) .catch(() => participant.defaultBranch ?? 'main'); @@ -154,6 +131,7 @@ export const task = defineTask(async (ctx) => { ); const commitSha = commitResult.stdout.match(/\[.+?\s+([a-f0-9]+)\]/)?.[1] ?? ''; + pushAttempted = true; const pushStart = performance.now(); await step('push', () => withTimeout( @@ -163,7 +141,6 @@ export const task = defineTask(async (ctx) => { ), ); pushMs = performance.now() - pushStart; - pushSucceeded = true; // Prepare a separate shallow clone so the pull actually fetches the new // branch from the remote instead of finding all objects already local. @@ -192,7 +169,7 @@ export const task = defineTask(async (ctx) => { data: { branch, cloneMs, pushMs, pullMs }, }); } finally { - if (pushSucceeded && useAuth) { + if (pushAttempted) { await runGit(['push', 'origin', '--delete', branch], workDir, timeout).catch(() => {}); } await fs.promises.rm(tempDir, { recursive: true, force: true }).catch(() => {}); diff --git a/benchmarks/git/legacy-results.ts b/benchmarks/git/legacy-results.ts index 6830e115..bbadd664 100644 --- a/benchmarks/git/legacy-results.ts +++ b/benchmarks/git/legacy-results.ts @@ -17,29 +17,19 @@ export function recordsToGitResults(participants: ParticipantRecords[]): GitBenc const d = (r.data ?? {}) as JsonObject; const base: GitTimingResult = { cloneMs: num(d.cloneMs), + pushMs: num(d.pushMs), + pullMs: num(d.pullMs), branch: typeof d.branch === 'string' ? d.branch : '', commitSha: typeof d.commitSha === 'string' ? d.commitSha : undefined, }; - if (d.pushSkipped) { - base.pushSkipped = true; - } else if (typeof d.pushMs === 'number') { - base.pushMs = d.pushMs; - } - if (d.pullSkipped) { - base.pullSkipped = true; - } else if (typeof d.pullMs === 'number') { - base.pullMs = d.pullMs; - } return r.status === 'error' ? { ...base, error: r.errorCode ?? 'error' } : base; }); const successful = iterations.filter((i) => !i.error); - const successfulPush = successful.filter((i) => !i.pushSkipped && typeof i.pushMs === 'number'); - const successfulPull = successful.filter((i) => !i.pullSkipped && typeof i.pullMs === 'number'); const summary = { cloneMs: computeStats(successful.map((i) => i.cloneMs)), - pushMs: computeStats(successfulPush.map((i) => i.pushMs as number)), - pullMs: computeStats(successfulPull.map((i) => i.pullMs as number)), + pushMs: computeStats(successful.map((i) => i.pushMs)), + pullMs: computeStats(successful.map((i) => i.pullMs)), }; return { @@ -62,19 +52,11 @@ export async function writeGitResultsJson(results: GitBenchmarkResult[], outPath iterations: r.iterations.map((i) => { const entry: Record = { cloneMs: i.cloneMs, + pushMs: i.pushMs, + pullMs: i.pullMs, branch: i.branch, }; if (i.commitSha !== undefined) entry.commitSha = i.commitSha; - if (i.pushSkipped) { - entry.pushSkipped = true; - } else if (i.pushMs !== undefined) { - entry.pushMs = i.pushMs; - } - if (i.pullSkipped) { - entry.pullSkipped = true; - } else if (i.pullMs !== undefined) { - entry.pullMs = i.pullMs; - } if (i.error !== undefined) entry.error = i.error; return entry; }), diff --git a/benchmarks/git/types.ts b/benchmarks/git/types.ts index d1a1f5e7..4d888538 100644 --- a/benchmarks/git/types.ts +++ b/benchmarks/git/types.ts @@ -1,14 +1,12 @@ import type { BaseParticipant } from '@benchsdk/client'; export interface GitProviderConfig extends BaseParticipant { - /** Optional fallback HTTPS URL of the repo to clone. */ - url?: string; - /** Env var that overrides `url` for the read/write repo. */ - repoUrlEnvVar?: string; - /** Optional env var holding an HTTPS auth token for push/pull. */ - tokenEnvVar?: string; - /** Username supplied to `git` via GIT_ASKPASS when a token is set. */ - tokenUsername?: string; + /** Env var holding the writable HTTPS repo URL. */ + repoUrlEnvVar: string; + /** Env var holding the HTTPS auth token for push/pull. */ + tokenEnvVar: string; + /** Username supplied to `git` via GIT_ASKPASS. */ + tokenUsername: string; /** Default branch to pull back into after pushing a test branch. */ defaultBranch?: string; /** Per-operation timeout in ms (default: 60000). */ @@ -18,14 +16,10 @@ export interface GitProviderConfig extends BaseParticipant { export interface GitTimingResult { /** Time to shallow clone the repo in ms. */ cloneMs: number; - /** Time to commit and push the test branch in ms, when measured. */ - pushMs?: number; - /** Time to pull the test branch in ms, when measured. */ - pullMs?: number; - /** True when push was skipped (no writable repo/token). */ - pushSkipped?: boolean; - /** True when pull was skipped (no writable repo/token). */ - pullSkipped?: boolean; + /** Time to commit and push the test branch in ms. */ + pushMs: number; + /** Time to pull the test branch in ms. */ + pullMs: number; /** Test branch that was pushed/pulled. */ branch: string; /** Commit SHA produced by the benchmark push, when available. */ diff --git a/benchmarks/src/merge-results.ts b/benchmarks/src/merge-results.ts index dbd1c054..7da01477 100644 --- a/benchmarks/src/merge-results.ts +++ b/benchmarks/src/merge-results.ts @@ -762,10 +762,8 @@ function printGitResultsTable(results: GitBenchmarkResult[]): void { const ok = r.iterations.filter((i) => !i.error).length; const total = r.iterations.length; const clone = (r.summary.cloneMs.median / 1000).toFixed(2) + 's'; - const pushAllSkipped = r.iterations.every((i) => i.pushSkipped); - const pullAllSkipped = r.iterations.every((i) => i.pullSkipped); - const push = pushAllSkipped ? '--' : (r.summary.pushMs.median / 1000).toFixed(2) + 's'; - const pull = pullAllSkipped ? '--' : (r.summary.pullMs.median / 1000).toFixed(2) + 's'; + const push = (r.summary.pushMs.median / 1000).toFixed(2) + 's'; + const pull = (r.summary.pullMs.median / 1000).toFixed(2) + 's'; console.log([r.provider.padEnd(14), clone.padEnd(12), push.padEnd(12), pull.padEnd(12), `${ok}/${total}`.padEnd(10)].join(' | ')); } console.log('='.repeat(95)); From 6d7168800b221cfb9ebda3212685848d81d17edb Mon Sep 17 00:00:00 2001 From: garrison Date: Wed, 5 Aug 2026 21:45:17 +0000 Subject: [PATCH 15/15] ci(git): scope workflow permissions and use frozen lockfile installs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/git-benchmarks.yml | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/.github/workflows/git-benchmarks.yml b/.github/workflows/git-benchmarks.yml index 38777692..d8088b8d 100644 --- a/.github/workflows/git-benchmarks.yml +++ b/.github/workflows/git-benchmarks.yml @@ -31,12 +31,10 @@ concurrency: group: git-benchmarks cancel-in-progress: true -permissions: - contents: write - pull-requests: write - jobs: bench: + permissions: + contents: read name: Bench ${{ matrix.provider }} runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe timeout-minutes: 60 @@ -52,12 +50,7 @@ jobs: node-version: 24 cache: 'pnpm' - name: Install dependencies - run: | - if [ "${{ github.event_name }}" = "schedule" ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi + run: pnpm install --frozen-lockfile - name: Clear stale results from checkout run: rm -rf results/git/ - name: Run git benchmark @@ -90,6 +83,8 @@ jobs: runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe needs: [bench] if: always() + permissions: + contents: write steps: - uses: actions/checkout@v4 with: @@ -100,12 +95,7 @@ jobs: node-version: 24 cache: 'pnpm' - name: Install dependencies - run: | - if [ "${{ github.event_name }}" = "schedule" ]; then - pnpm update - else - pnpm install --frozen-lockfile - fi + run: pnpm install --frozen-lockfile - name: Download all artifacts uses: actions/download-artifact@v4 with: