From 35e1501b9beb7c76e47b1645c33f2af50086031e Mon Sep 17 00:00:00 2001 From: garrison Date: Thu, 20 Aug 2026 13:57:17 +0000 Subject: [PATCH 1/4] Run reliability benchmark via tsx to support TypeScript source Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/reliability-tensorlake-opencode.yml | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reliability-tensorlake-opencode.yml b/.github/workflows/reliability-tensorlake-opencode.yml index 6dde26ee..ad464c79 100644 --- a/.github/workflows/reliability-tensorlake-opencode.yml +++ b/.github/workflows/reliability-tensorlake-opencode.yml @@ -57,7 +57,7 @@ jobs: else DRY_RUN_FLAG="" fi - pnpm exec bench run benchmarks/reliability/sandbox-opencode.bench.ts \ + pnpm exec tsx packages/benchsdk-runner/dist/bin.js run benchmarks/reliability/sandbox-opencode.bench.ts \ --provider tensorlake \ --iterations "$ITERATIONS" \ --stagger-delay-ms "$STAGGER_DELAY_MS" \ diff --git a/package.json b/package.json index c33caf5e..4911b1a9 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "bench:sail": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --iterations 3 --concurrency 3 --provider sail", "bench:opencomputer": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --iterations 3 --concurrency 3 --provider opencomputer", "bench:tensorlake": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --iterations 3 --concurrency 3 --provider tensorlake", - "bench:reliability:tensorlake:opencode": "bench run benchmarks/reliability/sandbox-opencode.bench.ts --provider tensorlake --iterations 3", + "bench:reliability:tensorlake:opencode": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/reliability/sandbox-opencode.bench.ts --provider tensorlake --iterations 3", "bench:browser": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts", "bench:browser:browserbase": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts --provider browserbase", "bench:browser:hyperbrowser": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts --provider hyperbrowser", From 06669e7e1328b25b69652830162adcd2bb235022 Mon Sep 17 00:00:00 2001 From: garrison Date: Thu, 20 Aug 2026 18:35:46 +0000 Subject: [PATCH 2/4] Make bench CLI load TypeScript benchmarks by registering tsx Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/reliability-tensorlake-opencode.yml | 2 +- package.json | 2 +- packages/benchsdk-runner/package.json | 3 ++- packages/benchsdk-runner/src/bin.ts | 2 ++ 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/reliability-tensorlake-opencode.yml b/.github/workflows/reliability-tensorlake-opencode.yml index ad464c79..6dde26ee 100644 --- a/.github/workflows/reliability-tensorlake-opencode.yml +++ b/.github/workflows/reliability-tensorlake-opencode.yml @@ -57,7 +57,7 @@ jobs: else DRY_RUN_FLAG="" fi - pnpm exec tsx packages/benchsdk-runner/dist/bin.js run benchmarks/reliability/sandbox-opencode.bench.ts \ + pnpm exec bench run benchmarks/reliability/sandbox-opencode.bench.ts \ --provider tensorlake \ --iterations "$ITERATIONS" \ --stagger-delay-ms "$STAGGER_DELAY_MS" \ diff --git a/package.json b/package.json index 4911b1a9..c33caf5e 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "bench:sail": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --iterations 3 --concurrency 3 --provider sail", "bench:opencomputer": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --iterations 3 --concurrency 3 --provider opencomputer", "bench:tensorlake": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --iterations 3 --concurrency 3 --provider tensorlake", - "bench:reliability:tensorlake:opencode": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/reliability/sandbox-opencode.bench.ts --provider tensorlake --iterations 3", + "bench:reliability:tensorlake:opencode": "bench run benchmarks/reliability/sandbox-opencode.bench.ts --provider tensorlake --iterations 3", "bench:browser": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts", "bench:browser:browserbase": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts --provider browserbase", "bench:browser:hyperbrowser": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts --provider hyperbrowser", diff --git a/packages/benchsdk-runner/package.json b/packages/benchsdk-runner/package.json index 686ba7af..566a6d0f 100644 --- a/packages/benchsdk-runner/package.json +++ b/packages/benchsdk-runner/package.json @@ -52,7 +52,8 @@ "node": ">=18.0.0" }, "dependencies": { - "@benchsdk/client": "workspace:*" + "@benchsdk/client": "workspace:*", + "tsx": "^4.23.12" }, "devDependencies": { "@types/node": "^20.0.0", diff --git a/packages/benchsdk-runner/src/bin.ts b/packages/benchsdk-runner/src/bin.ts index 0fbb6035..0f52f948 100644 --- a/packages/benchsdk-runner/src/bin.ts +++ b/packages/benchsdk-runner/src/bin.ts @@ -3,6 +3,8 @@ // benchmark module it loads share a single `@benchsdk/runner` instance — this // keeps `instanceof` checks (TaskError / NoAvailableParticipantsError) valid // across the bin and the dynamically imported `*.bench.ts`. +import { register } from 'tsx/esm/api'; import { run } from '@benchsdk/runner'; +register(); run(process.argv.slice(2)); From 797f76b930f5abe79036205523af0afc0bcb3bfb Mon Sep 17 00:00:00 2001 From: garrison Date: Thu, 20 Aug 2026 18:36:52 +0000 Subject: [PATCH 3/4] Update lockfile for tsx runtime dependency Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- pnpm-lock.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1cf85910..c062f845 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -258,6 +258,9 @@ importers: '@benchsdk/client': specifier: workspace:* version: link:../benchsdk + tsx: + specifier: ^4.23.12 + version: 4.23.12 devDependencies: '@types/node': specifier: ^20.0.0 From 09948809542cfb88ee70e1e02e3343b1f019dcea Mon Sep 17 00:00:00 2001 From: garrison Date: Thu, 20 Aug 2026 18:42:06 +0000 Subject: [PATCH 4/4] Use ESM build for bench binary to keep shared runner instance Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- packages/benchsdk-runner/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/benchsdk-runner/package.json b/packages/benchsdk-runner/package.json index 566a6d0f..e9a4170b 100644 --- a/packages/benchsdk-runner/package.json +++ b/packages/benchsdk-runner/package.json @@ -17,7 +17,7 @@ } }, "bin": { - "bench": "./dist/bin.cjs" + "bench": "./dist/bin.js" }, "files": [ "dist",