diff --git a/benchmarks/examples/00-hello-world.bench.ts b/benchmarks/examples/00-hello-world.bench.ts new file mode 100644 index 00000000..20239890 --- /dev/null +++ b/benchmarks/examples/00-hello-world.bench.ts @@ -0,0 +1,77 @@ +/** + * Hello-world benchmark. + * + * This is the smallest possible benchSDK benchmark. It demonstrates: + * - exporting a `config` created with `defineBenchmarkConfig` + * - exporting a `task` created with `defineTask` + * - a single participant + * - three named steps (create, exec, destroy) + * - attaching metrics with `measure()` + * - writing to the worker log artifact with `log()` + * + * Run with: + * bench run benchmarks/examples/00-hello-world.bench.ts --iterations 3 + * + * Add `--no-ingest` to run without a platform endpoint or API key. + */ +import '../src/env.js'; +import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner'; +import type { NoopParticipant } from './participants.js'; +import { createNoopParticipant } from './participants.js'; + +/** + * The benchmark config declares the platform identity and orchestration knobs. + * + * - `benchmarkSlug` is the URL-safe identifier used for the platform API and + * dashboard (`.../benchmarks/examples-hello-world/runs/...`). + * - `benchmarkName` is the human-readable name shown in the dashboard. + * - `iterations` is the total number of times the task runs per participant. + * - `concurrency` is the maximum number of tasks in flight at once for a single + * participant worker. + * - `participants` lists the providers to benchmark; the same task runs for each. + */ +export const config = defineBenchmarkConfig({ + benchmarkSlug: 'examples-hello-world', + benchmarkName: 'Examples: Hello World', + iterations: 3, + concurrency: 1, + participants: [createNoopParticipant('noop', 100)], +}); + +/** + * The task is the per-iteration workload. + * + * `defineTask` tells TypeScript the participant type so the + * task body can access `participant.createCompute()`. The context also gives + * `step`, `measure`, and `log`. + */ +export const task = defineTask(async ({ participant, step, measure, log }) => { + // `log` appends a free-form line to the worker log artifact. It is useful for + // narrating what the task is doing; the return value is not recorded. + log('starting hello-world iteration'); + + // `participant.createCompute()` is specific to the mock provider in this + // example; a real benchmark would call the provider SDK. + const compute = participant.createCompute(); + + // We capture our own start time because the runner's wall-clock timing + // includes the `destroy` step; for TTI we want create-through-first-command. + const start = performance.now(); + + // `step(name, fn)` runs `fn`, returns its value, and records a step record + // with timing, status, and any data measured inside it. + const sandbox = await step('create', () => compute.sandbox.create()); + try { + const result = await step('exec', () => sandbox.runCommand('node -v')); + + // `measure(data)` merges JSON into the currently active step, or into the + // task record if called outside a step. Here we attach the TTI and exit code + // to the task record's `data`. + measure({ ttiMs: performance.now() - start, exitCode: result.exitCode }); + } finally { + // `reportConcurrency: false` tells the worker not to count this step in its + // concurrency heartbeat samples; cleanup steps typically do not run while + // other tasks are also being launched. + await step('destroy', () => sandbox.destroy(), { reportConcurrency: false }); + } +}); diff --git a/benchmarks/examples/01-multiple-providers.bench.ts b/benchmarks/examples/01-multiple-providers.bench.ts new file mode 100644 index 00000000..62a53a30 --- /dev/null +++ b/benchmarks/examples/01-multiple-providers.bench.ts @@ -0,0 +1,59 @@ +/** + * Multi-provider benchmark. + * + * Demonstrates running the same task against several participants. The runner + * first filters out any participants whose `requiredEnvVars` are missing, then + * executes the task once per selected provider. + * + * Run with all providers: + * bench run benchmarks/examples/01-multiple-providers.bench.ts --iterations 5 --concurrency 2 + * + * Run with a subset: + * bench run benchmarks/examples/01-multiple-providers.bench.ts --iterations 5 --provider alpha,beta + * + * Add `--no-ingest` to run without a platform endpoint or API key. + */ +import '../src/env.js'; +import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner'; +import type { NoopParticipant } from './participants.js'; +import { exampleProviders } from './participants.js'; + +/** + * The config reuses `exampleProviders` from `participants.ts`. Each provider + * gets its own worker on the platform, and the same `task` is invoked for + * every iteration of every provider. + * + * - `iterations: 5` means each provider runs the task 5 times. + * - `concurrency: 2` means up to 2 of those tasks are in flight at once for a + * given provider's worker. + * - `--provider alpha,beta` would limit the run to just those two names. + */ +export const config = defineBenchmarkConfig({ + benchmarkSlug: 'examples-multiple-providers', + benchmarkName: 'Examples: Multiple Providers', + iterations: 5, + concurrency: 2, + participants: exampleProviders, +}); + +/** + * The task body is identical to the hello-world example, but `participant` + * changes per worker: the runner first completes all iterations for `alpha`, + * then `beta`, then `gamma` (default `groupBy: 'participant'`). + */ +export const task = defineTask(async ({ participant, step, measure, log }) => { + // Because the same task runs for every provider, logging the provider name + // makes the worker log easy to read. + log(`running on ${participant.name}`); + + const compute = participant.createCompute(); + const start = performance.now(); + + const sandbox = await step('create', () => compute.sandbox.create()); + try { + const result = await step('exec', () => sandbox.runCommand('node -v')); + measure({ ttiMs: performance.now() - start, exitCode: result.exitCode }); + } finally { + await step('destroy', () => sandbox.destroy(), { reportConcurrency: false }); + } +}); diff --git a/benchmarks/examples/02-phases.bench.ts b/benchmarks/examples/02-phases.bench.ts new file mode 100644 index 00000000..f3724ee6 --- /dev/null +++ b/benchmarks/examples/02-phases.bench.ts @@ -0,0 +1,68 @@ +/** + * Phases benchmark. + * + * Demonstrates `phases`: an ordered list of named segments, each with its own + * iteration count. The runner tags every task record with `data.phase` and the + * task receives `ctx.phase`, so the same task function can branch on the phase + * without doing index arithmetic. + * + * Run with: + * bench run benchmarks/examples/02-phases.bench.ts --concurrency 2 + * + * Add `--no-ingest` to run without a platform endpoint or API key. + */ +import '../src/env.js'; +import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner'; +import type { NoopParticipant } from './participants.js'; +import { createNoopParticipant } from './participants.js'; + +/** + * `phases` and `iterations` are mutually exclusive in `defineBenchmarkConfig`. + * When `phases` is set, the total number of task slots is the sum of all phase + * iteration counts (here 2 + 4 = 6). The slots run in phase order: first every + * `cold` slot, then every `warm` slot. + * + * Each slot is tagged with its phase name, and `ctx.phase` inside the task is + * set to that name. + */ +export const config = defineBenchmarkConfig({ + benchmarkSlug: 'examples-phases', + benchmarkName: 'Examples: Phases', + phases: [ + { name: 'cold', iterations: 2 }, + { name: 'warm', iterations: 4 }, + ], + concurrency: 2, + participants: [createNoopParticipant('noop', 100)], +}); + +/** + * The task uses `ctx.phase` to vary behavior. In a real benchmark this might + * mean using a cold-start payload for the `cold` phase and a warm-start payload + * for the `warm` phase. The phase name is also written into the measured data + * so the dashboard can group or filter by phase. + */ +export const task = defineTask(async ({ participant, step, measure, log, phase }) => { + log(`starting ${phase ?? 'unknown'} phase`); + + const compute = participant.createCompute(); + const start = performance.now(); + + // In a real benchmark you might use a different payload per phase. + const command = phase === 'cold' ? 'node -v' : 'node -e "console.log(1+1)"'; + + const sandbox = await step('create', () => compute.sandbox.create()); + try { + const result = await step('exec', () => sandbox.runCommand(command)); + + // `measure` expects a `JsonObject`; `phase` may be `undefined` in a config + // that does not use phases, so we only add it when it is present. + measure({ + ttiMs: performance.now() - start, + exitCode: result.exitCode, + ...(phase ? { phase } : {}), + }); + } finally { + await step('destroy', () => sandbox.destroy(), { reportConcurrency: false }); + } +}); diff --git a/benchmarks/examples/03-round-robin.bench.ts b/benchmarks/examples/03-round-robin.bench.ts new file mode 100644 index 00000000..bdfa777a --- /dev/null +++ b/benchmarks/examples/03-round-robin.bench.ts @@ -0,0 +1,60 @@ +/** + * Round-robin benchmark. + * + * Demonstrates `groupBy: 'round'`. In this mode every participant runs its Nth + * task before any participant starts its (N+1)th, so the Nth tasks of all + * providers happen back-to-back under the same conditions. The runner builds + * the task records manually and streams them to the platform via a reporter. + * + * Run with: + * bench run benchmarks/examples/03-round-robin.bench.ts --iterations 4 --concurrency 1 --group-by round + * + * Add `--no-ingest` to run without a platform endpoint or API key. + */ +import '../src/env.js'; +import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner'; +import type { NoopParticipant } from './participants.js'; +import { exampleProviders } from './participants.js'; + +/** + * `groupBy: 'round'` changes execution order: + * + * Round 0: alpha task 0, beta task 0, gamma task 0 + * Round 1: alpha task 1, beta task 1, gamma task 1 + * ... + * + * This is useful when you want the Nth iteration of every provider to start at + * roughly the same wall-clock time, rather than finishing all iterations for + * one provider before moving to the next. + * + * `concurrency` is set to 1 here because the round-robin path runs one task per + * round; the ordering is the primary concern, not per-participant burst. + */ +export const config = defineBenchmarkConfig({ + benchmarkSlug: 'examples-round-robin', + benchmarkName: 'Examples: Round Robin', + iterations: 4, + concurrency: 1, + groupBy: 'round', + participants: exampleProviders, +}); + +/** + * `taskIndex` is the zero-based slot index within the participant's assignment. + * In round mode it effectively represents the round number, so we log it to + * make the ordering visible in the worker log. + */ +export const task = defineTask(async ({ participant, step, measure, log, taskIndex }) => { + log(`round ${taskIndex + 1} for ${participant.name}`); + + const compute = participant.createCompute(); + const start = performance.now(); + + const sandbox = await step('create', () => compute.sandbox.create()); + try { + const result = await step('exec', () => sandbox.runCommand('node -v')); + measure({ ttiMs: performance.now() - start, exitCode: result.exitCode }); + } finally { + await step('destroy', () => sandbox.destroy(), { reportConcurrency: false }); + } +}); diff --git a/benchmarks/examples/04-shapes.bench.ts b/benchmarks/examples/04-shapes.bench.ts new file mode 100644 index 00000000..f443aa68 --- /dev/null +++ b/benchmarks/examples/04-shapes.bench.ts @@ -0,0 +1,71 @@ +/** + * Shapes benchmark. + * + * Demonstrates `shapes`: named variants of the same benchmark that swap the + * platform slug/name and any stable distinguishing knob (e.g. stagger delay). + * Scale knobs such as `iterations` and `concurrency` are still overridden from + * the CLI, so one file can back several platform benchmarks. + * + * Run the quick variant: + * bench run benchmarks/examples/04-shapes.bench.ts --shape quick --iterations 2 --concurrency 1 + * + * Run the thorough variant: + * bench run benchmarks/examples/04-shapes.bench.ts --shape thorough --iterations 10 --concurrency 3 + * + * Add `--no-ingest` to run without a platform endpoint or API key. + */ +import '../src/env.js'; +import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner'; +import type { NoopParticipant } from './participants.js'; +import { createNoopParticipant } from './participants.js'; + +/** + * The base config defines the default identity. `shapes` then declares variants. + * + * - `quick` reports under a different platform slug and has no stagger delay. + * - `thorough` reports under another slug and staggers each task start by 250ms. + * + * `--shape quick` swaps the identity before the run is created, so the platform + * records the run under `examples-shapes-quick` instead of `examples-shapes`. + * + * `iterations` and `concurrency` are not part of the shape because they are + * environment-specific scale knobs; override them from the CLI per run. + */ +export const config = defineBenchmarkConfig({ + benchmarkSlug: 'examples-shapes', + benchmarkName: 'Examples: Shapes', + iterations: 3, + concurrency: 1, + shapes: { + quick: { + slug: 'examples-shapes-quick', + name: 'Examples: Shapes (Quick)', + staggerDelayMs: 0, + }, + thorough: { + slug: 'examples-shapes-thorough', + name: 'Examples: Shapes (Thorough)', + staggerDelayMs: 250, + }, + }, + participants: [createNoopParticipant('noop', 100)], +}); + +/** + * The task itself is the same for both shapes; the only difference is the + * platform identity and the stagger delay selected by `--shape`. + */ +export const task = defineTask(async ({ participant, step, measure, log }) => { + log('starting shaped iteration'); + + const compute = participant.createCompute(); + const start = performance.now(); + + const sandbox = await step('create', () => compute.sandbox.create()); + try { + const result = await step('exec', () => sandbox.runCommand('node -v')); + measure({ ttiMs: performance.now() - start, exitCode: result.exitCode }); + } finally { + await step('destroy', () => sandbox.destroy(), { reportConcurrency: false }); + } +}); diff --git a/benchmarks/examples/05-step-options.bench.ts b/benchmarks/examples/05-step-options.bench.ts new file mode 100644 index 00000000..f451d9ee --- /dev/null +++ b/benchmarks/examples/05-step-options.bench.ts @@ -0,0 +1,56 @@ +/** + * Step options benchmark. + * + * Demonstrates per-step `concurrency` and `timeoutMs`. A step can invoke its + * function multiple times in parallel and abort any slow invocation with a + * `step_timeout` error. + * + * Run with: + * bench run benchmarks/examples/05-step-options.bench.ts --iterations 2 --no-ingest + */ +import '../src/env.js'; +import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner'; +import type { NoopParticipant } from './participants.js'; +import { createNoopParticipant } from './participants.js'; + +/** + * The config is minimal: one provider, low iterations. The interesting + * behavior is inside `ctx.step(...)` in the task. + */ +export const config = defineBenchmarkConfig({ + benchmarkSlug: 'examples-step-options', + benchmarkName: 'Examples: Step Options', + iterations: 2, + concurrency: 1, + participants: [createNoopParticipant('noop', 100)], +}); + +export const task = defineTask(async ({ participant, step, measure, log }) => { + const compute = participant.createCompute(); + + log('running three create/exec/destroy cycles in parallel inside one step'); + + // `concurrency: 3` invokes the step function three times in parallel. + // `timeoutMs: 2000` aborts any invocation that takes longer than 2s. + // The return value is an array containing one result per invocation. + const results = await step( + 'parallel-work', + async () => { + const start = performance.now(); + const sandbox = await compute.sandbox.create(); + try { + const result = await sandbox.runCommand('node -v'); + return { exitCode: result.exitCode, ttiMs: performance.now() - start }; + } finally { + await sandbox.destroy(); + } + }, + { concurrency: 3, timeoutMs: 2000 }, + ); + + // `results` is an array because `concurrency` was greater than 1. + const avgTtiMs = results.reduce((sum, r) => sum + r.ttiMs, 0) / results.length; + const maxTtiMs = Math.max(...results.map((r) => r.ttiMs)); + + measure({ avgTtiMs, maxTtiMs, created: results.length }); +}); diff --git a/benchmarks/examples/06-scoring.bench.ts b/benchmarks/examples/06-scoring.bench.ts new file mode 100644 index 00000000..7294d20a --- /dev/null +++ b/benchmarks/examples/06-scoring.bench.ts @@ -0,0 +1,65 @@ +/** + * Scoring benchmark. + * + * Demonstrates the run-level `onScore` hook and the exported `score` helper. + * The runner can compute a weighted composite score from measured metrics and + * submit it to the platform as a run summary. `onComplete` here prints the + * same score locally so it is visible even when running with `--no-ingest`. + * + * Run with: + * bench run benchmarks/examples/06-scoring.bench.ts --iterations 5 --no-ingest + */ +import '../src/env.js'; +import { defineBenchmarkConfig, defineTask, lowerIsBetter, score, type ScoringSpec } from '@benchsdk/runner'; +import type { NoopParticipant } from './participants.js'; +import { createNoopParticipant } from './participants.js'; + +// A scoring spec is an ordinary object: a list of metrics, each with a unit, +// a ceiling (best possible value for normalization), and a weighting across +// median/p95/p99. `lowerIsBetter` is a small helper that sets `higherIsBetter: false`. +const scoringSpec: ScoringSpec = { + metrics: [ + lowerIsBetter('ttiMs', { + unit: 'ms', + ceiling: 1000, + weights: { median: 0.5, p95: 0.3, p99: 0.2 }, + }), + ], +}; + +export const config = defineBenchmarkConfig({ + benchmarkSlug: 'examples-scoring', + benchmarkName: 'Examples: Scoring', + iterations: 5, + concurrency: 1, + participants: [createNoopParticipant('noop', 100)], + // `onScore` is called after all participants finish. When the run is being + // reported to the platform, the returned spec is used to build and submit + // a run summary. It has no effect in `--no-ingest` mode. + onScore: () => scoringSpec, + // `onComplete` is always called, so we use it (and the exported `score` + // function) to print the composite score locally as well. + onComplete: (outcome) => { + const scored = score(outcome, scoringSpec); + for (const result of scored) { + console.log( + ` ${result.provider}: compositeScore=${result.compositeScore}, successRate=${result.successRate}`, + ); + } + }, +}); + +export const task = defineTask(async ({ participant, step, measure, log }) => { + log('running scored iteration'); + + const compute = participant.createCompute(); + const start = performance.now(); + + const sandbox = await step('create', () => compute.sandbox.create()); + try { + const result = await step('exec', () => sandbox.runCommand('node -v')); + measure({ ttiMs: performance.now() - start, exitCode: result.exitCode }); + } finally { + await step('destroy', () => sandbox.destroy(), { reportConcurrency: false }); + } +}); diff --git a/benchmarks/examples/README.md b/benchmarks/examples/README.md new file mode 100644 index 00000000..3e3328b5 --- /dev/null +++ b/benchmarks/examples/README.md @@ -0,0 +1,84 @@ +# benchSDK Examples + +These examples are self-contained, runnable demonstrations of the benchSDK. They use a tiny `NoopParticipant` helper that simulates a sandbox provider, so no cloud credentials are required. + +## Prerequisites + +Build the local packages first (the CLI is not pre-built in the repo): + +```sh +pnpm -r --filter "./packages/**" build +``` + +## Running an example + +```sh +bench run benchmarks/examples/00-hello-world.bench.ts --iterations 3 +``` + +Or with `tsx` directly: + +```sh +pnpm tsx packages/benchsdk-runner/dist/bin.js run benchmarks/examples/00-hello-world.bench.ts --iterations 3 +``` + +## Dry-run mode (no platform needed) + +Every example can be run without a benchmarks platform endpoint or API key by adding `--no-ingest` (or setting `BENCHSDK_NO_INGEST=true`): + +```sh +bench run benchmarks/examples/00-hello-world.bench.ts --iterations 3 --no-ingest +``` + +In dry-run mode the runner executes the tasks locally, prints per-task results, and skips platform upload. This is useful for local development and CI smoke tests. + +To run against a real platform, set the endpoint and API key: + +```sh +export BENCHMARKS_PLATFORM_URL=http://localhost:3000 +export BENCHMARKS_PLATFORM_API_KEY=bp_... +``` + +See `.agents/skills/local-platform-e2e/SKILL.md` for a full local stack setup. + +## Examples + +| File | What it demonstrates | +|------|---------------------| +| `00-hello-world.bench.ts` | A single participant, one task with `step`, `measure`, and `log`. | +| `01-multiple-providers.bench.ts` | The same task run against multiple providers, with `--provider` filtering. | +| `02-phases.bench.ts` | Ordered `phases` with per-phase iteration counts and `ctx.phase`. | +| `03-round-robin.bench.ts` | `groupBy: 'round'` interleaving participant tasks. | +| `04-shapes.bench.ts` | `shapes` for named benchmark variants selected with `--shape`. | +| `05-step-options.bench.ts` | Per-step `concurrency` and `timeoutMs` for parallel step invocations. | +| `06-scoring.bench.ts` | `onScore` and the exported `score` helper for weighted composite scoring. | + +## What you should see + +For each example the CLI prints the resolved knobs, a run URL (unless `--no-ingest`), per-participant progress, and per-task success/failure lines: + +``` +Examples: Hello World (self-contained) +Date: 2026-08-12T17:30:00.000Z +Knobs: iterations=3, concurrency=1, staggerDelayMs=0, groupBy=participant + +Run created: examples-hello-world-... (run-id) +View at: http://localhost:3000/org/.../benchmarks/examples-hello-world/runs/run-id + +====================================================================== + Participant: noop +====================================================================== + [noop] Task 1/3: success {"ttiMs": 123, "exitCode": 0} + [noop] Task 2/3: success {"ttiMs": 145, "exitCode": 0} + [noop] Task 3/3: success {"ttiMs": 112, "exitCode": 0} + Done: 3/3 succeeded. + +All done. View at: http://localhost:3000/org/.../benchmarks/examples-hello-world/runs/run-id +``` + +With `--no-ingest` the output is the same except the run URL lines are replaced by `Dry run: no platform ingest or reporting.` and `All done. No platform run created.`. + +## Files + +- `participants.ts` — the mock provider factory used by every example. +- `*.bench.ts` — standalone benchmark modules, each exporting `config` and `task`. diff --git a/benchmarks/examples/participants.ts b/benchmarks/examples/participants.ts new file mode 100644 index 00000000..36e79eef --- /dev/null +++ b/benchmarks/examples/participants.ts @@ -0,0 +1,75 @@ +import type { BaseParticipant } from '@benchsdk/client'; + +/** + * Minimal sandbox shape used by the example benchmarks. + * + * A real provider returns a provider SDK instance with async methods that talk + * to a cloud API. The examples keep everything local by returning an object + * with the same `runCommand` / `destroy` shape so the benchmark task code does + * not need to know it is running against a mock. + */ +export interface NoopSandbox { + /** Simulated command execution. Returns an exit code and optional stderr. */ + runCommand(command: string): Promise<{ exitCode: number; stderr?: string }>; + /** Simulated teardown. */ + destroy(): Promise; +} + +/** + * A participant that the runner can schedule. + * + * `BaseParticipant` only requires `name` and `requiredEnvVars`. The runner + * filters participants by checking `requiredEnvVars`; if the array is empty the + * participant is always available. We extend `BaseParticipant` with the + * `createCompute` factory so the task implementation can call it. + */ +export interface NoopParticipant extends BaseParticipant { + /** Returns a compute adapter whose `sandbox.create()` produces a `NoopSandbox`. */ + createCompute(): { sandbox: { create(): Promise } }; +} + +/** A tiny promise-based sleep helper used to simulate network latency. */ +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Factory for a mock provider. + * + * Each call creates a new participant object with its own `latencyMs`. The + * sandbox returned by `createCompute` waits roughly `latencyMs` plus a random + * jitter before returning, so different providers report different TTI numbers + * in the dashboard and console output. + */ +export function createNoopParticipant(name: string, latencyMs = 100): NoopParticipant { + return { + // `name` is used as the participant slug in API paths, dashboard URLs, and + // the `--provider` CLI filter. + name, + // Empty array means this participant never gets skipped for missing env + // vars, so the examples run without credentials. + requiredEnvVars: [], + createCompute: () => ({ + sandbox: { + create: async () => ({ + runCommand: async (command: string) => { + // Add a small, variable delay so each provider reports distinct timing. + await sleep(latencyMs + Math.floor(Math.random() * latencyMs)); + return { exitCode: 0, stderr: '' }; + }, + destroy: async () => { + // Tiny cleanup delay so the destroy step has non-zero latency. + await sleep(10); + }, + }), + }, + }), + }; +} + +/** Three mock providers with increasing base latency for clear differentiation. */ +export const exampleProviders: NoopParticipant[] = [ + createNoopParticipant('alpha', 100), + createNoopParticipant('beta', 200), + createNoopParticipant('gamma', 300), +]; diff --git a/tsconfig.json b/tsconfig.json index 6f11950b..b88c2684 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,7 +18,8 @@ "benchmarks/browser/**/*.ts", "benchmarks/storage/**/*.ts", "benchmarks/ai-gateway/**/*.ts", - "benchmarks/scale/**/*.ts" + "benchmarks/scale/**/*.ts", + "benchmarks/examples/**/*.ts" ], "exclude": ["node_modules", "benchmarks/dist", "packages", "results", ".git"] }