Add TypeScript 7 external API benchmarks

README.md

@@ -22,6 +22,16 @@ After these files are created, they must be added to `scenarioConfig` in `script
 > [!NOTE]
 > Benchmark capacity is limited; we may not accept all benchmarks or run them automatically in CI.
 
+### Adding an API benchmark
+
+API benchmarks time an external program that uses the TypeScript 7 JS API. `cases/scenarios/vscode-languageserver-api-metamodel` is a good template.
+
+In `scenario.json`, set `"kind": "api"` and set `args` to a Node script (relative to the scenario directory) followed by its arguments. `${typescriptDirectory}` expands to the built compiler. The script runs in `cases/solutions/<name>` and is timed from start to finish. It should point the program at the built `typescript` package, run it, and exit nonzero if the output is wrong.
+
+`ts-perf` sets `TS_API_TIMING_DIR`, and each API client writes timing data there when it is closed. A run that produces no timing data fails, so both compared compiler commits must include timing support (microsoft/TypeScript#64846).
+
+To run the benchmark in CI, add it to `allScenarios` in `scripts/src/setupPipeline.ts` with `kind: "api"` and `runIn: RunType.None`, then request the `api` preset.
+
 ## Contributing
 
 This project welcomes contributions and suggestions. Most contributions require you to agree to a

cases/api/artifact.mjs

@@ -0,0 +1,24 @@
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+
+export async function verifyArtifact(builtDir) {
+    const apiDir = path.resolve(builtDir, "api");
+    const info = JSON.parse(await fs.readFile(path.join(builtDir, "info.json"), "utf8"));
+    assert.match(info.commit, /^[a-f0-9]{40}$/);
+    const packageDir = path.join(apiDir, "node_modules/typescript");
+    const pkg = JSON.parse(await fs.readFile(path.join(packageDir, "package.json"), "utf8"));
+    assert.equal(pkg.gitHead, info.commit, "Wrong JS wrapper commit");
+    const platformDir = path.join(apiDir, "node_modules/@typescript", `typescript-${process.platform}-${process.arch}`);
+    const native = JSON.parse(await fs.readFile(path.join(platformDir, "package.json"), "utf8"));
+    assert.equal(native.gitHead, info.commit, "Wrong native compiler commit");
+    const { default: getExePath } = await import(pathToFileURL(path.join(packageDir, "lib/getExePath.js")).href);
+    assert.equal(
+        await fs.realpath(getExePath()),
+        await fs.realpath(path.join(platformDir, "lib", process.platform === "win32" ? "tsc.exe" : "tsc")),
+        "Registry binary fallback",
+    );
+    await fs.access(getExePath(), fs.constants.X_OK);
+    return { commit: info.commit, packageDir, binary: getExePath() };
+}

cases/api/eslint-setup.mjs

@@ -0,0 +1,48 @@
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+
+const fileCount = 1024;
+
+const workDir = process.cwd();
+const rules = process.argv[2];
+assert(["floating", "recommended"].includes(rules));
+const performanceDir = path.join(workDir, "performance");
+
+// Use the generator unchanged, with Prettier and ESLint fixed by the consumer's pnpm lockfile.
+await fs.rm(path.join(performanceDir, "node_modules"), { force: true });
+await fs.symlink("../node_modules", path.join(performanceDir, "node_modules"), "junction");
+const { writeCaseFiles } = await import(
+    pathToFileURL(path.join(performanceDir, "src/creators/writeCaseFiles.ts")).href
+);
+const fixture = path.join(workDir, "fixture");
+await fs.mkdir(fixture, { recursive: true });
+await fs.writeFile(path.join(fixture, "package.json"), '{"type":"module","private":true}\n');
+await fs.rm(path.join(fixture, "node_modules"), { force: true });
+await fs.symlink("../node_modules", path.join(fixture, "node_modules"), "junction");
+const data = {
+    name: "api-lint",
+    files: fileCount,
+    layout: "even",
+    rules,
+    singleRun: false,
+    types: "native",
+};
+await writeCaseFiles(data, fixture);
+const packageFile = path.join(fixture, "package.json");
+const upstreamPackage = await import(
+    pathToFileURL(path.join(performanceDir, "src/creators/files/createPackageFile.ts")).href
+);
+await fs.writeFile(
+    packageFile,
+    JSON.stringify({ type: "module", scripts: upstreamPackage.createPackageFile(data).scripts }),
+);
+// ESLint's CLI does not clear parser caches; close its synchronous native client before process exit.
+await fs.appendFile(
+    path.join(fixture, "eslint.config.js"),
+    '\nimport { clearCaches } from "@typescript-eslint/parser";\nprocess.once("exit", clearCaches);\n',
+);
+const files = (await fs.readdir(path.join(fixture, "src"), { recursive: true }))
+    .filter(file => file.endsWith(".ts"));
+assert.equal(files.length, fileCount, "Generator produced an unexpected number of files");

cases/api/eslint-setup.sh

@@ -0,0 +1,23 @@
+clone_scenario https://github.com/JoshuaKGoldberg/typescript-eslint.git 30d55234e6d719d3386884c3eef5c3a6c9d255f3
+
+run_sandboxed sh -c '
+    set -ex
+    export NX_DAEMON=false NX_NO_CLOUD=true
+    PACKAGE_MANAGER=$(node -p "require(\"./package.json\").packageManager")
+    npx "$PACKAGE_MANAGER" --filter "typescript-eslint..." \
+        --filter @typescript-eslint/ast-spec \
+        --filter @typescript-eslint/rule-schema-to-typescript-types \
+        --filter @typescript-eslint/rule-tester \
+        --no-include-workspace-root install --frozen-lockfile --ignore-scripts \
+        --package-import-method=copy --store-dir .pnpm-store
+    npx "$PACKAGE_MANAGER" exec nx build typescript-eslint --skip-nx-cache
+'
+
+# Keep the generator checkout out of Nx's workspace discovery during the build.
+git clone --filter=blob:none --no-checkout https://github.com/typescript-eslint/performance.git performance
+git -C performance fetch origin 8b8a104c482f5d21ffeddf2d52a99b2eada22aa0
+git -C performance switch --detach FETCH_HEAD
+# Leave disposable sources so clone_scenario's normal cleanup removes them on reruns.
+rm -rf performance/.git
+cp ../../api/eslint-setup.mjs api-setup.mjs
+run_sandboxed sh -c 'node api-setup.mjs "$1"' -- "$1"

cases/api/eslint.mjs

@@ -0,0 +1,47 @@
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import path from "node:path";
+
+import { verifyArtifact } from "./artifact.mjs";
+import { run, verifyResolution } from "./utils.mjs";
+
+const workDir = process.cwd();
+const artifact = await verifyArtifact(process.argv[2]);
+const checkoutDir = workDir;
+// Workspace packages resolve peers locally; a fixture-level alias alone is insufficient.
+for (
+    const directory of [
+        checkoutDir,
+        ...(await fs.readdir(path.join(checkoutDir, "packages")))
+            .map(name => path.join(checkoutDir, "packages", name)),
+    ]
+) {
+    const alias = path.join(directory, "node_modules/@typescript/native");
+    const stat = await fs.lstat(alias).catch(error => {
+        if (error.code !== "ENOENT") throw error;
+    });
+    if (!stat) continue;
+    assert(stat.isSymbolicLink(), "Refusing to replace a non-link native dependency");
+    await fs.unlink(alias);
+    await fs.symlink(artifact.packageDir, alias, "junction");
+}
+await verifyResolution(path.join(checkoutDir, "packages/typescript-estree"), artifact);
+const fixture = await fs.realpath(path.join(workDir, "fixture"));
+const output = path.join(fixture, "results.json");
+await fs.rm(output, { force: true });
+try {
+    run("npm", ["run", "lint", "--", "--format", "json", "--output-file", "results.json"], fixture);
+    assert.fail("Expected one lint finding");
+}
+catch (error) {
+    if (error.status !== 1) throw error;
+}
+const results = JSON.parse(await fs.readFile(output, "utf8"));
+assert.equal(results.length, 1024, "ESLint skipped fixture files");
+const messages = results.flatMap(result => {
+    assert.equal(result.fatalErrorCount, 0, "ESLint parser failure");
+    return result.messages;
+});
+assert.equal(messages.length, 1, "Unexpected lint findings");
+assert.equal(messages[0].ruleId, "@typescript-eslint/no-floating-promises");
+await fs.unlink(output);

cases/api/lsp-metamodel.mjs

@@ -0,0 +1,23 @@
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import path from "node:path";
+
+import { verifyArtifact } from "./artifact.mjs";
+import { run, verifyResolution } from "./utils.mjs";
+
+const workDir = process.cwd();
+const artifact = await verifyArtifact(process.argv[2]);
+const repo = workDir;
+const alias = path.join(repo, "node_modules/@typescript/native");
+await fs.rm(alias, { recursive: true });
+await fs.symlink(artifact.packageDir, alias, "junction");
+await verifyResolution(path.join(repo, "tools"), artifact);
+const output = path.join(repo, "protocol/metaModel.json");
+await fs.rm(output, { force: true });
+run("npm", ["run", "generate:metaModel"], repo);
+assert.deepEqual(
+    JSON.parse(await fs.readFile(output, "utf8")),
+    JSON.parse(await fs.readFile(path.join(workDir, "expected.json"), "utf8")),
+    "Generated metamodel differs from the checked-in output",
+);
+await fs.unlink(output);

cases/api/utils.mjs

@@ -0,0 +1,22 @@
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import fs from "node:fs/promises";
+import { createRequire } from "node:module";
+import path from "node:path";
+
+export function run(command, args, cwd) {
+    return execFileSync(command, args, {
+        cwd,
+        stdio: "inherit",
+    });
+}
+
+export async function verifyResolution(from, artifact) {
+    const actual = createRequire(path.join(from, "package.json"));
+    const expected = createRequire(path.join(artifact.packageDir, "package.json"));
+    assert.equal(
+        await fs.realpath(actual.resolve("@typescript/native/unstable/sync")),
+        await fs.realpath(expected.resolve("typescript/unstable/sync")),
+        "Consumer resolved a registry compiler",
+    );
+}

cases/scenarios/typescript-eslint-api-floating/scenario.json

@@ -0,0 +1,5 @@
+{
+    "name": "typescript-eslint-api-floating",
+    "kind": "api",
+    "args": ["../../api/eslint.mjs", "${typescriptDirectory}"]
+}

cases/scenarios/typescript-eslint-api-floating/setup.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -exo pipefail
+cd "${0%/*}"
+
+source ../../common.sh
+source ../../api/eslint-setup.sh floating

cases/scenarios/typescript-eslint-api-recommended/scenario.json

@@ -0,0 +1,5 @@
+{
+    "name": "typescript-eslint-api-recommended",
+    "kind": "api",
+    "args": ["../../api/eslint.mjs", "${typescriptDirectory}"]
+}

cases/scenarios/typescript-eslint-api-recommended/setup.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -exo pipefail
+cd "${0%/*}"
+
+source ../../common.sh
+source ../../api/eslint-setup.sh recommended

cases/scenarios/vscode-languageserver-api-metamodel/scenario.json

@@ -0,0 +1,5 @@
+{
+    "name": "vscode-languageserver-api-metamodel",
+    "kind": "api",
+    "args": ["../../api/lsp-metamodel.mjs", "${typescriptDirectory}"]
+}

cases/scenarios/vscode-languageserver-api-metamodel/setup.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -exo pipefail
+cd "${0%/*}"
+
+source ../../common.sh
+clone_scenario https://github.com/andrewbranch/vscode-languageserver-node.git fe89061e918472537a1d54132b17fe3cadd85741
+git show HEAD:protocol/metaModel.json > expected.json
+run_sandboxed sh -c '
+    set -ex
+    npm ci --ignore-scripts --no-audit --no-fund
+    node node_modules/typescript/bin/tsc -b tools/tsconfig.json types/tsconfig.json jsonrpc/tsconfig.json protocol/src/common/tsconfig.json
+'

scripts/src/__tests__/__file_snapshots__/setupPipeline/api/compute.snap

@@ -0,0 +1 @@
+undefined

scripts/src/__tests__/__file_snapshots__/setupPipeline/api/error.snap

@@ -0,0 +1 @@
+[Error: The api preset requires the native TypeScript 7 compiler and its matching JS API.]

scripts/src/__tests__/__file_snapshots__/setupPipeline/api/matrix.snap

@@ -0,0 +1 @@
+undefined

scripts/src/__tests__/__file_snapshots__/setupPipeline/api/outputVariables.snap

@@ -0,0 +1 @@
+undefined

scripts/src/__tests__/__file_snapshots__/setupPipeline/api/parameters.snap

@@ -0,0 +1 @@
+undefined

scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/compute.snap

@@ -0,0 +1,7 @@
+{
+  "parallel": "3m 40s",
+  "perAgent": {
+    "any": "11m",
+  },
+  "total": "11m",
+}

scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/error.snap

@@ -0,0 +1 @@
+undefined

scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/matrix.snap

@@ -0,0 +1,40 @@
+{
+  "any": {
+    "api_node_24_21_0_typescript_eslint_api_floating": {
+      "TSPERF_JOB_HOST": "node@24.21.0",
+      "TSPERF_JOB_ITERATIONS": 6,
+      "TSPERF_JOB_KIND": "api",
+      "TSPERF_JOB_NAME": "api_node_24_21_0_typescript_eslint_api_floating",
+      "TSPERF_JOB_SCENARIO": "typescript-eslint-api-floating",
+      "TSPERF_JOB_WARMUPS": 1,
+    },
+    "api_node_24_21_0_typescript_eslint_api_recommended": {
+      "TSPERF_JOB_HOST": "node@24.21.0",
+      "TSPERF_JOB_ITERATIONS": 6,
+      "TSPERF_JOB_KIND": "api",
+      "TSPERF_JOB_NAME": "api_node_24_21_0_typescript_eslint_api_recommended",
+      "TSPERF_JOB_SCENARIO": "typescript-eslint-api-recommended",
+      "TSPERF_JOB_WARMUPS": 1,
+    },
+    "api_node_24_21_0_vscode_languageserver_api_metamodel": {
+      "TSPERF_JOB_HOST": "node@24.21.0",
+      "TSPERF_JOB_ITERATIONS": 6,
+      "TSPERF_JOB_KIND": "api",
+      "TSPERF_JOB_NAME": "api_node_24_21_0_vscode_languageserver_api_metamodel",
+      "TSPERF_JOB_SCENARIO": "vscode-languageserver-api-metamodel",
+      "TSPERF_JOB_WARMUPS": 1,
+    },
+  },
+  "ts-perf1": {},
+  "ts-perf10": {},
+  "ts-perf11": {},
+  "ts-perf12": {},
+  "ts-perf2": {},
+  "ts-perf3": {},
+  "ts-perf4": {},
+  "ts-perf5": {},
+  "ts-perf6": {},
+  "ts-perf7": {},
+  "ts-perf8": {},
+  "ts-perf9": {},
+}

scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/outputVariables.snap

@@ -0,0 +1,23 @@
+{
+  "MATRIX_any": "{"api_node_24_21_0_typescript_eslint_api_floating":{"TSPERF_JOB_HOST":"node@24.21.0","TSPERF_JOB_ITERATIONS":6,"TSPERF_JOB_KIND":"api","TSPERF_JOB_NAME":"api_node_24_21_0_typescript_eslint_api_floating","TSPERF_JOB_SCENARIO":"typescript-eslint-api-floating","TSPERF_JOB_WARMUPS":1},"api_node_24_21_0_typescript_eslint_api_recommended":{"TSPERF_JOB_HOST":"node@24.21.0","TSPERF_JOB_ITERATIONS":6,"TSPERF_JOB_KIND":"api","TSPERF_JOB_NAME":"api_node_24_21_0_typescript_eslint_api_recommended","TSPERF_JOB_SCENARIO":"typescript-eslint-api-recommended","TSPERF_JOB_WARMUPS":1},"api_node_24_21_0_vscode_languageserver_api_metamodel":{"TSPERF_JOB_HOST":"node@24.21.0","TSPERF_JOB_ITERATIONS":6,"TSPERF_JOB_KIND":"api","TSPERF_JOB_NAME":"api_node_24_21_0_vscode_languageserver_api_metamodel","TSPERF_JOB_SCENARIO":"vscode-languageserver-api-metamodel","TSPERF_JOB_WARMUPS":1}}",
+  "MATRIX_ts_perf1": "{}",
+  "MATRIX_ts_perf10": "{}",
+  "MATRIX_ts_perf11": "{}",
+  "MATRIX_ts_perf12": "{}",
+  "MATRIX_ts_perf2": "{}",
+  "MATRIX_ts_perf3": "{}",
+  "MATRIX_ts_perf4": "{}",
+  "MATRIX_ts_perf5": "{}",
+  "MATRIX_ts_perf6": "{}",
+  "MATRIX_ts_perf7": "{}",
+  "MATRIX_ts_perf8": "{}",
+  "MATRIX_ts_perf9": "{}",
+  "TSPERF_BASELINE_COMMIT": "HEAD^1",
+  "TSPERF_BASELINE_NAME": "baseline",
+  "TSPERF_IS_COMPARISON": "true",
+  "TSPERF_IS_CUSTOM_COMMIT_RANGE": "false",
+  "TSPERF_NEW_COMMIT": "HEAD",
+  "TSPERF_NEW_NAME": "pr",
+  "TSPERF_PREDICTABLE": "false",
+  "TSPERF_PROCESS_KINDS": "api",
+}

scripts/src/__tests__/__file_snapshots__/setupPipeline/api_tsgo/parameters.snap

@@ -0,0 +1,11 @@
+{
+  "baselineCommit": "HEAD^1",
+  "baselineName": "baseline",
+  "hosts": undefined,
+  "isComparison": true,
+  "isCustomCommitRange": false,
+  "newCommit": "HEAD",
+  "newName": "pr",
+  "predictable": undefined,
+  "preset": "api",
+}