From f1fb25d525ccbb93a8c9d850b91f25d661d18387 Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Tue, 22 Sep 2026 16:20:02 -0700 Subject: [PATCH 1/5] Run code generation on build --- CONTRIBUTING.md | 8 +- Herebyfile.mjs | 257 +++++++++++------- tools/scripts/gen/generatedFile.test.mts | 120 ++++---- tools/scripts/tsc/generate-encoder.ts | 18 +- tools/scripts/tsc/generate-enums.ts | 8 + tools/scripts/tsc/generate-go-ast.ts | 6 +- tools/scripts/tsc/generate-ts-ast.ts | 26 +- tools/scripts/tsc/generate.ts | 2 + tools/scripts/tsc/schema.ts | 17 +- .../_scripts/generate-unicode-data.mts | 6 +- 10 files changed, 283 insertions(+), 185 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 462f0e44e9fef..6a55dc58bfc01 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -117,6 +117,13 @@ to bypass incremental caches. `generate` includes every generator and fetches the pinned LSP model when its local cache is missing or stale. Use `generate:go` for the narrower scope of `go generate ./...` in the `tsc` module. +`npx hereby build` runs `generate` before compiling; a separate generation step +is not needed. The API and package build tasks share this prerequisite, and +`build:watch` runs the full incremental generation suite before every rebuild. +Each watch pass clears cached input fingerprints and runs generation in-process. +Restart the watcher after editing generator implementations. Generator-owned outputs are +excluded from watch events to avoid regeneration loops. + Package-specific commands: ```bash @@ -142,7 +149,6 @@ go -C ./tsc test -run='TestLocal/' ./internal/testrunner Run: ```bash -npx hereby generate npx hereby build npx hereby test npx hereby test:all diff --git a/Herebyfile.mjs b/Herebyfile.mjs index ad6f1d06e88de..ba8211789737c 100644 --- a/Herebyfile.mjs +++ b/Herebyfile.mjs @@ -2,7 +2,7 @@ import AdmZip from "adm-zip"; import chokidar from "chokidar"; -import { task } from "hereby"; +import { task as herebyTask } from "hereby"; import assert from "node:assert"; import crypto from "node:crypto"; import fs from "node:fs"; @@ -17,8 +17,14 @@ import * as tar from "tar"; import { xSync } from "tinyexec"; import { enableFileFingerprintCache, + invalidateFileFingerprints, run, } from "./tools/scripts/gen/utils.mts"; +import { outputs as encoderOutputs } from "./tools/scripts/tsc/generate-encoder.ts"; +import { outputs as enumOutputs } from "./tools/scripts/tsc/generate-enums.ts"; +import { outputs as goASTOutputs } from "./tools/scripts/tsc/generate-go-ast.ts"; +import { outputs as tsASTOutputs } from "./tools/scripts/tsc/generate-ts-ast.ts"; +import generateUnicodeData, { outputs as unicodeOutputs } from "./tsc/internal/stringutil/_scripts/generate-unicode-data.mts"; enableFileFingerprintCache(); @@ -29,6 +35,17 @@ if (process.platform === "win32") { const __filename = url.fileURLToPath(new URL(import.meta.url)); const __dirname = path.dirname(__filename); +/** @typedef {import("hereby").TaskOptions & { outputs?: readonly string[] }} TaskOptions */ +/** @type {TaskOptions[]} */ +const taskDescriptors = []; + +/** @param {TaskOptions} options */ +function task(options) { + const result = herebyTask(options); + taskDescriptors.push(options); + return result; +} + const isCI = !!process.env.CI || !!process.env.TF_BUILD; const stableThreeComponentVersionPatternSource = String.raw`^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$`; const stableThreeComponentVersionPattern = new RegExp(stableThreeComponentVersionPatternSource); @@ -217,7 +234,6 @@ async function runWithConcurrencyLimit(tasks, concurrency) { const builtLocal = "./built/local"; const libsDir = "./tsc/internal/bundled/libs"; -const libsRegexp = /(?:^|[\\/])internal[\\/]bundled[\\/]libs[\\/]/; /** * @param {string} out @@ -228,7 +244,7 @@ async function generateLibs(out) { const libs = await fs.promises.readdir(libsDir); await Promise.all(libs.map(async lib => { - fs.promises.copyFile(path.join(libsDir, lib), path.join(out, lib)); + await fs.promises.copyFile(path.join(libsDir, lib), path.join(out, lib)); })); } @@ -269,71 +285,6 @@ function buildTsc(opts) { }); } -export const tscBuild = task({ - name: "tsc:build", - description: "Builds the tsc binary.", - run: async () => { - await buildTsc({ extraFlags: options.release ? getReleaseBuildFlags() : [] }); - }, -}); - -export const tsgo = task({ - name: "tsgo", - dependencies: [lib, tscBuild], -}); - -export const local = task({ - name: "local", - dependencies: [tsgo], -}); - -export const build = task({ - name: "build", - dependencies: [local], -}); - -export const buildWatch = task({ - name: "build:watch", - description: "Builds the tsc binary and watches for changes.", - run: async () => { - await watchDebounced("build:watch", async (paths, abortSignal) => { - let libsChanged = false; - let goChanged = false; - - if (paths) { - for (const p of paths) { - if (libsRegexp.test(p)) { - libsChanged = true; - } - else if (p.endsWith(".go")) { - goChanged = true; - } - if (libsChanged && goChanged) { - break; - } - } - } - else { - libsChanged = true; - goChanged = true; - } - - if (libsChanged) { - console.log("Generating libs..."); - await generateLibs(builtLocal); - } - - if (goChanged) { - console.log("Building tsgo..."); - await buildTsc({ abortSignal }); - } - }, { - paths: ["tsc/cmd", "tsc/internal"], - ignored: path => /[\\/]testdata[\\/]/.test(path), - }); - }, -}); - export const cleanBuilt = task({ name: "clean:built", hiddenFromTaskList: true, @@ -404,8 +355,9 @@ async function runGoGenerator(name, { file, ...spec }) { /** * @param {string} name * @param {GoGenerator[] | (() => Promise)} generators + * @param {readonly string[]} [extraOutputs] */ -function goGenerateTask(name, generators) { +function goGenerateTask(name, generators, extraOutputs = []) { const run = typeof generators === "function" ? generators : async () => { for (const generator of generators) await runGoGenerator(name, generator); }; @@ -413,6 +365,10 @@ function goGenerateTask(name, generators) { return task({ name, description: `Generates ${name.slice("generate:".length)} files. Pass --force to regenerate unchanged files.`, + outputs: [ + ...(typeof generators === "function" ? [] : generators.flatMap(generator => generator.outputs.map(output => path.resolve(__dirname, generator.cwd ?? path.dirname(generator.file), output)))), + ...extraOutputs, + ], run, }); } @@ -455,11 +411,13 @@ function moqGenerator(file, type, output, { source = ".", packageName, inputs = }; } +const astStringerGenerator = stringerGenerator("tsc/internal/ast/kind_generated.go", "Kind", "kind_stringer_generated.go"); + async function runGenerateASTStringer() { - await runGoGenerator("generate:ast-stringer", stringerGenerator("tsc/internal/ast/kind_generated.go", "Kind", "kind_stringer_generated.go")); + await runGoGenerator("generate:ast-stringer", astStringerGenerator); } -export const generateASTStringer = goGenerateTask("generate:ast-stringer", runGenerateASTStringer); +export const generateASTStringer = goGenerateTask("generate:ast-stringer", [astStringerGenerator]); export const generateBundled = goGenerateTask("generate:bundled", [{ file: "tsc/internal/bundled/bundled.go", @@ -489,19 +447,22 @@ export const generateTristate = goGenerateTask("generate:tristate", [ stringerGenerator("tsc/internal/core/tristate.go", "Tristate", "tristate_stringer_generated.go"), ]); +const diagnosticLocaleDir = "loc"; +const diagnosticOutputs = ["diagnostics_generated.go", "loc_generated.go"]; + export const generateDiagnostics = goGenerateTask("generate:diagnostics", [ { file: "tsc/internal/diagnostics/diagnostics.go", inputs: ["generate.go", "diagnosticMessages.json", "extraDiagnosticMessages.json", "../{collections,json}/*.go", "../locale/lcl/*/diagnosticMessages/diagnosticMessages.generated.json.lcl"], exclude: ["**/*_test.go"], - outputs: ["diagnostics_generated.go", "loc_generated.go", "loc/*.json.gz"], + outputs: [...diagnosticOutputs, `${diagnosticLocaleDir}/*.json.gz`], commands: [ - ["go", "run", "generate.go", "-diagnostics", "diagnostics_generated.go", "-loc", "loc_generated.go", "-locdir", "loc"], + ["go", "run", "generate.go", "-diagnostics", diagnosticOutputs[0], "-loc", diagnosticOutputs[1], "-locdir", diagnosticLocaleDir], ["dprint", "fmt", "diagnostics_generated.go", "loc_generated.go"], ], }, stringerGenerator("tsc/internal/diagnostics/diagnostics.go", "Category", "stringer_generated.go"), -]); +], [path.join("tsc/internal/diagnostics", diagnosticLocaleDir)]); export const generateAutoImport = goGenerateTask("generate:autoimport", [ stringerGenerator("tsc/internal/ls/autoimport/export.go", "ExportSyntax", "export_stringer_generated.go"), @@ -535,16 +496,18 @@ export const generateVFSMatch = goGenerateTask("generate:vfsmatch", [ ]); export const generateUnicode = goGenerateTask("generate:unicode", async () => { - const { default: generate } = await import("./tsc/internal/stringutil/_scripts/generate-unicode-data.mts"); - await generate(!!options.force); -}); + await generateUnicodeData(!!options.force); +}, unicodeOutputs.map(output => path.join("tsc/internal/stringutil", output))); + +const extensionOutputs = ["packages/vscode-typescript/l10n/bundle.l10n.json"]; +const extensionTestOutputs = ["packages/vscode-typescript/l10n/bundle.l10n.qps-ploc.json", "packages/vscode-typescript/package.nls.qps-ploc.json"]; async function runGenerateExtension() { const { default: cache } = await import("./tools/scripts/gen/cache.mts"); await cache({ cwd: __dirname, inputs: [__filename, "packages/vscode-typescript/package.json", "packages/vscode-typescript/src/**/*"], - outputs: ["packages/vscode-typescript/l10n/bundle.l10n.json"], + outputs: extensionOutputs, commands: [["npm", "run", "-w", "native-preview", "generateLocBundle"]], envInputs: [], force: !!options.force, @@ -554,6 +517,7 @@ async function runGenerateExtension() { export const generateExtension = task({ name: "generate:extension", description: "Generates files in the extension. Pass --force to regenerate unchanged files.", + outputs: extensionOutputs, run: runGenerateExtension, }); @@ -562,7 +526,7 @@ async function runGenerateExtensionTest() { await cache({ cwd: __dirname, inputs: [__filename, "packages/vscode-typescript/package.json", "packages/vscode-typescript/l10n/bundle.l10n.json", "packages/vscode-typescript/package.nls.json"], - outputs: ["packages/vscode-typescript/l10n/bundle.l10n.qps-ploc.json", "packages/vscode-typescript/package.nls.qps-ploc.json"], + outputs: extensionTestOutputs, commands: [["npm", "run", "-w", "native-preview", "generateLocTest"]], envInputs: [], force: !!options.force, @@ -573,20 +537,24 @@ export const generateExtensionTest = task({ name: "generate:extension-test", description: "Generates pseudo-localized extension resources. Pass --force to regenerate unchanged files.", dependencies: [generateExtension], + outputs: extensionTestOutputs, run: runGenerateExtensionTest, }); +const lspModelOutputs = ["metaModel.json", "metaModelSchema.mts"]; +const lspOutputs = ["tsc/internal/lsp/lsproto/lsp_generated.go"]; + async function runGenerateLSP() { const { GeneratedFile } = await import("./tools/scripts/gen/generatedFile.mts"); const directory = path.join(__dirname, "tsc/internal/lsp/lsproto/_generate"); - const modelFiles = ["metaModel.json", "metaModelSchema.mts"].map(file => new GeneratedFile(path.join(directory, file), [path.join(directory, "fetchModel.mts"), path.join(__dirname, "package-lock.json")])); + const modelFiles = lspModelOutputs.map(file => new GeneratedFile(path.join(directory, file), [path.join(directory, "fetchModel.mts"), path.join(__dirname, "package-lock.json")])); if (!modelFiles.every(file => file.isCurrent(!!options.force))) { for (const file of modelFiles) file.invalidate(); const { default: fetchModel } = await import("./tsc/internal/lsp/lsproto/_generate/fetchModel.mts"); await fetchModel(); for (const file of modelFiles) file.markCurrent(); } - const output = new GeneratedFile(path.join(directory, "../lsp_generated.go"), [ + const output = new GeneratedFile(path.join(__dirname, lspOutputs[0]), [ __filename, path.join(directory, "generate.mts"), ...modelFiles.map(file => file.fileName), @@ -604,6 +572,7 @@ async function runGenerateLSP() { export const generateLSP = task({ name: "generate:lsp", description: "Generates LSP bindings from the pinned protocol model. Pass --force to regenerate unchanged files.", + outputs: [...lspOutputs, ...lspModelOutputs.map(output => path.join("tsc/internal/lsp/lsproto/_generate", output))], run: runGenerateLSP, }); @@ -615,6 +584,7 @@ async function runGenerateEnums() { export const generateEnums = task({ name: "generate:enums", description: "Generates TypeScript enum files from Go source. Pass --force to regenerate unchanged files.", + outputs: enumOutputs, run: runGenerateEnums, }); @@ -627,6 +597,7 @@ async function runGenerateAST() { export const generateAST = task({ name: "generate:ast", description: "Generates AST, kind stringer, and encoder files from ast.json. Pass --force to regenerate unchanged files.", + outputs: [...goASTOutputs, ...tsASTOutputs, ...encoderOutputs], run: runGenerateAST, }); @@ -638,9 +609,12 @@ async function runGenerateSync() { export const generateSync = task({ name: "generate:sync", description: "Generates synchronous and generator APIs and tests. Pass --force to regenerate unchanged files.", + outputs: ["packages/typescript/src/api/sync", "packages/typescript/test/sync", "packages/typescript/test/generators/api.bench.ts"], run: runGenerateSync, }); +const apiOutputs = ["packages/typescript/src/api/proto.generated.ts"]; + async function runGenerateAPI() { await runGoGenerator("generate:api", { file: "tsc/internal/api/proto.go", @@ -656,7 +630,7 @@ async function runGenerateAPI() { ], exclude: ["**/*_test.go", "**/*_generated.go"], envInputs: [], - outputs: ["packages/typescript/src/api/proto.generated.ts"], + outputs: apiOutputs, commands: [ ["go", "-C", "./tools", "run", "./gen-proto", "../tsc/internal/api/proto.go", "../packages/typescript/src/api/proto.generated.ts"], ["dprint", "fmt", "packages/typescript/src/api/proto.generated.ts"], @@ -664,7 +638,7 @@ async function runGenerateAPI() { }); } -export const generateAPI = goGenerateTask("generate:api", runGenerateAPI); +export const generateAPI = goGenerateTask("generate:api", runGenerateAPI, apiOutputs); const vendorJsonrpcDir = "packages/typescript/vendor/vscode-jsonrpc"; const vendorJsonrpcSrc = "node_modules/vscode-jsonrpc"; @@ -697,23 +671,102 @@ async function runGenerateVendor() { export const generateVendor = task({ name: "generate:vendor", description: "Updates the vendored copy of vscode-jsonrpc from node_modules. Pass --force to regenerate unchanged files.", + outputs: [vendorJsonrpcDir], run: runGenerateVendor, }); -const generateCompiler = task({ - name: "generate:compiler", - hiddenFromTaskList: true, - dependencies: [generateAST, generateLSP], - run: async () => { - await runGenerateGo(); - await runGenerateEnums(); - }, -}); +async function runGenerate() { + await runWithConcurrencyLimit([ + async () => { + await runWithConcurrencyLimit([runGenerateAST, runGenerateLSP], 2); + await runGenerateGo(); + await runGenerateEnums(); + }, + runGenerateSync, + async () => { + await runGenerateExtension(); + await runGenerateExtensionTest(); + }, + runGenerateVendor, + ], 4); +} export const generate = task({ name: "generate", description: "Runs all code generation, including AST, LSP, APIs, extension localization, and vendored dependencies.", - dependencies: [generateCompiler, generateSync, generateExtensionTest, generateVendor], + run: runGenerate, +}); + +export const tscBuild = task({ + name: "tsc:build", + description: "Generates sources and builds the tsc binary.", + dependencies: [generate], + run: async () => { + await buildTsc({ extraFlags: options.release ? getReleaseBuildFlags() : [] }); + }, +}); + +export const tsgo = task({ + name: "tsgo", + dependencies: [lib, tscBuild], +}); + +export const local = task({ + name: "local", + dependencies: [tsgo], +}); + +export const build = task({ + name: "build", + dependencies: [local], +}); + +function getGeneratedOutputPaths() { + const outputs = taskDescriptors.flatMap(descriptor => descriptor.outputs ?? []); + return new Set([...outputs, ...fs.globSync(outputs, { cwd: __dirname })].map(output => path.resolve(__dirname, output))); +} + +export const buildWatch = task({ + name: "build:watch", + description: "Watches sources and generator inputs, regenerating and rebuilding the tsc binary on changes.", + run: async () => { + const generatedOutputs = getGeneratedOutputPaths(); + await watchDebounced("build:watch", async (_paths, abortSignal) => { + abortSignal.throwIfAborted(); + invalidateFileFingerprints(); + await runGenerate(); + abortSignal.throwIfAborted(); + await generateLibs(builtLocal); + abortSignal.throwIfAborted(); + console.log("Building tsgo..."); + await buildTsc({ abortSignal }); + }, { + paths: [ + "tsc/cmd", + "tsc/internal", + "tools/scripts/gen", + "tools/scripts/tsc", + "tools/gen-proto", + "packages/typescript/scripts", + "packages/typescript/src/api/async", + "packages/typescript/test/async", + "packages/vscode-typescript/src", + "packages/vscode-typescript/package.json", + "packages/vscode-typescript/package.nls.json", + "node_modules/vscode-jsonrpc/package.json", + "Herebyfile.mjs", + "package-lock.json", + ".dprint.jsonc", + "go.work", + "go.work.sum", + "tsc/go.mod", + "tsc/go.sum", + "tools/go.mod", + "tools/go.sum", + ], + ignored: file => generatedOutputs.has(path.resolve(file)) || path.normalize(file).split(path.sep).includes("testdata"), + }); + }, }); const coverageDir = path.join(__dirname, "coverage"); @@ -957,20 +1010,20 @@ export const testCodegen = task({ export const buildAPI = task({ name: "build:api", description: "Builds @typescript/typescript JS API.", + dependencies: [generate], run: async () => { await run("npm", ["run", "-w", "@typescript/typescript", "build"]); }, }); -async function runBuildAPITests(generateSources = true) { - if (generateSources) await runGenerateSync(); +async function runBuildAPITests() { await run("npm", ["run", "-w", "@typescript/typescript", "build:test"]); } export const buildAPITests = task({ name: "build:api:test", description: "Builds the @typescript/typescript JS API tests.", - dependencies: [generateEnums, generateAPI], + dependencies: [generate], run: runBuildAPITests, }); @@ -1128,12 +1181,9 @@ async function runFormat() { export const validate = task({ name: "validate", - description: "Generates, builds, tests, lints, and formats the repo. Pass --api to include API tests, or --all to include all code generation and ancillary repository tests Benchmarks are separate: test:benchmarks and test:benchmarks:api.", - dependencies: [options.all ? generate : generateGo], + description: "Generates, builds, tests, lints, and formats the repo. Pass --api to include API tests, or --all to include API and ancillary repository tests. Benchmarks are separate: test:benchmarks and test:benchmarks:api.", + dependencies: [build], run: async () => { - await generateLibs(builtLocal); - await buildTsc({ extraFlags: options.release ? getReleaseBuildFlags() : [] }); - /** @type {{ name: string; error: unknown }[]} */ const failures = []; /** @param {string} name @param {() => Promise} action */ @@ -1150,8 +1200,7 @@ export const validate = task({ await runValidation("test:tsc", runTests); await runValidation("test:extension", runTestExtension); if (options.api || options.all) { - if (!options.all) await runGenerateEnums(); - await runBuildAPITests(!options.all); + await runBuildAPITests(); await runValidation("test:api", runTestAPI); } if (options.all) { @@ -2143,6 +2192,7 @@ function stripConditionsFromValue(value) { export const buildNativePreviewPackages = task({ name: "typescript:build", description: "Builds TypeScript npm packages for the current platform. Pass --respectGoEnv to preserve caller-provided Go build settings.", + dependencies: usePublishedPlatformPackagesForVsix ? undefined : [generate], run: runBuildNativePreviewPackages, }); @@ -2830,6 +2880,7 @@ export const vscodeTypescriptRelease = task({ export const nativePreviewRelease = task({ name: "typescript:release", hiddenFromTaskList: true, + dependencies: [generate], run: async () => { if (releaseVscodeTypescript) { throw new Error("typescript:release cannot be used with --vscodeTypescriptRelease; use vscode-typescript:release"); diff --git a/tools/scripts/gen/generatedFile.test.mts b/tools/scripts/gen/generatedFile.test.mts index 283fd1d8233cc..7b71f5f8dd555 100644 --- a/tools/scripts/gen/generatedFile.test.mts +++ b/tools/scripts/gen/generatedFile.test.mts @@ -1,14 +1,14 @@ import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; import { createHash } from "node:crypto"; +import { once } from "node:events"; import fs from "node:fs"; import { syncBuiltinESMExports } from "node:module"; import os from "node:os"; import path from "node:path"; import { test } from "node:test"; import { stripVTControlCharacters } from "node:util"; -import { runInNewContext } from "node:vm"; import { x } from "tinyexec"; -import ts from "typescript"; import cache from "./cache.mts"; import { defaultCacheDirectory, @@ -21,59 +21,63 @@ import { run, } from "./utils.mts"; -test("validate generates before building and selects the generation scope", async () => { - const fileName = path.resolve(import.meta.dirname, "../../../Herebyfile.mjs"); - const source = ts.createSourceFile(fileName, fs.readFileSync(fileName, "utf8"), ts.ScriptTarget.Latest, true, ts.ScriptKind.JS); - const declaration = source.statements.filter(ts.isVariableStatement) - .flatMap(statement => statement.declarationList.declarations) - .find(declaration => declaration.name.getText(source) === "validate"); - assert.ok(declaration?.initializer); - for (const options of [{}, { api: true }, { all: true }]) { - const calls: string[] = []; - const action = (name: string) => async () => { - calls.push(name); - }; - const generate = { run: action("generate") }; - const generateGo = { run: action("generate:go") }; - const build = { run: action("build") }; - const validation = runInNewContext(declaration.initializer.getText(source), { - task: (spec: unknown) => spec, - options, - generate, - generateGo, - build, - builtLocal: "./built/local", - generateLibs: action("lib"), - buildTsc: action("build"), - getReleaseBuildFlags: () => [], - runGenerateGo: action("generate:go"), - runGenerateEnums: action("generate:enums"), - runGenerateAPI: action("generate:api"), - runGenerateExtension: action("generate:extension"), - runGenerateVendor: action("generate:vendor"), - runBuildAPITests: async (generate = true) => { - if (generate) calls.push("generate:sync"); - calls.push("build:api:test"); - }, - runTests: action("test:tsc"), - runTestExtension: action("test:extension"), - runTestAPI: action("test:api"), - runTestBenchmarks: action("test:benchmarks"), - runTestTools: action("test:tools"), - runSmokeTest: action("test:smoke"), - runLint: action("lint"), - runFormat: action("format"), - }) as { dependencies: { run: () => Promise; }[]; run: () => Promise; }; - await Promise.all(validation.dependencies.map(dependency => dependency.run())); - await validation.run(); - assert.equal(calls[0], "all" in options ? "generate" : "generate:go"); - assert.ok(calls.indexOf("build") > 0); - assert.ok(calls.indexOf("build") < calls.indexOf("test:tsc")); - assert.equal(calls.includes("test:api"), "api" in options || "all" in options); - assert.equal(calls.includes("test:tools"), "all" in options); - if ("all" in options) { - assert.deepEqual(calls.filter(name => name.startsWith("generate")), ["generate"]); +test("build watch reloads schema edits without generated-output restarts", async context => { + const root = path.resolve(import.meta.dirname, "../../.."); + const schema = path.join(root, "tools/scripts/tsc/ast.json"); + const original = fs.readFileSync(schema); + const kindOutput = path.join(root, "tsc/internal/ast/kind_generated.go"); + const originalKindOutput = fs.readFileSync(kindOutput); + const script = `process.argv = [process.execPath, ${JSON.stringify(path.join(root, "node_modules/hereby/bin/hereby.js"))}, "build:watch"]; + process.once("message", () => { process.emit("SIGINT"); process.disconnect(); }); + await import("./node_modules/hereby/bin/hereby.js");`; + const child = spawn(process.execPath, ["--input-type=module", "-e", script], { cwd: root, stdio: ["ignore", "pipe", "pipe", "ipc"] }); + const exited = once(child, "close"); + let output = ""; + let remaining = ""; + let idle = Promise.withResolvers(); + const collect = (data: Buffer) => { + const text = stripVTControlCharacters(data.toString()); + output += text; + remaining += text; + const marker = "[build:watch] run complete, waiting for changes..."; + const index = remaining.indexOf(marker); + if (index >= 0) { + idle.resolve(remaining.slice(0, index)); + remaining = remaining.slice(index + marker.length); + } + }; + assert.ok(child.stdout && child.stderr); + child.stdout.on("data", collect); + child.stderr.on("data", collect); + const deadline = Promise.withResolvers(); + const timer = setTimeout(() => deadline.reject(new Error(`Watch did not complete:\n${output.slice(-8000)}`)), 300_000); + const unexpectedExit = exited.then(() => { + throw new Error(`Watcher exited unexpectedly:\n${output.slice(-8000)}`); + }); + context.after(async () => { + clearTimeout(timer); + if (child.exitCode === null && child.signalCode === null) { + child.send("stop"); + await exited; } + fs.writeFileSync(schema, original); + fs.writeFileSync(kindOutput, originalKindOutput); + }); + const initial = await Promise.race([idle.promise, deadline.promise, unexpectedExit]); + assert.match(initial, /\$ go build/); + for (const comment of ["First watch schema edit", "Second watch schema edit"]) { + timer.refresh(); + idle = Promise.withResolvers(); + const updated = JSON.parse(original.toString()); + updated.kinds.elements.unshift({ comment }); + fs.writeFileSync(schema, JSON.stringify(updated, undefined, 4) + "\n"); + const rebuilt = await Promise.race([idle.promise, deadline.promise, unexpectedExit]); + assert.match(rebuilt, /changed due to .*ast\.json/); + assert.match(rebuilt, /Wrote .*ast_generated\.go/); + assert.match(rebuilt, /\$ go build/); + assert.doesNotMatch(rebuilt, /changed due to .*_generated\.go|aborting in-progress run|Error in /); + assert.ok(rebuilt.indexOf("Wrote ") < rebuilt.indexOf("$ go build")); + assert.ok(fs.readFileSync(kindOutput, "utf8").includes(comment)); } }); @@ -370,12 +374,8 @@ test("generate includes standalone generators without Go traversal", async () => assert.ok(index >= 0, `Missing task event: ${event}`); return index; }; - const compilerStart = eventIndex("Starting generate:compiler"); - assert.ok(eventIndex("Finished generate:ast ") < compilerStart); - assert.ok(eventIndex("Finished generate:lsp ") < compilerStart); - assert.ok(eventIndex("Starting generate:sync") < compilerStart); - assert.ok(eventIndex("Starting generate:vendor") < compilerStart); - assert.ok(eventIndex("Finished generate:extension ") < eventIndex("Starting generate:extension-test")); + assert.ok(eventIndex("Starting generate") < eventIndex("Finished generate")); + assert.ok(eventIndex("generateLocBundle") < eventIndex("generateLocTest")); assert.doesNotMatch(stdout, /\$ go generate|npm run --silent cache/); assert.match(stdout, /Unicode tables/); assert.match(stdout, /Enums are up to date|All generated values match Go/); diff --git a/tools/scripts/tsc/generate-encoder.ts b/tools/scripts/tsc/generate-encoder.ts index 1b106bb8ff555..791bbbbe238d8 100644 --- a/tools/scripts/tsc/generate-encoder.ts +++ b/tools/scripts/tsc/generate-encoder.ts @@ -2059,35 +2059,43 @@ function writeAndFormat(filePath: string, generate: () => string, force: boolean console.log(`Wrote ${filePath}`); } +export const outputs = [ + "tsc/internal/api/encoder/encoder_generated.go", + "tsc/internal/api/encoder/decoder_generated.go", + "packages/typescript/src/api/node/protocol.generated.ts", + "packages/typescript/src/api/node/encoder.generated.ts", + "packages/typescript/src/api/node/node.generated.ts", +] as const; + export default function main(force = false) { console.log("Generating encoder/decoder code..."); writeAndFormat( - path.join(ROOT, "tsc/internal/api/encoder/encoder_generated.go"), + path.join(ROOT, outputs[0]), generateGoEncoder, force, ); writeAndFormat( - path.join(ROOT, "tsc/internal/api/encoder/decoder_generated.go"), + path.join(ROOT, outputs[1]), generateGoDecoder, force, ); writeAndFormat( - path.join(ROOT, "packages/typescript/src/api/node/protocol.generated.ts"), + path.join(ROOT, outputs[2]), generateTSProtocol, force, ); writeAndFormat( - path.join(ROOT, "packages/typescript/src/api/node/encoder.generated.ts"), + path.join(ROOT, outputs[3]), generateTSEncoder, force, ); writeAndFormat( - path.join(ROOT, "packages/typescript/src/api/node/node.generated.ts"), + path.join(ROOT, outputs[4]), generateTSNodeGenerated, force, ); diff --git a/tools/scripts/tsc/generate-enums.ts b/tools/scripts/tsc/generate-enums.ts index f0c67100f5239..c32e43079db5e 100644 --- a/tools/scripts/tsc/generate-enums.ts +++ b/tools/scripts/tsc/generate-enums.ts @@ -341,6 +341,14 @@ function renderEnumTS(def: EnumDef, members: EnumMember[]): string { const enumValuesGeneratedGoPath = path.join(ROOT, "tsc/internal/api/enum_values_generated.go"); +export const outputs = [ + enumValuesGeneratedGoPath, + ...enumDefs.flatMap(def => { + const camelName = def.name.charAt(0).toLowerCase() + def.name.slice(1); + return [path.join(ROOT, def.outDir, `${camelName}.enum.ts`), path.join(ROOT, def.outDir, `${camelName}.ts`)]; + }), +]; + interface GeneratedEnum { def: EnumDef; code: string; diff --git a/tools/scripts/tsc/generate-go-ast.ts b/tools/scripts/tsc/generate-go-ast.ts index 2f1bf452d09af..bc43e832544d0 100644 --- a/tools/scripts/tsc/generate-go-ast.ts +++ b/tools/scripts/tsc/generate-go-ast.ts @@ -1092,13 +1092,15 @@ function writeAndFormat(filePath: string, generateContent: () => string, force: console.log(`Wrote ${filePath}`); } +export const outputs = ["tsc/internal/ast/ast_generated.go", "tsc/internal/ast/kind_generated.go"] as const; + export default function main(force = false) { console.log("Generating Go AST code..."); - const outPath = path.join(ROOT, "tsc/internal/ast/ast_generated.go"); + const outPath = path.join(ROOT, outputs[0]); writeAndFormat(outPath, () => generate() + "\n", force); - const kindOutPath = path.join(ROOT, "tsc/internal/ast/kind_generated.go"); + const kindOutPath = path.join(ROOT, outputs[1]); writeAndFormat(kindOutPath, () => generateKind() + "\n", force); } diff --git a/tools/scripts/tsc/generate-ts-ast.ts b/tools/scripts/tsc/generate-ts-ast.ts index 3955d9b9f34d0..79b8c57d4cab7 100644 --- a/tools/scripts/tsc/generate-ts-ast.ts +++ b/tools/scripts/tsc/generate-ts-ast.ts @@ -41,7 +41,7 @@ function isInheritedFromTsBase(member: MemberInfo): boolean { // NodeBase fields (e.g. Flags) are class-level properties on NodeObject, // not data members stored in _data. -const nodeBaseFieldNames = new Set( +let nodeBaseFieldNames = new Set( api.bases().find(b => b.name === "NodeBase")?.fields.filter(f => !f.noTS).map(f => f.name) ?? [], ); @@ -107,7 +107,7 @@ function syntaxKindChecksForNode(node: NodeType): string[] { // Bases for ast.generated.ts // ──────────────────────────────────────────────────────────────────────────── -const goOnlyBases = new Set( +let goOnlyBases = new Set( api.bases().filter(base => base.isGoOnly()).map(base => base.key), ); @@ -216,7 +216,7 @@ function computeTsVariants(): TsVariant[] { return result; } -const tsVariants = computeTsVariants(); +let tsVariants = computeTsVariants(); function visitorNodes(): NodeType[] { return api.nodes().filter(node => !isVariantNode(node)); @@ -1821,13 +1821,23 @@ function writeAndFormat(filePath: string, generate: () => string, force: boolean console.log(`Generated ${filePath}`); } +export const outputs = [ + "packages/typescript/src/ast/factory.generated.ts", + "packages/typescript/src/ast/is.generated.ts", + "packages/typescript/src/ast/ast.generated.ts", + "packages/typescript/src/ast/visitor.generated.ts", +] as const; + export default function main(force = false) { console.log("Generating TS AST code..."); - - const factoryPath = path.join(ROOT, "packages/typescript/src/ast/factory.generated.ts"); - const isGenPath = path.join(ROOT, "packages/typescript/src/ast/is.generated.ts"); - const astGenPath = path.join(ROOT, "packages/typescript/src/ast/ast.generated.ts"); - const visitorPath = path.join(ROOT, "packages/typescript/src/ast/visitor.generated.ts"); + nodeBaseFieldNames = new Set(api.bases().find(base => base.name === "NodeBase")?.fields.filter(field => !field.noTS).map(field => field.name) ?? []); + goOnlyBases = new Set(api.bases().filter(base => base.isGoOnly()).map(base => base.key)); + tsVariants = computeTsVariants(); + + const factoryPath = path.join(ROOT, outputs[0]); + const isGenPath = path.join(ROOT, outputs[1]); + const astGenPath = path.join(ROOT, outputs[2]); + const visitorPath = path.join(ROOT, outputs[3]); writeAndFormat(astGenPath, generateAstGenerated, force); writeAndFormat(factoryPath, generateFactory, force); diff --git a/tools/scripts/tsc/generate.ts b/tools/scripts/tsc/generate.ts index 65f2a3997980a..434db91c38ca1 100644 --- a/tools/scripts/tsc/generate.ts +++ b/tools/scripts/tsc/generate.ts @@ -3,8 +3,10 @@ import { parseGeneratorArgs } from "../gen/utils.mts"; import generateEncoder from "./generate-encoder.ts"; import generateGoAST from "./generate-go-ast.ts"; import generateTSAST from "./generate-ts-ast.ts"; +import { reloadSchema } from "./schema.ts"; export default function generate(force = false) { + reloadSchema(); generateEncoder(force); generateGoAST(force); generateTSAST(force); diff --git a/tools/scripts/tsc/schema.ts b/tools/scripts/tsc/schema.ts index 6d2985d433435..1dcaf10796f02 100644 --- a/tools/scripts/tsc/schema.ts +++ b/tools/scripts/tsc/schema.ts @@ -1261,7 +1261,16 @@ export class SchemaAPI { } } -export const api = new SchemaAPI(JSON.parse( - fs.readFileSync(path.join(ROOT, "tools/scripts/tsc/ast.json"), "utf-8"), -) as Schema); -api.validate(); +function readSchema(): SchemaAPI { + const schema = new SchemaAPI(JSON.parse( + fs.readFileSync(path.join(ROOT, "tools/scripts/tsc/ast.json"), "utf-8"), + ) as Schema); + schema.validate(); + return schema; +} + +export let api = readSchema(); + +export function reloadSchema(): void { + api = readSchema(); +} diff --git a/tsc/internal/stringutil/_scripts/generate-unicode-data.mts b/tsc/internal/stringutil/_scripts/generate-unicode-data.mts index e24c8a5d7152b..504db9bcc6ac5 100644 --- a/tsc/internal/stringutil/_scripts/generate-unicode-data.mts +++ b/tsc/internal/stringutil/_scripts/generate-unicode-data.mts @@ -231,14 +231,16 @@ ${renderRangeTable("unicodeESNextIdentifierPart", partTable)} `; } +export const outputs = ["js_case_generated.go", "identifier_parts_generated.go"] as const; + export default async function generateUnicode(force = false, outDir = path.resolve(scriptDir, "..")) { outDir = path.resolve(outDir); const inputs = [ import.meta.filename, fileURLToPath(import.meta.resolve(`${PACKAGE}/package.json`)), ]; - const caseOutput = new GeneratedFile(path.join(outDir, "js_case_generated.go"), inputs); - const identifierOutput = new GeneratedFile(path.join(outDir, "identifier_parts_generated.go"), inputs); + const caseOutput = new GeneratedFile(path.join(outDir, outputs[0]), inputs); + const identifierOutput = new GeneratedFile(path.join(outDir, outputs[1]), inputs); const generatedFiles = [caseOutput, identifierOutput]; if (generatedFiles.every(file => file.isCurrent(force))) { console.log("Unicode tables are up to date."); From 5d2ca2fa8322aa26a02bb8e20bda10f1172c0c42 Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Tue, 22 Sep 2026 16:40:14 -0700 Subject: [PATCH 2/5] Simplify the absolute unit of commands in CONTRIBUTING --- CONTRIBUTING.md | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6a55dc58bfc01..0ce8f6d842df4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -149,18 +149,8 @@ go -C ./tsc test -run='TestLocal/' ./internal/testrunner Run: ```bash -npx hereby build -npx hereby test -npx hereby test:all -npx hereby lint -npx hereby format -npx hereby check:format -npm run -w @typescript/typescript build -npm run -w @typescript/typescript test -npm run -w native-preview build -go -C ./tsc mod tidy -diff -go -C ./tools mod tidy -diff -go work sync +npx hereby validate --all +npx hereby tidy git diff --exit-code ``` From 5a9eaff4fcdc7ffb854d43a00438d31d9ea56664 Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Tue, 22 Sep 2026 20:10:46 -0700 Subject: [PATCH 3/5] Remove bad test, move schema validation to before generation, remove in-proc codegen from watch targets --- Herebyfile.mjs | 16 +++-- tools/scripts/gen/generatedFile.test.mts | 79 ++++++------------------ tools/scripts/tsc/generate-go-ast.ts | 6 +- 3 files changed, 34 insertions(+), 67 deletions(-) diff --git a/Herebyfile.mjs b/Herebyfile.mjs index ba8211789737c..2b9dc0f7c3fe9 100644 --- a/Herebyfile.mjs +++ b/Herebyfile.mjs @@ -731,6 +731,10 @@ export const buildWatch = task({ description: "Watches sources and generator inputs, regenerating and rebuilding the tsc binary on changes.", run: async () => { const generatedOutputs = getGeneratedOutputPaths(); + const generatorDirectories = [ + "tsc/internal/lsp/lsproto/_generate", + "tsc/internal/stringutil/_scripts", + ].map(directory => path.resolve(directory)); await watchDebounced("build:watch", async (_paths, abortSignal) => { abortSignal.throwIfAborted(); invalidateFileFingerprints(); @@ -744,17 +748,14 @@ export const buildWatch = task({ paths: [ "tsc/cmd", "tsc/internal", - "tools/scripts/gen", - "tools/scripts/tsc", + "tools/scripts/tsc/ast.json", "tools/gen-proto", - "packages/typescript/scripts", "packages/typescript/src/api/async", "packages/typescript/test/async", "packages/vscode-typescript/src", "packages/vscode-typescript/package.json", "packages/vscode-typescript/package.nls.json", "node_modules/vscode-jsonrpc/package.json", - "Herebyfile.mjs", "package-lock.json", ".dprint.jsonc", "go.work", @@ -764,7 +765,12 @@ export const buildWatch = task({ "tools/go.mod", "tools/go.sum", ], - ignored: file => generatedOutputs.has(path.resolve(file)) || path.normalize(file).split(path.sep).includes("testdata"), + ignored: file => { + const absolutePath = path.resolve(file); + return generatedOutputs.has(absolutePath) + || generatorDirectories.some(directory => absolutePath === directory || absolutePath.startsWith(directory + path.sep)) + || path.normalize(file).split(path.sep).includes("testdata"); + }, }); }, }); diff --git a/tools/scripts/gen/generatedFile.test.mts b/tools/scripts/gen/generatedFile.test.mts index 7b71f5f8dd555..a5feda166bedb 100644 --- a/tools/scripts/gen/generatedFile.test.mts +++ b/tools/scripts/gen/generatedFile.test.mts @@ -1,7 +1,5 @@ import assert from "node:assert/strict"; -import { spawn } from "node:child_process"; import { createHash } from "node:crypto"; -import { once } from "node:events"; import fs from "node:fs"; import { syncBuiltinESMExports } from "node:module"; import os from "node:os"; @@ -21,63 +19,26 @@ import { run, } from "./utils.mts"; -test("build watch reloads schema edits without generated-output restarts", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const schema = path.join(root, "tools/scripts/tsc/ast.json"); - const original = fs.readFileSync(schema); - const kindOutput = path.join(root, "tsc/internal/ast/kind_generated.go"); - const originalKindOutput = fs.readFileSync(kindOutput); - const script = `process.argv = [process.execPath, ${JSON.stringify(path.join(root, "node_modules/hereby/bin/hereby.js"))}, "build:watch"]; - process.once("message", () => { process.emit("SIGINT"); process.disconnect(); }); - await import("./node_modules/hereby/bin/hereby.js");`; - const child = spawn(process.execPath, ["--input-type=module", "-e", script], { cwd: root, stdio: ["ignore", "pipe", "pipe", "ipc"] }); - const exited = once(child, "close"); - let output = ""; - let remaining = ""; - let idle = Promise.withResolvers(); - const collect = (data: Buffer) => { - const text = stripVTControlCharacters(data.toString()); - output += text; - remaining += text; - const marker = "[build:watch] run complete, waiting for changes..."; - const index = remaining.indexOf(marker); - if (index >= 0) { - idle.resolve(remaining.slice(0, index)); - remaining = remaining.slice(index + marker.length); - } - }; - assert.ok(child.stdout && child.stderr); - child.stdout.on("data", collect); - child.stderr.on("data", collect); - const deadline = Promise.withResolvers(); - const timer = setTimeout(() => deadline.reject(new Error(`Watch did not complete:\n${output.slice(-8000)}`)), 300_000); - const unexpectedExit = exited.then(() => { - throw new Error(`Watcher exited unexpectedly:\n${output.slice(-8000)}`); - }); - context.after(async () => { - clearTimeout(timer); - if (child.exitCode === null && child.signalCode === null) { - child.send("stop"); - await exited; - } - fs.writeFileSync(schema, original); - fs.writeFileSync(kindOutput, originalKindOutput); - }); - const initial = await Promise.race([idle.promise, deadline.promise, unexpectedExit]); - assert.match(initial, /\$ go build/); - for (const comment of ["First watch schema edit", "Second watch schema edit"]) { - timer.refresh(); - idle = Promise.withResolvers(); - const updated = JSON.parse(original.toString()); - updated.kinds.elements.unshift({ comment }); - fs.writeFileSync(schema, JSON.stringify(updated, undefined, 4) + "\n"); - const rebuilt = await Promise.race([idle.promise, deadline.promise, unexpectedExit]); - assert.match(rebuilt, /changed due to .*ast\.json/); - assert.match(rebuilt, /Wrote .*ast_generated\.go/); - assert.match(rebuilt, /\$ go build/); - assert.doesNotMatch(rebuilt, /changed due to .*_generated\.go|aborting in-progress run|Error in /); - assert.ok(rebuilt.indexOf("Wrote ") < rebuilt.indexOf("$ go build")); - assert.ok(fs.readFileSync(kindOutput, "utf8").includes(comment)); +test("Go AST generator validates bases after schema reloads", async context => { + const { default: generateGoAST } = await import("../tsc/generate-go-ast.ts"); + const schema = await import("../tsc/schema.ts"); + context.mock.method(GeneratedFile.prototype, "isCurrent", () => true); + context.after(() => schema.reloadSchema()); + generateGoAST(); + + for ( + const { name, extendsKeys, error } of [ + { name: "duplicate bases", extendsKeys: ["NodeBase", "NodeBase"], error: /declares duplicate embedded bases/ }, + { name: "missing NodeBase", extendsKeys: [], error: /does not embed NodeBase at offset zero/ }, + ] + ) { + await context.test(name, context => { + schema.reloadSchema(); + const node = [...schema.api.nodes()][0]; + assert.ok(node); + context.mock.getter(node, "extendsKeys", () => extendsKeys); + assert.throws(() => generateGoAST(), error); + }); } }); diff --git a/tools/scripts/tsc/generate-go-ast.ts b/tools/scripts/tsc/generate-go-ast.ts index bc43e832544d0..d34faf27bf380 100644 --- a/tools/scripts/tsc/generate-go-ast.ts +++ b/tools/scripts/tsc/generate-go-ast.ts @@ -193,9 +193,6 @@ function verifyNodeBaseAtOffsetZero(): void { } } -verifyNoDuplicateBases(); -verifyNodeBaseAtOffsetZero(); - // A base with no fields and no extends is a struct{} marker. Go pads a struct whose last field is // zero-size (so the field's address stays in bounds), so markers are embedded first, never last; being // zero-size, they don't move NodeBase off offset zero. @@ -1095,6 +1092,9 @@ function writeAndFormat(filePath: string, generateContent: () => string, force: export const outputs = ["tsc/internal/ast/ast_generated.go", "tsc/internal/ast/kind_generated.go"] as const; export default function main(force = false) { + verifyNoDuplicateBases(); + verifyNodeBaseAtOffsetZero(); + console.log("Generating Go AST code..."); const outPath = path.join(ROOT, outputs[0]); From 9bfefb28688a605034790cb44b01dee7189ca60e Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Tue, 22 Sep 2026 20:32:56 -0700 Subject: [PATCH 4/5] Make `cache` testable without invoking command line processes, remove all hereby task definition integration tests from generatedFile.test.mts --- Herebyfile.mjs | 3 +- tools/scripts/gen/cache.mts | 15 +- tools/scripts/gen/cache.test.mts | 187 ++++++-- tools/scripts/gen/generatedFile.test.mts | 544 +---------------------- tools/scripts/gen/utils.mts | 30 +- 5 files changed, 190 insertions(+), 589 deletions(-) diff --git a/Herebyfile.mjs b/Herebyfile.mjs index 2b9dc0f7c3fe9..8620ae279d1c5 100644 --- a/Herebyfile.mjs +++ b/Herebyfile.mjs @@ -1006,9 +1006,8 @@ export const testTools = task({ export const testCodegen = task({ name: "test:codegen", - description: "Runs incremental codegen tests.", + description: "Runs codegen cache unit tests without invoking generators.", run: async () => { - await run("go", ["-C", "tsc", "mod", "download"]); await run("node", ["--test", "--test-concurrency=1", "./tools/scripts/gen/*.test.mts"]); }, }); diff --git a/tools/scripts/gen/cache.mts b/tools/scripts/gen/cache.mts index 99b9e6e3efb1f..1b74eade7bfab 100644 --- a/tools/scripts/gen/cache.mts +++ b/tools/scripts/gen/cache.mts @@ -8,6 +8,7 @@ import { goInputs, repoRoot, run, + withFileFingerprintWrites, } from "./utils.mts"; export interface CacheOptions { @@ -21,6 +22,10 @@ export interface CacheOptions { force?: boolean; } +export interface CacheHost { + run(command: string, args: readonly string[], options: { cwd: string; env: NodeJS.ProcessEnv; }): Promise; +} + function expand(patterns: string[], cwd: string, exclude: string[] = []) { return [...new Set(fs.globSync(patterns, { cwd, exclude }).map(file => path.resolve(cwd, file)))].sort(); } @@ -34,7 +39,7 @@ export default async function cache({ envInputs = ["GOOS", "GOARCH", "GOFLAGS", "GOTOOLCHAIN", "GOEXPERIMENT", "CGO_ENABLED", "GOWORK"], env, force = false, -}: CacheOptions): Promise { +}: CacheOptions, host: CacheHost = { run }): Promise { if (!outputs.length || !commands.length || commands.some(command => !command.length)) { throw new Error("Cached generation requires outputs and nonempty commands."); } @@ -91,9 +96,11 @@ export default async function cache({ return true; } for (const file of previous) file.invalidate(); - for (const [command, ...args] of commands) { - await run(command, args, { cwd, env: environment }); - } + await withFileFingerprintWrites(async () => { + for (const [command, ...args] of commands) { + await host.run(command, args, { cwd, env: environment }); + } + }); if (!complete()) throw new Error(`Generation did not produce all declared outputs: ${outputs.join(", ")}`); if (snapshotInputs(true).hash === before.hash) { for (const file of artifacts(outputFiles())) file.markCurrent(); diff --git a/tools/scripts/gen/cache.test.mts b/tools/scripts/gen/cache.test.mts index aeb4445078a4f..35e5f45133e2c 100644 --- a/tools/scripts/gen/cache.test.mts +++ b/tools/scripts/gen/cache.test.mts @@ -1,59 +1,121 @@ import assert from "node:assert/strict"; -import * as fs from "node:fs"; +import fs from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; import os from "node:os"; import path from "node:path"; import { test } from "node:test"; -import cache from "./cache.mts"; +import cache, { type CacheHost } from "./cache.mts"; +import { GeneratedFile } from "./generatedFile.mts"; +import { enableFileFingerprintCache } from "./utils.mts"; + +test("command cache snapshots and generated files share invocation fingerprints", async context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-fingerprint-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const input = path.join(directory, "input.txt"); + fs.writeFileSync(input, "input"); + const host: CacheHost = { + async run() { + fs.writeFileSync(path.join(directory, "output.txt"), "generated"); + }, + }; + const options = { + cwd: directory, + inputs: ["input.txt"], + outputs: ["output.txt"], + commands: [["generate"]], + }; + assert.equal(await cache(options, host), false); + context.after(enableFileFingerprintCache()); + const reads = context.mock.method(fs, "readFileSync"); + syncBuiltinESMExports(); + context.after(() => { + reads.mock.restore(); + syncBuiltinESMExports(); + }); + assert.equal(await cache(options, host), true); + assert.equal(await cache(options, host), true); + new GeneratedFile(path.join(directory, "other.txt"), [input]); + assert.equal(reads.mock.calls.filter(call => call.arguments[0] === input).length, 1); +}); test("cache commands use and track an explicit environment without changing the parent", async context => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache-env-")); context.after(() => fs.rmSync(directory, { recursive: true, force: true })); const parentValue = process.env.GOFILE; + let calls = 0; + const host: CacheHost = { + async run(command, args, { cwd, env }) { + calls++; + assert.equal(command, "generate"); + assert.deepEqual(args, []); + fs.writeFileSync(path.join(cwd, "output.json"), JSON.stringify({ file: env.GOFILE, pkg: env.GOPACKAGE, cwd })); + }, + }; const options = { cwd: directory, inputs: [], outputs: ["output.json"], envInputs: ["GOFILE", "GOPACKAGE"], env: { GOFILE: "source.go", GOPACKAGE: "example" }, - commands: [[process.execPath, "-e", "require('node:fs').writeFileSync('output.json', JSON.stringify({ file: process.env.GOFILE, pkg: process.env.GOPACKAGE, cwd: process.cwd() }))"]], + commands: [["generate"]], }; - assert.equal(await cache(options), false); + assert.equal(await cache(options, host), false); assert.deepEqual(JSON.parse(fs.readFileSync(path.join(directory, "output.json"), "utf8")), { file: "source.go", pkg: "example", cwd: directory, }); - assert.equal(await cache(options), true); - assert.equal(await cache({ ...options, env: { ...options.env, GOFILE: "other.go" } }), false); + assert.equal(await cache(options, host), true); + assert.equal(calls, 1); + assert.equal(await cache({ ...options, env: { ...options.env, GOFILE: "other.go" } }, host), false); + assert.equal(calls, 2); assert.equal(process.env.GOFILE, parentValue); }); test("cache commands preserve the working directory and arguments", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_cache probe-")); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache probe-")); context.after(() => fs.rmSync(directory, { recursive: true, force: true, maxRetries: 10 })); fs.writeFileSync(path.join(directory, "input.txt"), "input"); - fs.writeFileSync(path.join(directory, "generate.cjs"), "const fs = require('node:fs'); fs.writeFileSync('output.txt', fs.readFileSync('input.txt') + JSON.stringify(process.argv.slice(2)));\n"); - fs.writeFileSync(path.join(directory, "finish.cjs"), "require('node:fs').appendFileSync('output.txt', ' finished');\n"); + const calls: string[] = []; + const host: CacheHost = { + async run(command, args, { cwd }) { + calls.push(command); + assert.equal(cwd, directory); + if (command === "generate") { + assert.deepEqual(args, ["--help", "--force", "space value"]); + fs.writeFileSync(path.join(cwd, "output.txt"), fs.readFileSync(path.join(cwd, "input.txt")) + JSON.stringify(args)); + } + else { + assert.deepEqual(args, []); + if (command === "fail") throw new Error("Command failed"); + fs.appendFileSync(path.join(cwd, "output.txt"), " finished"); + } + }, + }; const generate = (force = false, failure = false) => cache({ cwd: directory, - inputs: ["input.txt", "*.cjs"], + inputs: ["input.txt"], outputs: ["output.txt"], commands: [ - [process.execPath, "generate.cjs", "--help", "--force", "space value"], - [process.execPath, failure ? "missing.cjs" : "finish.cjs"], + ["generate", "--help", "--force", "space value"], + [failure ? "fail" : "finish"], + ["last"], ], force, - }); + }, host); const output = path.join(directory, "output.txt"); assert.equal(await generate(), false); - assert.equal(fs.readFileSync(output, "utf8"), 'input["--help","--force","space value"] finished'); - const timestamp = fs.statSync(output).mtimeMs; + assert.equal(fs.readFileSync(output, "utf8"), 'input["--help","--force","space value"] finished finished'); + assert.deepEqual(calls, ["generate", "finish", "last"]); + calls.length = 0; assert.equal(await generate(), true); - assert.equal(fs.statSync(output).mtimeMs, timestamp); + assert.deepEqual(calls, []); assert.equal(await generate(true), false); - await assert.rejects(generate(false, true)); + assert.deepEqual(calls, ["generate", "finish", "last"]); + calls.length = 0; + await assert.rejects(generate(false, true), /Command failed/); + assert.deepEqual(calls, ["generate", "fail"]); assert.equal(await generate(), false); assert.equal(await generate(), true); }); @@ -64,26 +126,38 @@ test("cache commands track inputs, outputs, arguments, and failures", async cont const input = path.join(directory, "input.txt"); const output = path.join(directory, "output.txt"); fs.writeFileSync(input, "first"); - fs.writeFileSync(path.join(directory, "generate.cjs"), "const fs = require('node:fs'); fs.writeFileSync('output.txt', fs.readFileSync('input.txt'));\n"); - fs.writeFileSync(path.join(directory, "finish.cjs"), "const fs = require('node:fs'); fs.appendFileSync('output.txt', process.argv[2]); fs.writeFileSync('extra.txt', 'extra');\n"); + const host: CacheHost = { + async run(command, args, { cwd }) { + if (command === "generate") fs.writeFileSync(output, fs.readFileSync(input)); + else if (command === "fail") throw new Error("Command failed"); + else { + fs.appendFileSync(output, args[0]); + fs.writeFileSync(path.join(cwd, "extra.txt"), "extra"); + } + }, + }; const generate = (suffix = " done", force = false, failure = false, env: NodeJS.ProcessEnv = {}) => cache({ cwd: directory, - inputs: ["input*.txt", "*.cjs"], + inputs: ["input*.txt"], + exclude: ["input-ignored.txt"], outputs: ["output.txt", "extra*.txt"], envInputs: ["TSGO_CACHE_PROBE"], env, commands: [ - [process.execPath, "generate.cjs"], - [process.execPath, failure ? "missing.cjs" : "finish.cjs", suffix], + ["generate"], + [failure ? "fail" : "finish", suffix], ], force, - }); + }, host); await generate(); assert.equal(fs.readFileSync(output, "utf8"), "first done"); const timestamp = fs.statSync(output).mtimeMs; assert.equal(await generate(), true); assert.equal(fs.statSync(output).mtimeMs, timestamp); + fs.writeFileSync(path.join(directory, "input-ignored.txt"), "ignored"); + fs.writeFileSync(path.join(directory, "unrelated.txt"), "unrelated"); + assert.equal(await generate(), true); fs.writeFileSync(input, "second"); await generate(); assert.equal(fs.readFileSync(output, "utf8"), "second done"); @@ -113,40 +187,77 @@ test("cache commands track inputs, outputs, arguments, and failures", async cont assert.equal(await generate("--force"), true); }); +test("failed commands invalidate metadata even when outputs are unchanged", async context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache-")); + context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + let fail = false; + let calls = 0; + const host: CacheHost = { + async run() { + calls++; + fs.writeFileSync(path.join(directory, "output.txt"), "generated"); + if (fail) throw new Error("Command failed"); + }, + }; + const options = { cwd: directory, inputs: [], outputs: ["output.txt"], commands: [["generate"]] }; + assert.equal(await cache(options, host), false); + fail = true; + await assert.rejects(cache({ ...options, force: true }, host), /Command failed/); + fail = false; + assert.equal(await cache(options, host), false); + assert.equal(await cache(options, host), true); + assert.equal(calls, 3); +}); + test("cache commands reject missing inputs, outputs, and empty commands", async context => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache-")); context.after(() => fs.rmSync(directory, { recursive: true, force: true })); + let calls = 0; + const host: CacheHost = { + async run() { + calls++; + fs.writeFileSync(path.join(directory, "output.txt"), "generated"); + }, + }; const options = { cwd: directory, inputs: ["input.txt"], outputs: ["output.txt"], - commands: [[process.execPath, "-e", "require('node:fs').writeFileSync('output.txt', 'generated')"]], + commands: [["generate"]], }; - await assert.rejects(cache(options), /No inputs matched/); + await assert.rejects(cache(options, host), /No inputs matched/); assert.equal(fs.existsSync(path.join(directory, "output.txt")), false); fs.writeFileSync(path.join(directory, "input.txt"), "input"); - await assert.rejects(cache({ ...options, inputs: ["missing*.txt"] }), /No inputs matched/); - await assert.rejects(cache({ ...options, commands: [] }), /nonempty commands/); - await assert.rejects(cache({ ...options, commands: [[]] }), /nonempty commands/); - await assert.rejects(cache({ ...options, outputs: [] }), /requires outputs/); - await assert.rejects(cache({ ...options, outputs: ["output.txt", "missing.txt"] }), /did not produce all declared outputs/); - assert.equal(await cache(options), false); - assert.equal(await cache(options), true); + await assert.rejects(cache({ ...options, inputs: ["missing*.txt"] }, host), /No inputs matched/); + await assert.rejects(cache({ ...options, commands: [] }, host), /nonempty commands/); + await assert.rejects(cache({ ...options, commands: [[]] }, host), /nonempty commands/); + await assert.rejects(cache({ ...options, outputs: [] }, host), /requires outputs/); + assert.equal(calls, 0); + await assert.rejects(cache({ ...options, outputs: ["output.txt", "missing.txt"] }, host), /did not produce all declared outputs/); + assert.equal(calls, 1); + assert.equal(await cache(options, host), false); + assert.equal(await cache(options, host), true); }); test("cache commands do not cache inputs changed during execution", async context => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-cache-")); context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - for (const change of ["fs.writeFileSync('input.txt', 'changed')", "fs.writeFileSync('input-added.txt', 'added')"]) { + for (const changedFile of ["input.txt", "input-added.txt"]) { fs.writeFileSync(path.join(directory, "input.txt"), "initial"); + const host: CacheHost = { + async run() { + fs.writeFileSync(path.join(directory, "output.txt"), "generated"); + fs.writeFileSync(path.join(directory, changedFile), "changed"); + }, + }; const options = { cwd: directory, inputs: ["input*.txt"], outputs: ["output.txt"], - commands: [[process.execPath, "-e", `const fs = require('node:fs'); fs.writeFileSync('output.txt', 'generated'); ${change};`]], + commands: [["generate"]], }; - assert.equal(await cache(options), false); - assert.equal(await cache(options), false); - assert.equal(await cache(options), true); + assert.equal(await cache(options, host), false); + assert.equal(await cache(options, host), false); + assert.equal(await cache(options, host), true); } }); diff --git a/tools/scripts/gen/generatedFile.test.mts b/tools/scripts/gen/generatedFile.test.mts index a5feda166bedb..ce69e2976c2f8 100644 --- a/tools/scripts/gen/generatedFile.test.mts +++ b/tools/scripts/gen/generatedFile.test.mts @@ -5,9 +5,6 @@ import { syncBuiltinESMExports } from "node:module"; import os from "node:os"; import path from "node:path"; import { test } from "node:test"; -import { stripVTControlCharacters } from "node:util"; -import { x } from "tinyexec"; -import cache from "./cache.mts"; import { defaultCacheDirectory, GeneratedFile, @@ -16,7 +13,7 @@ import { enableFileFingerprintCache, getFileFingerprint, repoRoot, - run, + withFileFingerprintWrites, } from "./utils.mts"; test("Go AST generator validates bases after schema reloads", async context => { @@ -127,7 +124,7 @@ test("generated files share input fingerprints and refresh changed inputs", cont assert.equal(new GeneratedFile(output, [input], cache).isCurrent(), true); }); -test("invocation fingerprints reuse metadata and invalidate writes and commands", async context => { +test("invocation fingerprints reuse metadata and invalidate writes", async context => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-fingerprint-")); context.after(() => fs.rmSync(directory, { recursive: true, force: true })); context.after(enableFileFingerprintCache()); @@ -163,21 +160,26 @@ test("invocation fingerprints reuse metadata and invalidate writes and commands" assert.equal(new GeneratedFile(output, [input], metadata).isCurrent(), false); for (const failure of [false, true]) { const before = getFileFingerprint(input); - const content = failure ? "failed command" : "successful command"; - const command = run(process.execPath, ["-e", `require('node:fs').writeFileSync(process.argv[1], ${JSON.stringify(content)}); process.exit(${failure ? 1 : 0});`, input]); - if (failure) await assert.rejects(command); - else await command; + const content = failure ? "failed write" : "successful write"; + const writing = withFileFingerprintWrites(() => { + fs.writeFileSync(input, content); + if (failure) throw new Error("Write failed"); + }); + if (failure) await assert.rejects(writing, /Write failed/); + else await writing; assert.notEqual(getFileFingerprint(input), before); assert.equal(getFileFingerprint(input), createHash("sha256").update(content).digest("hex")); } - const running = run(process.execPath, ["-e", "process.exit(0)"]); + const finished = Promise.withResolvers(); + const writing = withFileFingerprintWrites(() => finished.promise); try { const before = getFileFingerprint(input); - fs.writeFileSync(input, "changed while a command is running"); + fs.writeFileSync(input, "changed while a writer is active"); assert.notEqual(getFileFingerprint(input), before); } finally { - await running; + finished.resolve(); + await writing; } }); @@ -213,31 +215,6 @@ test("fingerprint cache scopes can overlap", context => { } }); -test("command cache snapshots and generated files share invocation fingerprints", async context => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-fingerprint-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const input = path.join(directory, "input.txt"); - fs.writeFileSync(input, "input"); - const options = { - cwd: directory, - inputs: ["input.txt"], - outputs: ["output.txt"], - commands: [[process.execPath, "-e", "require('node:fs').writeFileSync('output.txt', 'generated')"]], - }; - assert.equal(await cache(options), false); - context.after(enableFileFingerprintCache()); - const reads = context.mock.method(fs, "readFileSync"); - syncBuiltinESMExports(); - context.after(() => { - reads.mock.restore(); - syncBuiltinESMExports(); - }); - assert.equal(await cache(options), true); - assert.equal(await cache(options), true); - new GeneratedFile(path.join(directory, "other.txt"), [input]); - assert.equal(reads.mock.calls.filter(call => call.arguments[0] === input).length, 1); -}); - test("changing inputs during generation does not mark stale output current", context => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-codegen-")); context.after(() => fs.rmSync(directory, { recursive: true, force: true })); @@ -308,496 +285,3 @@ test("generated files track each output independently", context => { assert.equal(files[0].isCurrent(), true); assert.equal(files[1].isCurrent(), false); }); - -test("generate:go runs Go generators directly and shares caches with Go fallback", async () => { - const root = path.resolve(import.meta.dirname, "../../.."); - const generate = () => x("npx", ["hereby", "generate:go"], { throwOnError: true, nodeOptions: { cwd: root } }); - const first = await generate(); - assert.doesNotMatch(first.stdout, /\$ go generate|npm run --silent cache|\$ node .*generate-unicode-data/); - const files = fs.globSync(["tsc/internal/**/*generated.go", "packages/typescript/src/api/proto.generated.ts"], { cwd: root }); - const timestamps = files.map(file => fs.statSync(path.join(root, file)).mtimeMs); - const current = await generate(); - assert.doesNotMatch(current.stdout, /Generated codegen outputs|Generated Unicode tables/); - assert.match(current.stdout, /Unicode tables are up to date/); - assert.deepEqual(files.map(file => fs.statSync(path.join(root, file)).mtimeMs), timestamps); - const fallback = await x("go", ["-C", "./tsc", "generate", "./internal/diagnostics"], { throwOnError: true, nodeOptions: { cwd: root } }); - assert.equal(fallback.stdout.match(/codegen outputs are already up to date/g)?.length, 2); - const nested = await x("npx", ["hereby", "generate:compileroptions"], { throwOnError: true, nodeOptions: { cwd: path.join(root, "tsc/internal/core") } }); - assert.equal(nested.stdout.match(/codegen outputs are already up to date/g)?.length, 2); -}); - -test("generate includes standalone generators without Go traversal", async () => { - const root = path.resolve(import.meta.dirname, "../../.."); - const { stdout } = await x("npx", ["hereby", "generate"], { throwOnError: true, nodeOptions: { cwd: root } }); - const log = stripVTControlCharacters(stdout); - const eventIndex = (event: string) => { - const index = log.indexOf(event); - assert.ok(index >= 0, `Missing task event: ${event}`); - return index; - }; - assert.ok(eventIndex("Starting generate") < eventIndex("Finished generate")); - assert.ok(eventIndex("generateLocBundle") < eventIndex("generateLocTest")); - assert.doesNotMatch(stdout, /\$ go generate|npm run --silent cache/); - assert.match(stdout, /Unicode tables/); - assert.match(stdout, /Enums are up to date|All generated values match Go/); - assert.match(stdout, /Sync API is up to date|Formatting\.\.\./); - assert.match(stdout, /LSP bindings are up to date|Successfully generated .*lsp_generated\.go/); - assert.match(stdout, /generateLocBundle/); - assert.match(stdout, /generateLocTest/); - assert.deepEqual( - fs.readFileSync(path.join(root, "packages/typescript/vendor/vscode-jsonrpc/package.json")), - fs.readFileSync(path.join(root, "node_modules/vscode-jsonrpc/package.json")), - ); - const lspOutput = path.join(root, "tsc/internal/lsp/lsproto/lsp_generated.go"); - const timestamp = fs.statSync(lspOutput).mtimeMs; - const current = await x("npx", ["hereby", "generate:lsp"], { throwOnError: true, nodeOptions: { cwd: root } }); - assert.match(current.stdout, /LSP bindings are up to date/); - assert.doesNotMatch(current.stdout, /Using vscode-languageclient/); - assert.equal(fs.statSync(lspOutput).mtimeMs, timestamp); -}); - -test("localization and vendoring preserve current outputs", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const generate = (force = false) => x("npx", ["hereby", "generate:extension-test", "generate:vendor", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const outputs = fs.globSync([ - "packages/vscode-typescript/l10n/*.json", - "packages/vscode-typescript/package.nls.qps-ploc.json", - "packages/typescript/vendor/vscode-jsonrpc/**/*", - ], { cwd: root }).map(file => path.join(root, file)).filter(file => fs.statSync(file).isFile()); - const contents = outputs.map(file => fs.readFileSync(file)); - context.after(() => { - for (const [index, file] of outputs.entries()) fs.writeFileSync(file, contents[index]); - }); - const oldTime = new Date("2000-01-01T00:00:00Z"); - for (const file of outputs) fs.utimesSync(file, oldTime, oldTime); - const timestamps = outputs.map(file => fs.statSync(file).mtimeMs); - await generate(); - assert.deepEqual(outputs.map(file => fs.statSync(file).mtimeMs), timestamps); - await generate(true); - for (const file of outputs) assert.notEqual(fs.statSync(file).mtimeMs, oldTime.getTime()); - for (const [index, file] of outputs.entries()) { - if (index % 2) fs.rmSync(file); - else fs.writeFileSync(file, "modified"); - } - await generate(); - assert.deepEqual(outputs.map(file => fs.readFileSync(file)), contents); -}); - -test("localization tracks source membership and package strings", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const extension = path.join(root, "packages/vscode-typescript"); - const source = path.join(extension, "src/cache-probe.ts"); - const packageStrings = path.join(extension, "package.nls.json"); - const bundle = path.join(extension, "l10n/bundle.l10n.json"); - const pseudoBundle = path.join(extension, "l10n/bundle.l10n.qps-ploc.json"); - const pseudoPackage = path.join(extension, "package.nls.qps-ploc.json"); - const originals = [packageStrings, bundle, pseudoBundle, pseudoPackage].map(file => [file, fs.readFileSync(file)] as const); - assert.equal(fs.existsSync(source), false); - context.after(() => { - fs.rmSync(source, { force: true }); - for (const [file, content] of originals) fs.writeFileSync(file, content); - }); - const generate = () => x("npx", ["hereby", "generate:extension-test"], { throwOnError: true, nodeOptions: { cwd: root } }); - for (const message of ["Codegen cache probe", "Updated codegen cache probe"]) { - fs.writeFileSync(source, `import * as vscode from "vscode";\nexport const message = vscode.l10n.t(${JSON.stringify(message)});\n`); - await generate(); - assert.ok(Object.hasOwn(JSON.parse(fs.readFileSync(bundle, "utf8")), message)); - assert.ok(Object.hasOwn(JSON.parse(fs.readFileSync(pseudoBundle, "utf8")), message)); - } - fs.rmSync(source); - await generate(); - assert.deepEqual(fs.readFileSync(bundle), originals[1][1]); - assert.deepEqual(fs.readFileSync(pseudoBundle), originals[2][1]); - const strings = JSON.parse(fs.readFileSync(packageStrings, "utf8")); - strings["cache.probe"] = "Package cache probe"; - fs.writeFileSync(packageStrings, JSON.stringify(strings)); - await generate(); - assert.ok(Object.hasOwn(JSON.parse(fs.readFileSync(pseudoPackage, "utf8")), "cache.probe")); -}); - -test("vendoring tracks package versions and supports force", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const source = path.join(root, "node_modules/vscode-jsonrpc/lib/cache-probe.txt"); - const destination = path.join(root, "packages/typescript/vendor/vscode-jsonrpc/lib/cache-probe.txt"); - const sourceManifest = path.join(root, "node_modules/vscode-jsonrpc/package.json"); - const destinationManifest = path.join(root, "packages/typescript/vendor/vscode-jsonrpc/package.json"); - const originalSourceManifest = fs.readFileSync(sourceManifest); - const originalDestinationManifest = fs.readFileSync(destinationManifest); - assert.equal(fs.existsSync(source), false); - assert.equal(fs.existsSync(destination), false); - context.after(() => { - fs.rmSync(source, { force: true }); - fs.rmSync(destination, { force: true }); - fs.writeFileSync(sourceManifest, originalSourceManifest); - fs.writeFileSync(destinationManifest, originalDestinationManifest); - }); - const generate = (force = false) => x("npx", ["hereby", "generate:vendor", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - fs.writeFileSync(source, "first"); - await generate(); - assert.equal(fs.existsSync(destination), false); - const manifest = JSON.parse(originalSourceManifest.toString()); - manifest.version += "-cache-probe"; - fs.writeFileSync(sourceManifest, JSON.stringify(manifest)); - await generate(); - assert.equal(fs.readFileSync(destination, "utf8"), "first"); - assert.equal(JSON.parse(fs.readFileSync(destinationManifest, "utf8")).version, manifest.version); - fs.writeFileSync(source, "changed"); - await generate(); - assert.equal(fs.readFileSync(destination, "utf8"), "first"); - await generate(true); - assert.equal(fs.readFileSync(destination, "utf8"), "changed"); - fs.rmSync(source); - await generate(); - assert.equal(fs.readFileSync(destination, "utf8"), "changed"); - fs.rmSync(destinationManifest); - await generate(); - assert.equal(fs.existsSync(destination), false); - assert.equal(JSON.parse(fs.readFileSync(destinationManifest, "utf8")).version, manifest.version); -}); - -test("bundled generation skips unchanged library outputs", async () => { - const root = path.resolve(import.meta.dirname, "../../.."); - const generate = () => x("go", ["-C", "./tsc", "generate", "./internal/bundled"], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const files = ["libs_generated.go", "embed_generated.go"].map(file => path.join(root, "tsc/internal/bundled", file)); - const timestamps = files.map(file => fs.statSync(file).mtimeMs); - assert.match((await generate()).stdout, /codegen outputs are already up to date/); - assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); -}); - -test("bundled generation tracks libraries, validates inputs, and supports force", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_bundled-probe-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true, maxRetries: 10 })); - const notice = path.join(directory, "CopyrightNotice.txt"); - const libs = path.join(directory, "libs"); - fs.mkdirSync(libs); - fs.writeFileSync(notice, "Fixture notice\n\n"); - const library = path.join(libs, "lib.first.d.ts"); - const content = "Fixture notice\n\n\ndeclare const first: string;\n"; - fs.writeFileSync(library, content); - const options = { - cwd: directory, - inputs: [path.join(root, "tsc/internal/bundled/generate.go"), "CopyrightNotice.txt", "libs/*"], - outputs: ["libs_generated.go", "embed_generated.go"], - commands: [["go", "run", path.join(root, "tsc/internal/bundled/generate.go")]], - }; - const generate = (force = false) => cache({ ...options, force }); - await generate(); - const files = ["libs_generated.go", "embed_generated.go"].map(file => path.join(directory, file)); - const originals = files.map(file => fs.readFileSync(file)); - assert.equal(await generate(), true); - const added = path.join(libs, "lib.second.d.ts"); - fs.writeFileSync(added, content); - assert.equal(await generate(), false); - for (const file of files) assert.match(fs.readFileSync(file, "utf8"), /lib\.second\.d\.ts/); - fs.rmSync(added); - assert.equal(await generate(), false); - for (const [index, file] of files.entries()) { - assert.deepEqual(fs.readFileSync(file), originals[index]); - fs.writeFileSync(file, "modified"); - assert.equal(await generate(), false); - assert.deepEqual(fs.readFileSync(file), originals[index]); - fs.rmSync(file, { recursive: true, maxRetries: 10 }); - assert.equal(await generate(), false); - assert.deepEqual(fs.readFileSync(file), originals[index]); - } - fs.writeFileSync(library, content.replace("string", "number")); - assert.equal(await generate(), false); - fs.writeFileSync(library, content.replaceAll("\n", "\r\n")); - await assert.rejects(generate()); - fs.writeFileSync(library, content); - assert.equal(await generate(), false); - fs.mkdirSync(path.join(libs, "unexpected")); - await assert.rejects(generate()); - fs.rmdirSync(path.join(libs, "unexpected")); - assert.equal(await generate(), false); - fs.writeFileSync(notice, "Changed fixture notice\n\n"); - await assert.rejects(generate()); - fs.writeFileSync(notice, "Fixture notice\n\n"); - assert.equal(await generate(), false); - assert.equal(await generate(true), false); - assert.equal(await generate(), true); -}); - -test("stringer generation tracks declared inputs, options, output, and force", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_stringer-probe-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const source = path.join(directory, "kind.go"); - const extra = path.join(directory, "extra.go"); - const output = path.join(directory, "kind_stringer_generated.go"); - fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); - const generate = (trimPrefix = false, force = false) => - cache({ - cwd: directory, - inputs: ["kind.go"], - outputs: ["kind_stringer_generated.go"], - env: { GOFILE: "kind.go" }, - force, - commands: [ - ["go", "tool", "golang.org/x/tools/cmd/stringer", "-type=Kind", "-output=kind_stringer_generated.go", ...(trimPrefix ? ["-trimprefix=Kind"] : [])], - ["dprint", "fmt", "kind_stringer_generated.go"], - ], - }); - await generate(); - const timestamp = fs.statSync(output).mtimeMs; - assert.equal(await generate(), true); - assert.equal(fs.statSync(output).mtimeMs, timestamp); - fs.writeFileSync(extra, "package probe\n\nconst Unrelated = 1\n"); - assert.equal(await generate(), true); - fs.appendFileSync(source, "\nconst KindSecond Kind = 1\n"); - assert.equal(await generate(), false); - assert.match(fs.readFileSync(output, "utf8"), /KindSecond/); - fs.rmSync(extra); - assert.equal(await generate(), true); - fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); - assert.equal(await generate(), false); - assert.doesNotMatch(fs.readFileSync(output, "utf8"), /KindSecond/); - assert.equal(await generate(true), false); - assert.match(fs.readFileSync(output, "utf8"), /_Kind_name = "First"/); - assert.equal(await generate(true), true); - fs.appendFileSync(output, "\n"); - assert.equal(await generate(true), false); - fs.rmSync(output); - assert.equal(await generate(true), false); - assert.equal(await generate(true, true), false); - assert.equal(await generate(true), true); - fs.appendFileSync(source, "\nconst KindInvalid Kind = missing\n"); - await assert.rejects(generate(true)); - fs.writeFileSync(source, "package probe\n\ntype Kind int\n\nconst KindFirst Kind = 0\n"); - assert.equal(await generate(true), false); - assert.equal(await generate(true), true); -}); - -test("moq generation tracks interface inputs, output, and force", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_moq-probe-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const source = path.join(directory, "client.go"); - const base = path.join(directory, "base.go"); - fs.mkdirSync(path.join(directory, "mock")); - const output = path.join(directory, "mock/mock_generated.go"); - fs.writeFileSync(base, "package probe\n\ntype Base interface { Read() string }\n"); - fs.writeFileSync(source, "package probe\n\ntype Client interface { Base }\n"); - const options = { - cwd: directory, - inputs: ["client.go", "base.go"], - outputs: ["mock/mock_generated.go"], - commands: [ - ["go", "tool", "github.com/matryer/moq", "-stub", "-fmt=goimports", "-pkg=mock", "-out=mock/mock_generated.go", ".", "Client"], - ["dprint", "fmt", "mock/mock_generated.go"], - ], - }; - const generate = (force = false) => cache({ ...options, force }); - await generate(); - const timestamp = fs.statSync(output).mtimeMs; - assert.equal(await generate(), true); - assert.equal(fs.statSync(output).mtimeMs, timestamp); - fs.writeFileSync(base, "package probe\n\ntype Base interface { Read() string; Close() error }\n"); - assert.equal(await generate(), false); - assert.match(fs.readFileSync(output, "utf8"), /CloseFunc/); - fs.rmSync(output); - assert.equal(await generate(), false); - assert.equal(await generate(true), false); - assert.equal(await generate(), true); -}); - -test("diagnostic generation tracks Go and locale outputs and supports force", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_diagnostics-probe-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const output = path.join(directory, "diagnostics_generated.go"); - const localized = path.join(directory, "loc_generated.go"); - const options = { - cwd: path.join(root, "tsc/internal/diagnostics"), - inputs: ["generate.go", "diagnosticMessages.json", "extraDiagnosticMessages.json", "../{collections,json}/*.go", "../locale/lcl/*/diagnosticMessages/diagnosticMessages.generated.json.lcl"], - exclude: ["**/*_test.go"], - outputs: [output, localized, path.join(directory, "loc/*.json.gz")], - commands: [ - ["go", "run", "generate.go", "-diagnostics", output, "-loc", localized, "-locdir", path.join(directory, "loc")], - ["dprint", "fmt", output, localized], - ], - }; - const generate = (force = false) => cache({ ...options, force }); - await generate(); - const locales = fs.globSync("loc/*.json.gz", { cwd: directory }).map(file => path.join(directory, file)); - assert.ok(locales.length > 0); - const files = [output, localized, ...locales]; - const timestamps = files.map(file => fs.statSync(file).mtimeMs); - assert.equal(await generate(), true); - assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); - const archive = fs.readFileSync(locales[0]); - fs.rmSync(locales[0]); - assert.equal(await generate(), false); - assert.deepEqual(fs.readFileSync(locales[0]), archive); - fs.writeFileSync(locales[0], "modified"); - assert.equal(await generate(), false); - assert.deepEqual(fs.readFileSync(locales[0]), archive); - const unexpected = path.join(directory, "loc/unexpected.json.gz"); - fs.writeFileSync(unexpected, "unexpected"); - assert.equal(await generate(), false); - assert.equal(fs.existsSync(unexpected), false); - fs.rmSync(path.join(directory, "loc"), { recursive: true }); - assert.equal(await generate(), false); - assert.deepEqual(fs.readFileSync(locales[0]), archive); - fs.rmSync(localized); - assert.equal(await generate(), false); - assert.equal(await generate(true), false); - assert.equal(await generate(), true); -}); - -test("Unicode generation skips unchanged tables and formatting", async () => { - const root = path.resolve(import.meta.dirname, "../../.."); - const generate = () => x("go", ["-C", "./tsc", "generate", "./internal/stringutil"], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const files = ["js_case_generated.go", "identifier_parts_generated.go"].map(file => path.join(root, "tsc/internal/stringutil", file)); - const timestamps = files.map(file => fs.statSync(file).mtimeMs); - const current = await generate(); - assert.match(current.stdout, /Unicode tables are up to date\./); - assert.doesNotMatch(current.stdout, /Formatted/); - assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); -}); - -test("Unicode generation repairs outputs and supports force", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const directory = fs.mkdtempSync(path.join(root, "tsc/internal/_unicode-probe-")); - context.after(() => fs.rmSync(directory, { recursive: true, force: true })); - const command = [path.join(root, "tsc/internal/stringutil/_scripts/generate-unicode-data.mts"), "--outDir", directory]; - const generate = (...args: string[]) => x(process.execPath, [...command, ...args], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const files = ["js_case_generated.go", "identifier_parts_generated.go"].map(file => path.join(directory, file)); - const originals = files.map(file => fs.readFileSync(file)); - const timestamps = files.map(file => fs.statSync(file).mtimeMs); - assert.match((await generate()).stdout, /Unicode tables are up to date\./); - assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); - for (const [index, file] of files.entries()) { - assert.deepEqual(originals[index], fs.readFileSync(path.join(root, "tsc/internal/stringutil", path.basename(file)))); - fs.writeFileSync(file, "modified"); - assert.match((await generate()).stdout, /Generated Unicode tables\./); - assert.deepEqual(fs.readFileSync(file), originals[index]); - fs.rmSync(file); - assert.match((await generate()).stdout, /Generated Unicode tables\./); - assert.deepEqual(fs.readFileSync(file), originals[index]); - } - const cacheFile = path.join(defaultCacheDirectory, createHash("sha256").update(files[0]).digest("hex") + ".json"); - context.after(() => fs.rmSync(cacheFile, { force: true })); - fs.rmSync(cacheFile); - assert.match((await generate()).stdout, /Generated Unicode tables\./); - assert.match((await generate("--force")).stdout, /Generated Unicode tables\./); - assert.match((await generate()).stdout, /Unicode tables are up to date\./); - const forced = await x(process.execPath, command, { - throwOnError: true, - nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, - }); - assert.match(forced.stdout, /Generated Unicode tables\./); - const overridden = await x(process.execPath, [...command, "--no-force"], { - throwOnError: true, - nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, - }); - assert.match(overridden.stdout, /Unicode tables are up to date\./); -}); - -test("enum generation skips unchanged outputs and Go verification", async () => { - const root = path.resolve(import.meta.dirname, "../../.."); - const generate = (force = false) => x("npx", ["hereby", "generate:enums", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const files = [ - ...fs.globSync("packages/typescript/src/enums/*.ts", { cwd: root }), - "tsc/internal/api/enum_values_generated.go", - ].map(file => path.join(root, file)); - const timestamps = files.map(file => fs.statSync(file).mtimeMs); - const { stdout } = await generate(); - assert.match(stdout, /Enums are up to date\./); - assert.doesNotMatch(stdout, /Getting values from go/); - assert.deepEqual(files.map(file => fs.statSync(file).mtimeMs), timestamps); - const forced = await generate(true); - assert.match(forced.stdout, /All generated values match Go\./); - assert.match((await generate()).stdout, /Enums are up to date\./); - const verifier = path.join(root, "tsc/internal/api/enum_values_generated.go"); - const cacheFile = path.join(defaultCacheDirectory, createHash("sha256").update(verifier).digest("hex") + ".json"); - fs.rmSync(cacheFile); - const regenerated = await generate(); - assert.match(regenerated.stdout, /All generated values match Go\./); - assert.match((await generate()).stdout, /Enums are up to date\./); -}); - -test("AST generation forwards force to schema generators and the kind stringer", async () => { - const root = path.resolve(import.meta.dirname, "../../.."); - const generate = (force = false) => x("npx", ["hereby", "generate:ast", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const output = path.join(root, "tsc/internal/ast/kind_stringer_generated.go"); - const oldTime = new Date("2000-01-01T00:00:00Z"); - fs.utimesSync(output, oldTime, oldTime); - const forced = await generate(true); - assert.match(forced.stdout, /Wrote .*encoder_generated\.go/); - assert.match(forced.stdout, /Wrote .*ast_generated\.go/); - assert.match(forced.stdout, /Generated .*ast\.generated\.ts/); - assert.notEqual(fs.statSync(output).mtimeMs, oldTime.getTime()); - const timestamp = fs.statSync(output).mtimeMs; - assert.doesNotMatch(forced.stdout, /\$ node .*tools\/scripts\/tsc\/generate\.ts/); - const current = await generate(); - assert.doesNotMatch(current.stdout, /(?:Wrote|Generated) /); - assert.match(current.stdout, /codegen outputs are already up to date/); - assert.equal(fs.statSync(output).mtimeMs, timestamp); -}); - -test("package generation forwards force to AST, encoder, and sync generators", async () => { - const root = path.resolve(import.meta.dirname, "../../.."); - const generate = (force = false) => x("npm", ["run", "-w", "@typescript/typescript", "generate", ...(force ? ["--", "--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const forced = await generate(true); - assert.match(forced.stdout, /Generated .*ast\.generated\.ts/); - assert.match(forced.stdout, /Wrote .*encoder_generated\.go/); - assert.match(forced.stdout, /Formatting\.\.\./); - const current = await generate(); - assert.doesNotMatch(current.stdout, /(?:Wrote|Generated) |Formatting\.\.\./); - assert.match(current.stdout, /Sync API is up to date\./); -}); - -test("API protocol generation caches formatted output and supports force", async context => { - const root = path.resolve(import.meta.dirname, "../../.."); - const output = path.join(root, "packages/typescript/src/api/proto.generated.ts"); - const generate = (force = false) => x("npx", ["hereby", "generate:api", ...(force ? ["--force"] : [])], { throwOnError: true, nodeOptions: { cwd: root } }); - await generate(); - const timestamp = fs.statSync(output).mtimeMs; - const current = await generate(); - assert.match(current.stdout, /codegen outputs are already up to date/); - assert.doesNotMatch(current.stdout, /\$ node .*cache\.mts/); - assert.equal(fs.statSync(output).mtimeMs, timestamp); - const unrelated = path.join(root, "tsc/internal/parser/codegen_cache_probe.go"); - assert.equal(fs.existsSync(unrelated), false); - context.after(() => fs.rmSync(unrelated, { force: true })); - fs.writeFileSync(unrelated, "package parser\n"); - assert.match((await generate()).stdout, /codegen outputs are already up to date/); - fs.rmSync(unrelated); - const schema = path.join(root, "tsc/internal/api/requestfilesystem/codegen_cache_probe.go"); - assert.equal(fs.existsSync(schema), false); - const original = fs.readFileSync(output, "utf8"); - context.after(async () => { - if (fs.existsSync(schema)) { - fs.rmSync(schema); - await generate(true); - } - }); - fs.writeFileSync(schema, 'package requestfilesystem\n\nconst KindCodegenCacheProbe Kind = "codegen-cache-probe"\n'); - await generate(); - assert.match(fs.readFileSync(output, "utf8"), /"codegen-cache-probe"/); - fs.rmSync(schema); - await generate(); - assert.equal(fs.readFileSync(output, "utf8"), original); - const oldTime = new Date("2000-01-01T00:00:00Z"); - fs.utimesSync(output, oldTime, oldTime); - await generate(true); - assert.notEqual(fs.statSync(output).mtimeMs, oldTime.getTime()); - assert.match((await generate()).stdout, /codegen outputs are already up to date/); - fs.utimesSync(output, oldTime, oldTime); - const nested = await x("go", ["-C", "./tsc", "generate", "./internal/api"], { - throwOnError: true, - nodeOptions: { cwd: root, env: { ...process.env, TSGO_HEREBY_FORCE: "1" } }, - }); - assert.notEqual(fs.statSync(output).mtimeMs, oldTime.getTime()); - assert.match((await generate()).stdout, /codegen outputs are already up to date/); -}); diff --git a/tools/scripts/gen/utils.mts b/tools/scripts/gen/utils.mts index fc8e830103d62..bae735bc518a5 100644 --- a/tools/scripts/gen/utils.mts +++ b/tools/scripts/gen/utils.mts @@ -42,6 +42,16 @@ function beginFileFingerprintWrite(): () => void { }; } +export async function withFileFingerprintWrites(action: () => T | PromiseLike): Promise { + const finishWrite = beginFileFingerprintWrite(); + try { + return await action(); + } + finally { + finishWrite(); + } +} + export function getFileFingerprint(file: string, fresh = false): string { file = path.resolve(file); fresh ||= activeFingerprintWriters > 0; @@ -76,9 +86,8 @@ export function formatCommandArg(arg: string) { export async function run(command: string, args: readonly string[] = [], options: RunOptions = {}) { console.log("$ " + [command, ...args].map(formatCommandArg).join(" ")); - const finishWrite = beginFileFingerprintWrite(); - try { - return await x(command, args, { + return withFileFingerprintWrites(() => + x(command, args, { throwOnError: true, ...(options.signal ? { signal: options.signal } : {}), nodeOptions: { @@ -86,11 +95,8 @@ export async function run(command: string, args: readonly string[] = [], options env: options.env ? { ...process.env, ...options.env } : undefined, stdio: options.captureOutput ? "pipe" : "inherit", }, - }); - } - finally { - finishWrite(); - } + }) + ); } export function globInputs(patterns: string[], exclude: string[] = []): string[] { @@ -115,13 +121,7 @@ export function parseGeneratorArgs } export async function formatFiles(files: string[]): Promise { - const finishWrite = beginFileFingerprintWrite(); - try { - await x("dprint", ["fmt", ...files], { throwOnError: true, nodeOptions: { cwd: repoRoot, stdio: "inherit" } }); - } - finally { - finishWrite(); - } + await withFileFingerprintWrites(() => x("dprint", ["fmt", ...files], { throwOnError: true, nodeOptions: { cwd: repoRoot, stdio: "inherit" } })); } export function formatFilesSync(files: string[]): void { From 20d95d30b9e42704495a536ed6657afcb17299e5 Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Mon, 19 Oct 2026 13:59:55 -0700 Subject: [PATCH 5/5] Fix lspModel codegen by onboarding to GeneratedFile incrementalism --- CONTRIBUTING.md | 1 + Herebyfile.mjs | 15 ++++++--------- tools/scripts/gen/lspTypeScript.test.mts | 10 +++++++--- tools/scripts/lsp/fetchModel.mts | 18 +++++++++++++++--- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ce8f6d842df4..280b9e7379a3b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,6 +102,7 @@ The repository uses a Go workspace with modules in `tsc/` and `tools/`. npx hereby build # Build the native compiler into built/local/tsc npx hereby test # Run compiler and language-service Go tests npx hereby test:all # Also run benchmarks, tools, and API tests +npx hereby test:codegen # Run codegen tests without regenerating source outputs npx hereby lint # Run custom golangci-lint for both Go modules npx hereby generate:go # Run the generators also exposed through go generate npx hereby generate # Regenerate all sources, assets, localization, and vendored files diff --git a/Herebyfile.mjs b/Herebyfile.mjs index 77af5e66e5922..117920769adb4 100644 --- a/Herebyfile.mjs +++ b/Herebyfile.mjs @@ -21,6 +21,10 @@ import { invalidateFileFingerprints, run, } from "./tools/scripts/gen/utils.mts"; +import { + ensureModel as ensureLSPModel, + outputs as lspModelOutputs, +} from "./tools/scripts/lsp/fetchModel.mts"; import { outputs as encoderOutputs } from "./tools/scripts/tsc/generate-encoder.ts"; import { outputs as enumOutputs } from "./tools/scripts/tsc/generate-enums.ts"; import { outputs as goASTOutputs } from "./tools/scripts/tsc/generate-go-ast.ts"; @@ -587,19 +591,12 @@ export const generateExtensionTest = task({ run: runGenerateExtensionTest, }); -const lspModelOutputs = ["metaModel.json", "metaModelSchema.mts"]; const lspOutputs = ["tsc/internal/lsp/lsproto/lsp_generated.go", "packages/typescript/src/vscode/protocol.generated.ts"]; async function runGenerateLSP() { const { GeneratedFile } = await import("./tools/scripts/gen/generatedFile.mts"); const directory = path.join(__dirname, "tools/scripts/lsp"); - const modelFiles = lspModelOutputs.map(file => new GeneratedFile(path.join(directory, file), [path.join(directory, "fetchModel.mts"), path.join(__dirname, "package-lock.json")])); - if (!modelFiles.every(file => file.isCurrent(!!options.force))) { - for (const file of modelFiles) file.invalidate(); - const { default: fetchModel } = await import("./tools/scripts/lsp/fetchModel.mts"); - await fetchModel(); - for (const file of modelFiles) file.markCurrent(); - } + const modelFiles = await ensureLSPModel(!!options.force); const output = new GeneratedFile(path.join(__dirname, lspOutputs[0]), [ __filename, path.join(directory, "generate.mts"), @@ -1097,7 +1094,7 @@ export const testTools = task({ export const testCodegen = task({ name: "test:codegen", - description: "Runs codegen cache unit tests without invoking generators.", + description: "Runs codegen tests without regenerating sources.", run: async () => { await run("node", ["--test", "--test-concurrency=1", "./tools/scripts/gen/*.test.mts", "./tools/scripts/tsc/*.test.ts"]); }, diff --git a/tools/scripts/gen/lspTypeScript.test.mts b/tools/scripts/gen/lspTypeScript.test.mts index 0a487d3765e88..b0d407cb03955 100644 --- a/tools/scripts/gen/lspTypeScript.test.mts +++ b/tools/scripts/gen/lspTypeScript.test.mts @@ -4,13 +4,17 @@ import fs from "node:fs"; import path from "node:path"; import { test } from "node:test"; import ts from "typescript"; -import { getTypeScriptModel } from "../lsp/generate.mts"; -import { getMiddlewareMethods } from "../lsp/generateTypeScript.mts"; +import { ensureModel } from "../lsp/fetchModel.mts"; import type { MetaModel, Type, } from "../lsp/metaModelSchema.mts"; -import { generateTypeScript } from "../lsp/typeScript.mts"; + +await ensureModel(); + +const { getTypeScriptModel } = await import("../lsp/generate.mts"); +const { getMiddlewareMethods } = await import("../lsp/generateTypeScript.mts"); +const { generateTypeScript } = await import("../lsp/typeScript.mts"); function fixture(): MetaModel { return { diff --git a/tools/scripts/lsp/fetchModel.mts b/tools/scripts/lsp/fetchModel.mts index e24a717d8ba9d..25a945a535cd1 100644 --- a/tools/scripts/lsp/fetchModel.mts +++ b/tools/scripts/lsp/fetchModel.mts @@ -5,15 +5,27 @@ import fs from "node:fs"; import path from "node:path"; import url from "node:url"; +import { GeneratedFile } from "../gen/generatedFile.mts"; const __filename = url.fileURLToPath(new URL(import.meta.url)); const __dirname = path.dirname(__filename); -const metaModelPath = path.join(__dirname, "metaModel.json"); -const metaModelSchemaPath = path.join(__dirname, "metaModelSchema.mts"); +export const outputs = ["metaModel.json", "metaModelSchema.mts"] as const; +const metaModelPath = path.join(__dirname, outputs[0]); +const metaModelSchemaPath = path.join(__dirname, outputs[1]); +const lockfilePath = path.resolve(__dirname, "../../../package-lock.json"); + +export async function ensureModel(force = false): Promise { + const files = outputs.map(output => new GeneratedFile(path.join(__dirname, output), [__filename, lockfilePath])); + if (!files.every(file => file.isCurrent(force))) { + for (const file of files) file.invalidate(); + await fetchModel(); + for (const file of files) file.markCurrent(); + } + return files; +} export default async function fetchModel() { - const lockfilePath = path.resolve(__dirname, "../../../package-lock.json"); const lockfile = JSON.parse(fs.readFileSync(lockfilePath, "utf-8")); const clientVersion: string = lockfile.packages["node_modules/vscode-languageclient"].version;