diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 462f0e44e9fef..03aeea3543eec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -104,6 +104,7 @@ npx hereby test # Run compiler and language-service Go tests npx hereby test:all # Also run benchmarks, tools, and API tests npx hereby lint # Run custom golangci-lint for both Go modules npx hereby generate:go # Run the generators also exposed through go generate +npx hereby generate:libs # Incrementally copy declaration libraries to built/local npx hereby generate # Regenerate all sources, assets, localization, and vendored files npx hereby format # Format Go, TypeScript, JSON, and YAML npx hereby check:format # Check formatting without changing files @@ -117,6 +118,11 @@ to bypass incremental caches. `generate` includes every generator and fetches the pinned LSP model when its local cache is missing or stale. Use `generate:go` for the narrower scope of `go generate ./...` in the `tsc` module. +`generate` also includes `generate:libs`, which caches declaration libraries as +one set. When sources or copies change, it removes the destination's `lib*.d.ts` +files and recopies the current set, preserving non-library files. The `lib` task +remains an alias for `generate:libs`. + Package-specific commands: ```bash diff --git a/Herebyfile.mjs b/Herebyfile.mjs index ad6f1d06e88de..133a9d6cb4f08 100644 --- a/Herebyfile.mjs +++ b/Herebyfile.mjs @@ -223,19 +223,33 @@ const libsRegexp = /(?:^|[\\/])internal[\\/]bundled[\\/]libs[\\/]/; * @param {string} out */ async function generateLibs(out) { + const { GeneratedDirectory } = await import("./tools/scripts/gen/generatedFile.mts"); + out = path.resolve(__dirname, out); + const sourceDir = path.resolve(__dirname, libsDir); + const libs = await fs.promises.readdir(sourceDir); + const output = new GeneratedDirectory(out, [__filename, ...libs.map(lib => path.join(sourceDir, lib))], "lib*.d.ts"); + if (output.isCurrent(!!options.force)) { + console.log(`Library files in ${out} are up to date.`); + return; + } + output.invalidate(); await fs.promises.mkdir(out, { recursive: true }); - - const libs = await fs.promises.readdir(libsDir); - - await Promise.all(libs.map(async lib => { - fs.promises.copyFile(path.join(libsDir, lib), path.join(out, lib)); - })); + await Promise.all(fs.globSync("lib*.d.ts", { cwd: out }).map(lib => fs.promises.unlink(path.join(out, lib)))); + await Promise.all(libs.map(lib => fs.promises.copyFile(path.join(sourceDir, lib), path.join(out, lib)))); + output.markCurrent(); + console.log(`Copied ${libs.length} library files to ${out}.`); } +export const generateLib = task({ + name: "generate:libs", + description: "Copies the libs to built/local. Pass --force to recopy unchanged files.", + run: () => generateLibs(builtLocal), +}); + export const lib = task({ name: "lib", - description: "Copies the libs to built/local.", - run: () => generateLibs(builtLocal), + description: "Alias for generate:libs.", + dependencies: [generateLib], }); /** @@ -712,8 +726,8 @@ const generateCompiler = task({ export const generate = task({ name: "generate", - description: "Runs all code generation, including AST, LSP, APIs, extension localization, and vendored dependencies.", - dependencies: [generateCompiler, generateSync, generateExtensionTest, generateVendor], + description: "Runs all code generation, including AST, LSP, APIs, library copies, extension localization, and vendored dependencies.", + dependencies: [generateCompiler, generateSync, generateExtensionTest, generateVendor, generateLib], }); const coverageDir = path.join(__dirname, "coverage"); diff --git a/tools/scripts/gen/generatedFile.mts b/tools/scripts/gen/generatedFile.mts index 58fcfce5d499c..3cab0ae0d4d62 100644 --- a/tools/scripts/gen/generatedFile.mts +++ b/tools/scripts/gen/generatedFile.mts @@ -59,9 +59,13 @@ export class GeneratedFile { return JSON.stringify([this.inputHash, digest(output)]); } + protected readOutput(): Buffer | undefined { + return readIfExists(this.fileName); + } + isCurrent(force = false): boolean { if (force) return false; - const output = readIfExists(this.fileName); + const output = this.readOutput(); return output !== undefined && readIfExists(this.cacheFile)?.toString() === this.state(output); } @@ -79,8 +83,29 @@ export class GeneratedFile { markCurrent(): void { invalidateFileFingerprints(this.fileName); if (this.inputHash !== this.hashInputs(true)) return; - const state = this.state(fs.readFileSync(this.fileName)); + const output = this.readOutput(); + if (output === undefined) throw new Error(`Missing generated output: ${this.fileName}`); + const state = this.state(output); fs.mkdirSync(path.dirname(this.cacheFile), { recursive: true }); fs.writeFileSync(this.cacheFile, state); } } + +export class GeneratedDirectory extends GeneratedFile { + private readonly pattern: string; + + constructor(directory: string, inputs: readonly string[], pattern: string) { + super(directory, inputs, undefined, pattern); + this.pattern = pattern; + } + + protected override readOutput(): Buffer | undefined { + if (!fs.existsSync(this.fileName)) return undefined; + return Buffer.from(JSON.stringify( + fs.globSync(this.pattern, { cwd: this.fileName }).sort().map(file => [ + file, + digest(fs.readFileSync(path.join(this.fileName, file))), + ]), + )); + } +} diff --git a/tools/scripts/gen/generatedFile.test.mts b/tools/scripts/gen/generatedFile.test.mts index 283fd1d8233cc..0cfe45eaf9119 100644 --- a/tools/scripts/gen/generatedFile.test.mts +++ b/tools/scripts/gen/generatedFile.test.mts @@ -12,6 +12,7 @@ import ts from "typescript"; import cache from "./cache.mts"; import { defaultCacheDirectory, + GeneratedDirectory, GeneratedFile, } from "./generatedFile.mts"; import { @@ -133,6 +134,86 @@ test("generated files are current only while their inputs and formatted output m assert.equal(generated().isCurrent(), false); }); +test("generated directories track matching file names and contents", context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-directory-")); + const generated = () => new GeneratedDirectory(directory, [], "lib*.d.ts"); + const output = generated(); + context.after(() => { + output.invalidate(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + const first = path.join(directory, "lib.first.d.ts"); + const second = path.join(directory, "lib.second.d.ts"); + const unrelated = path.join(directory, "notes.txt"); + fs.writeFileSync(first, "first"); + fs.writeFileSync(unrelated, "unrelated"); + assert.equal(output.isCurrent(), false); + output.markCurrent(); + assert.equal(generated().isCurrent(), true); + fs.utimesSync(first, new Date(0), new Date(0)); + fs.writeFileSync(unrelated, "changed but ignored"); + assert.equal(generated().isCurrent(), true); + fs.rmSync(unrelated); + assert.equal(generated().isCurrent(), true); + fs.writeFileSync(first, "edited"); + assert.equal(generated().isCurrent(), false); + fs.writeFileSync(first, "first"); + assert.equal(generated().isCurrent(), true); + fs.renameSync(first, second); + assert.equal(generated().isCurrent(), false); + fs.writeFileSync(first, "first"); + assert.equal(generated().isCurrent(), false); + fs.rmSync(second); + assert.equal(generated().isCurrent(), true); + fs.rmSync(first); + assert.equal(generated().isCurrent(), false); +}); + +test("generated directories distinguish missing and empty outputs and support invalidation", context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-directory-")); + const destination = path.join(directory, "output"); + const generated = () => new GeneratedDirectory(destination, [], "*.d.ts"); + const output = generated(); + context.after(() => { + output.invalidate(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + assert.equal(output.isCurrent(), false); + assert.throws(() => output.markCurrent(), /Missing generated output/); + fs.mkdirSync(destination); + output.markCurrent(); + assert.equal(generated().isCurrent(), true); + assert.equal(generated().isCurrent(true), false); + assert.equal(new GeneratedDirectory(destination, [], "*.txt").isCurrent(), false); + output.invalidate(); + assert.equal(generated().isCurrent(), false); + output.markCurrent(); + assert.equal(generated().isCurrent(), true); + fs.rmdirSync(destination); + assert.equal(generated().isCurrent(), false); +}); + +test("generated directories reject stale inputs when marking current", context => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-directory-")); + const input = path.join(directory, "input.txt"); + fs.writeFileSync(input, "initial"); + fs.writeFileSync(path.join(directory, "lib.d.ts"), "generated"); + const generated = () => new GeneratedDirectory(directory, [input], "*.d.ts"); + const output = generated(); + context.after(() => { + output.invalidate(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + output.markCurrent(); + assert.equal(generated().isCurrent(), true); + fs.writeFileSync(input, "changed during generation"); + assert.equal(generated().isCurrent(), false); + output.markCurrent(); + assert.equal(generated().isCurrent(), false); + generated().markCurrent(); + assert.equal(generated().isCurrent(), true); +}); + test("generated files share input fingerprints and refresh changed inputs", context => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tsgo-fingerprint-")); context.after(() => fs.rmSync(directory, { recursive: true, force: true }));