diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ef248a39..d51a45cff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## Unreleased + +### Fixed + +- **`graft build` no longer aborts partway through very large repos.** The + breadth-tier extractor created a web-tree-sitter parser per file and never + freed it or the syntax tree; those live in the WASM heap, which is capped at + 2 GB, so on a C source base of ~65,000 files every parse after ~12,000 failed + with `Aborted()`. One parser per grammar is now reused and every tree is + deleted after extraction, in the container (`.vue`) tier as well as the + breadth tier. If the runtime does abort, the build reports it once and says + to restart, instead of once per remaining file. +- **A parse failure is retried on the next `graft build`** instead of being + replayed from the extract cache until the file's bytes change, so a one-off + failure no longer needs a manual cache delete to clear. +- **A `.vue` wrapper grammar that throws is now a build error** for that file, + not a silently symbol-less file cached as a success. +- **A deeply nested syntax tree no longer fails with `Maximum call stack size + exceeded`.** A C++ header with a large brace initializer misparses under the + C grammar into a tree one level deep per element; the breadth-tier walks now + use an explicit stack instead of recursing. + ## 0.19.0 ### Added diff --git a/src/cli.ts b/src/cli.ts index 54e97a9f2..e7656d563 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -583,7 +583,11 @@ program }, { repo: buildRoot }, ); - for (const e of g.errors) console.error(`✗ ${e}`); + // A poisoned WASM runtime or a broken grammar can fail tens of thousands of + // files identically; the first 20 say everything the rest would. + const MAX_ERRORS_SHOWN = 20; + for (const e of g.errors.slice(0, MAX_ERRORS_SHOWN)) console.error(`✗ ${e}`); + if (g.errors.length > MAX_ERRORS_SHOWN) console.error(`✗ … and ${g.errors.length - MAX_ERRORS_SHOWN} more (${g.errors.length} files failed to parse)`); const rel = relative(process.cwd(), g.contextDir) || "graft"; if (process.env.GRAFT_NO_GITIGNORE) { diff --git a/src/graph/build.ts b/src/graph/build.ts index b3094051d..cf316d793 100644 --- a/src/graph/build.ts +++ b/src/graph/build.ts @@ -107,7 +107,7 @@ export interface GraphBuildResult { /** Per-file wiring cards written (Tier-2 passive surface). */ cards: number; files: number; - /** Files re-parsed this run (the rest were replayed from the extraction cache). */ + /** Files parsed this run (the rest were replayed from the extraction cache). Includes files whose previous build recorded a parse error: those are always re-attempted. */ parsed: number; /** Files replayed from the extraction cache. */ reused: number; @@ -244,14 +244,19 @@ export async function buildGraph( } const hash = contentHash(source); - if (cached && hash === cached.hash) { + // A cached FAILURE is never replayed. Most parse failures are not properties + // of the file: a WASM grammar that aborted because the heap was exhausted by + // the files before it fails again only if the heap is exhausted again. Replaying + // the error would pin it to the file until its bytes changed; on a 65k-file + // repo that was 52k files stuck failing after a single abort. Re-parsing a + // genuinely broken file costs one parse per build, which is nothing. (The + // pre-query refresh never rebuilds solely to retry an errored file: + // fingerprint.ts keeps the entry's real hash on its stat fast path. A rebuild + // triggered by other drift re-parses it like any build.) + if (cached && hash === cached.hash && !cached.error) { entries[rel] = { ...cached, size: f.size, mtimeMs: f.mtimeMs }; sources.set(rel, source); reused++; - if (cached.error) { - errors.push(cached.error); // this file failed to parse last time too - return; - } nodes.push(...cached.nodes); rawEdges.push(...cached.rawEdges); langs.add(label); diff --git a/src/graph/container.ts b/src/graph/container.ts index 3015069e7..2e1c63ac0 100644 --- a/src/graph/container.ts +++ b/src/graph/container.ts @@ -22,7 +22,7 @@ * fixtures whose true line numbers are known. */ import { extractFile, mintId, type ExtractResult, type Language, type RawEdge } from "./extract.js"; -import { loadWasmLanguage, parseWasm, type TsNode } from "./generic.js"; +import { loadWasmLanguage, parseWasm, type TsNode, type WasmTree } from "./generic.js"; import { contentHash } from "../util/id.js"; import type { NodeV1 } from "./types.js"; @@ -66,6 +66,14 @@ export function containerExtensions(): string[] { const loaded = new Map(); +/** Test seam, same shape as generic.ts's `swapGrammarForTest`. */ +export function swapContainerGrammarForTest(name: string, language: unknown | null): unknown | null { + const prev = loaded.get(name) ?? null; + if (language) loaded.set(name, language); + else loaded.delete(name); + return prev; +} + /** Warm the container grammars this repo needs. Same contract as * `warmGenericGrammars`: await once before the synchronous parse loop, and an * unavailable grammar is skipped rather than fatal (its files then extract to a @@ -144,9 +152,11 @@ function blocks(root: TsNode, lang: ContainerLang): TsNode[] { /** * Extract one container file. Synchronous; needs the grammar pre-warmed. * - * Never throws: a missing grammar, an unparseable SFC or a script block the - * inner extractor chokes on all degrade to "fewer nodes", because a build must - * not fail over one component. + * A missing grammar, or a script block the inner extractor rejects, still + * degrades to "fewer nodes", because a build must not fail over one component. + * But a wrapper grammar that THROWS (a WASM abort, not a per-file syntax error) + * propagates as a build error: swallowing it would cache a clean symbol-less + * file as success, the exact mistake the generic tier used to make. */ export function extractContainer(rel: string, source: string, lang: ContainerLang): ExtractResult { const nodes: NodeV1[] = []; @@ -154,51 +164,66 @@ export function extractContainer(rel: string, source: string, lang: ContainerLan const residuals: string[] = []; const language = loaded.get(lang.name); - const root = language ? parseWasm(language, source) : null; - - if (root) { - // Ids are minted per file by the inner extractor, so two script blocks that - // both define `setup` would collide. Threading one set across the blocks - // makes the second one `path#setup~2`, and the rename is applied to that - // block's edges too so nothing points at an id that no longer exists. - const minted = new Set([rel]); - - for (const body of blocks(root, lang)) { - const script = source.slice(body.startIndex, body.endIndex); - - let inner: ExtractResult; - try { - inner = extractFile(rel, script, lang.inner); - } catch { - continue; // one bad block, not a bad build - } - - // `raw_text` starts immediately after the `>` of the opening tag, so its - // row IS the tag's row and the slice begins with that line's newline. - // Script line 1 is therefore the tail of the tag line, and script line N - // lands on `.vue` line row + N — which is exactly "add the start row to a - // 1-based span". Taking the row from the tag node instead would look - // equivalent and be right only when the tag has no attributes. - const shift = body.startPosition.row; - - // nodes[0] is the script's own file node: it describes the block, not the - // file, so it is dropped and its residual folded into the .vue file node. - const [scriptFile, ...symbols] = inner.nodes; - if (scriptFile?.body_text) residuals.push(scriptFile.body_text); - - const renamed = new Map(); - for (const node of symbols) { - const id = mintId(node.id, minted); - if (id !== node.id) renamed.set(node.id, id); - nodes.push({ ...node, id, span: shiftSpan(node.span, shift) }); - } + let tree: WasmTree | null = null; + if (language) { + try { + tree = parseWasm(language, source); + } catch (err) { + // Same contract as the generic tier: a grammar that throws is a build error + // for this file, never a clean file node cached as success. + throw new Error(`${lang.name} grammar threw: ${err instanceof Error ? err.message : String(err)}`); + } + } - for (const edge of inner.rawEdges) { - const source_ = renamed.get(edge.source) ?? edge.source; - const targetId = edge.targetId === undefined ? undefined : (renamed.get(edge.targetId) ?? edge.targetId); - rawEdges.push({ ...edge, source: source_, ...(targetId === undefined ? {} : { targetId }) }); + try { + if (tree) { + const root = tree.rootNode; + // Ids are minted per file by the inner extractor, so two script blocks that + // both define `setup` would collide. Threading one set across the blocks + // makes the second one `path#setup~2`, and the rename is applied to that + // block's edges too so nothing points at an id that no longer exists. + const minted = new Set([rel]); + + for (const body of blocks(root, lang)) { + const script = source.slice(body.startIndex, body.endIndex); + + let inner: ExtractResult; + try { + inner = extractFile(rel, script, lang.inner); + } catch { + continue; // one bad block, not a bad build + } + + // `raw_text` starts immediately after the `>` of the opening tag, so its + // row IS the tag's row and the slice begins with that line's newline. + // Script line 1 is therefore the tail of the tag line, and script line N + // lands on `.vue` line row + N — which is exactly "add the start row to a + // 1-based span". Taking the row from the tag node instead would look + // equivalent and be right only when the tag has no attributes. + const shift = body.startPosition.row; + + // nodes[0] is the script's own file node: it describes the block, not the + // file, so it is dropped and its residual folded into the .vue file node. + const [scriptFile, ...symbols] = inner.nodes; + if (scriptFile?.body_text) residuals.push(scriptFile.body_text); + + const renamed = new Map(); + for (const node of symbols) { + const id = mintId(node.id, minted); + if (id !== node.id) renamed.set(node.id, id); + nodes.push({ ...node, id, span: shiftSpan(node.span, shift) }); + } + + for (const edge of inner.rawEdges) { + const source_ = renamed.get(edge.source) ?? edge.source; + const targetId = edge.targetId === undefined ? undefined : (renamed.get(edge.targetId) ?? edge.targetId); + rawEdges.push({ ...edge, source: source_, ...(targetId === undefined ? {} : { targetId }) }); + } } } + } finally { + // The wrapper tree is a WASM-heap object; nothing frees it but this. + tree?.delete(); } // Built last so it can carry the residual, but unshifted first so the file node diff --git a/src/graph/generic.ts b/src/graph/generic.ts index b599deb77..840418652 100644 --- a/src/graph/generic.ts +++ b/src/graph/generic.ts @@ -83,13 +83,50 @@ const KIND: Record = { object: "class", property: "variable", // Scala object; Swift/Scala property }; +/** The parse-side of web-tree-sitter graft touches. Both are Emscripten-heap + * objects: nothing frees them but an explicit `delete()`. Declared here rather + * than imported so a fake can stand in for tests. `new Parser()` is assignable + * to `WasmParser` without a cast; if an upgrade breaks that, this is where it + * shows. */ +export interface WasmTree { rootNode: TsNode; delete(): void } +export interface WasmParser { + parse(cb: (index: number) => string): WasmTree | null; + reset(): void; + delete(): void; +} // Loaded grammars + compiled tags queries, keyed by graft lang name. Populated by // warmGenericGrammars; read synchronously by extractGeneric. -export interface Loaded { language: unknown; query: unknown | null } +/** One warmed grammar. `parser` is created once per grammar and reused for every + * file: creating one per parse and never deleting it is what exhausted the 2 GB + * WASM heap on a 65k-file repo (`Aborted()` on every file after ~12k). */ +export interface Loaded { language: unknown; query: unknown | null; parser: WasmParser } const loaded = new Map(); let tsMod: typeof import("web-tree-sitter") | null = null; let initPromise: Promise | null = null; +/** The first abort message web-tree-sitter produced in this process, if any. + * Emscripten's abort() sets a process-wide flag and the heap never recovers, so + * after one real abort every further parse is doomed: better one clear error + * than one per remaining file. */ +let poisoned: string | null = null; +export function wasmRuntimePoisoned(): string | null { return poisoned; } +export function clearWasmPoisonForTest(): void { poisoned = null; } +function notePoison(err: unknown): string { + const msg = err instanceof Error ? err.message : String(err); + // WebAssembly is a real Node runtime global but is NOT in this project's TS lib + // (ES2022, no DOM), so read its RuntimeError constructor off globalThis through a + // locally typed view. A `declare global { namespace WebAssembly … }` augmentation + // would conflict with lib.dom's `interface`+`var` for downstream consumers and be + // emitted verbatim into dist/*.d.ts; this keeps the type strictly local. + const RuntimeError = (globalThis as { WebAssembly?: { RuntimeError?: abstract new (...args: never[]) => Error } }) + .WebAssembly?.RuntimeError; + if ((RuntimeError !== undefined && err instanceof RuntimeError) || /^Aborted\(/.test(msg)) poisoned = poisoned ?? msg; + return msg; +} +function poisonedError(langName: string): Error { + return new Error(`${langName} grammar unavailable: web-tree-sitter aborted earlier in this process (${poisoned}); restart the process and rebuild`); +} + function requireWasm(wasm: string): Buffer | null { // Resolve the grammar wasm from the tree-sitter-wasm bundle (its package.json // `exports` maps the bare "/…" subpath to the actual "out//…" file). @@ -141,7 +178,9 @@ export async function warmGenericGrammars(langNames: Iterable): Promise< if (scm) { try { query = new Query(language, scm); } catch { query = null; } } - loaded.set(name, { language, query }); + const parser = new tsMod.Parser(); + parser.setLanguage(language); + loaded.set(name, { language, query, parser }); } catch { /* grammar failed to instantiate — skip; files extract as file-only */ } @@ -165,6 +204,10 @@ export function swapGrammarForTest(name: string, entry: Loaded | null): Loaded | return prev; } +/** Test seam: pre-seed the parser parseWasm will use for `language`, so a + * throwing parse() can be driven without a real crashing grammar. */ +export function seedWasmParserForTest(language: object, parser: WasmParser): void { parsersByLanguage.set(language, parser); } + /** Load one grammar from the tree-sitter-wasms bundle, initialising * web-tree-sitter on first call. Null when the wasm is missing or won't * instantiate — never throws, so a caller degrades instead of failing the build. @@ -189,18 +232,30 @@ export async function loadWasmLanguage(wasm: string): Promise { const PARSE_CHUNK = 16384; // <32KB slices — same tree-sitter limit workaround as extract.ts -/** Parse with an already-loaded grammar. Returns the root node, or null if the - * grammar was never warmed or the parse blew up. Companion to - * `loadWasmLanguage` for callers outside this module. */ -export function parseWasm(language: unknown, source: string): TsNode | null { - if (!tsMod) return null; +/** One shared parser per language object, for callers that loaded a grammar via + * `loadWasmLanguage` (the container tier) rather than `warmGenericGrammars`. */ +const parsersByLanguage = new WeakMap(); + +/** Parse with an already-loaded grammar. Returns the tree, or null only when + * web-tree-sitter was never initialised. Throws when the parse blows up, so a + * WASM abort is a build error and not a silently symbol-less file. The CALLER + * owns the tree and must `delete()` it: it lives in the WASM heap, not V8's. */ +export function parseWasm(language: unknown, source: string): WasmTree | null { + if (!tsMod || typeof language !== "object" || language === null) return null; + if (poisoned) throw poisonedError("wasm"); + let parser = parsersByLanguage.get(language); + if (!parser) { + const p = new tsMod.Parser(); + p.setLanguage(language as never); + parser = p; + parsersByLanguage.set(language, parser); + } try { - const parser = new tsMod.Parser(); - parser.setLanguage(language as never); - const tree = parser.parse((i: number) => source.slice(i, i + PARSE_CHUNK)); - return (tree?.rootNode as TsNode) ?? null; - } catch { - return null; + return parser.parse((i: number) => source.slice(i, i + PARSE_CHUNK)); + } catch (err) { + const msg = notePoison(err); + try { parser.reset(); } catch { /* runtime gone */ } + throw new Error(msg); } } @@ -225,63 +280,101 @@ export function extractGeneric(rel: string, source: string, langName: string): E const entry = loaded.get(langName); if (!entry || !tsMod) return { nodes, rawEdges }; - let tree; + if (poisoned) throw poisonedError(langName); + let tree: WasmTree | null; try { - const parser = new tsMod.Parser(); - parser.setLanguage(entry.language as never); - tree = parser.parse((i: number) => source.slice(i, i + PARSE_CHUNK)); + tree = entry.parser.parse((i: number) => source.slice(i, i + PARSE_CHUNK)); } catch (err) { // A grammar that throws (e.g. a WASM "memory access out of bounds" from an // external scanner) is not a per-file syntax problem: swallowing it here left // the file with only its file node and the build reporting success. Rethrow so - // build.ts records it in `errors`, the CLI prints it, and the extract cache - // remembers the failure instead of caching an empty result as clean. - throw new Error(`${langName} grammar threw: ${err instanceof Error ? err.message : String(err)}`); + // build.ts records it in `errors` and the CLI prints it. + // + // The parser is shared across files, so put it back to a known state first: + // a parse that threw mid-way can leave the stack half-built, and the next + // file would inherit it. + const msg = notePoison(err); + try { entry.parser.reset(); } catch { /* the runtime is already gone */ } + throw new Error(`${langName} grammar threw: ${msg}`); } if (!tree) return { nodes, rawEdges }; - const minted = new Set([rel]); - const lines = source.split("\n"); - const defs: Def[] = []; - // One definition per source span: a grammar's tags.scm can capture the same node - // under two @definition kinds (Swift `func` is method AND function), and the - // walker can revisit a nested match — both would emit near-duplicate nodes. - const spanSeen = new Set(); - // Mint one definition node from a whole-definition tree node. Shared by both - // the tags.scm path and the walker fallback so id/span/signature/body_text are - // built identically. - const mkDef = (name: string, kind: Kind, whole: TsNode): void => { - if (spanSeen.has(whole.startIndex)) return; - spanSeen.add(whole.startIndex); - const idBase = `${rel}#${name}`; - let id = idBase, n = 2; - while (minted.has(id)) id = `${idBase}~${n++}`; - minted.add(id); - const startRow = whole.startPosition.row, endRow = whole.endPosition.row; - const sigLine = (lines[startRow] ?? "").trim().replace(/\s*\{?\s*$/, ""); - nodes.push({ - id, name, kind, path: rel, - span: `L${startRow + 1}-L${endRow + 1}`, - signature: sigLine || null, exported: true, origin: "generic", - body_hash: contentHash(source.slice(whole.startIndex, whole.endIndex)), - body_text: source.slice(whole.startIndex, whole.endIndex).replace(/\s+/g, " ").slice(0, 5000), - summary_state: "pending", summary: null, crux: null, - }); - defs.push({ id, startIndex: whole.startIndex, endIndex: whole.endIndex }); - }; + try { + const minted = new Set([rel]); + const lines = source.split("\n"); + const defs: Def[] = []; + // One definition per source span: a grammar's tags.scm can capture the same node + // under two @definition kinds (Swift `func` is method AND function), and the + // walker can revisit a nested match — both would emit near-duplicate nodes. + const spanSeen = new Set(); + // Mint one definition node from a whole-definition tree node. Shared by both + // the tags.scm path and the walker fallback so id/span/signature/body_text are + // built identically. + const mkDef = (name: string, kind: Kind, whole: TsNode): void => { + if (spanSeen.has(whole.startIndex)) return; + spanSeen.add(whole.startIndex); + const idBase = `${rel}#${name}`; + let id = idBase, n = 2; + while (minted.has(id)) id = `${idBase}~${n++}`; + minted.add(id); + const startRow = whole.startPosition.row, endRow = whole.endPosition.row; + const sigLine = (lines[startRow] ?? "").trim().replace(/\s*\{?\s*$/, ""); + nodes.push({ + id, name, kind, path: rel, + span: `L${startRow + 1}-L${endRow + 1}`, + signature: sigLine || null, exported: true, origin: "generic", + body_hash: contentHash(source.slice(whole.startIndex, whole.endIndex)), + body_text: source.slice(whole.startIndex, whole.endIndex).replace(/\s+/g, " ").slice(0, 5000), + summary_state: "pending", summary: null, crux: null, + }); + defs.push({ id, startIndex: whole.startIndex, endIndex: whole.endIndex }); + }; + + if (entry.query) { + tagsExtract(entry.query, tree.rootNode as TsNode, rel, mkDef, defs, rawEdges, langName); + } else { + walkExtract(tree.rootNode as TsNode, mkDef); // no tags.scm → symbols only + } + // The preprocessor is invisible to tags.scm, but in C/C++ a local `#include "x.h"` + // IS the dependency graph — capture it as a file→file import. Likewise a Rust + // `use crate::…` is an in-crate module dependency. + if (langName === "c" || langName === "cpp") extractIncludes(tree.rootNode as TsNode, rel, rawEdges); + else if (langName === "rust") extractUses(tree.rootNode as TsNode, rel, rawEdges); + else if (langName === "php") extractPhpUses(tree.rootNode as TsNode, rel, rawEdges); + return { nodes, rawEdges }; + } catch (err) { + // The walk is WASM-backed too (query.matches() runs in the grammar's runtime), + // so a real abort can surface here, not only from parse(). Route it through + // notePoison so one abort poisons the runtime and every later file gets the + // stable "aborted earlier" error instead of re-attempting a dead runtime. + // Rethrow the ORIGINAL error object unchanged so an ordinary walk failure keeps + // its message (build.ts records it per file). + notePoison(err); + throw err; + } finally { + // Every tree lives in the Emscripten heap until deleted. Without this, a + // 65k-file build leaks ~2 GB and tree-sitter's allocator calls abort(). + // Swallow a throw from delete(): the primary error (from the try or the catch) + // must never be masked by a delete failure, and a delete() that throws means the + // runtime is already gone — there is nothing left to free. + try { tree.delete(); } catch { /* runtime already gone; keep the real error */ } + } +} - if (entry.query) { - tagsExtract(entry.query, tree.rootNode as TsNode, rel, mkDef, defs, rawEdges, langName); - } else { - walkExtract(tree.rootNode as TsNode, mkDef); // no tags.scm → symbols only +/** Pre-order DFS over `root` and its named descendants, on an explicit stack: a + * misparse can nest one level per element (a C++ array initializer under the C + * grammar runs thousands deep), which overflows V8's stack if the walk recurses. Children + * are pushed in reverse so the visit order matches a recursive walk. */ +function forEachNamedNode(root: TsNode, fn: (n: TsNode) => void): void { + const stack: TsNode[] = [root]; + while (stack.length) { + const n = stack.pop()!; + fn(n); + for (let i = (n.namedChildCount ?? 0) - 1; i >= 0; i--) { + const c = n.namedChild?.(i); + if (c) stack.push(c); + } } - // The preprocessor is invisible to tags.scm, but in C/C++ a local `#include "x.h"` - // IS the dependency graph — capture it as a file→file import. Likewise a Rust - // `use crate::…` is an in-crate module dependency. - if (langName === "c" || langName === "cpp") extractIncludes(tree.rootNode as TsNode, rel, rawEdges); - else if (langName === "rust") extractUses(tree.rootNode as TsNode, rel, rawEdges); - else if (langName === "php") extractPhpUses(tree.rootNode as TsNode, rel, rawEdges); - return { nodes, rawEdges }; } /** PHP `use App\Models\User;` → a file→class-file `imports` raw edge, one per imported @@ -289,16 +382,11 @@ export function extractGeneric(rel: string, source: string, langName: string): E * those name a symbol, not a PSR-4 class file. resolve.ts settles the fully-qualified * name to the in-repo file by namespace suffix, and drops it when it can't. */ function extractPhpUses(root: TsNode, rel: string, rawEdges: RawEdge[]): void { - const visit = (n: TsNode): void => { + forEachNamedNode(root, (n) => { if (n.type === "namespace_use_declaration") { for (const fqn of phpUseNames(n.text)) rawEdges.push({ source: rel, relation: "imports", specifier: fqn, file: rel }); } - for (let i = 0; i < (n.namedChildCount ?? 0); i++) { - const c = n.namedChild?.(i); - if (c) visit(c); - } - }; - visit(root); + }); } /** The fully-qualified class names a PHP `use` declaration imports. Handles a plain @@ -322,17 +410,12 @@ function phpUseNames(text: string): string[] { * `super::`, `self::`, external crates, and globs are skipped — resolve.ts settles the * path against the file's crate root, and drops it when it can't. */ function extractUses(root: TsNode, rel: string, rawEdges: RawEdge[]): void { - const visit = (n: TsNode): void => { + forEachNamedNode(root, (n) => { if (n.type === "use_declaration") { const spec = rustUseModule(n.text); if (spec !== null) rawEdges.push({ source: rel, relation: "imports", specifier: spec ? `crate/${spec}` : "crate", file: rel }); } - for (let i = 0; i < (n.namedChildCount ?? 0); i++) { - const c = n.namedChild?.(i); - if (c) visit(c); - } - }; - visit(root); + }); } /** The crate-relative path a Rust `use crate::…` names (`::`→`/`), or null when it is not @@ -357,7 +440,7 @@ function rustUseModule(text: string): string | null { * in-repo header (relative to the including file, else a unique path-suffix match), and * keeps it as an external string when it cannot — never a guessed edge. */ function extractIncludes(root: TsNode, rel: string, rawEdges: RawEdge[]): void { - const visit = (n: TsNode): void => { + forEachNamedNode(root, (n) => { if (n.type === "preproc_include") { const raw = n.childForFieldName?.("path")?.text ?? ""; if (raw.startsWith('"')) { @@ -365,12 +448,7 @@ function extractIncludes(root: TsNode, rel: string, rawEdges: RawEdge[]): void { if (spec) rawEdges.push({ source: rel, relation: "imports", specifier: spec, file: rel }); } } - for (let i = 0; i < (n.namedChildCount ?? 0); i++) { - const c = n.namedChild?.(i); - if (c) visit(c); - } - }; - visit(root); + }); } /** tags.scm path: @definition. → nodes, @reference.call/@reference.send → @@ -471,18 +549,13 @@ function nodeName(node: TsNode): string | null { /** Walker fallback: DFS every named node, emit a def for each classified one * (symbols only — no call resolution without a query). */ function walkExtract(root: TsNode, mkDef: (name: string, kind: Kind, whole: TsNode) => void): void { - const visit = (n: TsNode): void => { + forEachNamedNode(root, (n) => { const kind = classifyKind(n.type); if (kind) { const name = nodeName(n); if (name) mkDef(name, kind, n); } - for (let i = 0; i < (n.namedChildCount ?? 0); i++) { - const c = n.namedChild?.(i); - if (c) visit(c); - } - }; - visit(root); + }); } /** Minimal structural view of a web-tree-sitter node — shared with container.ts. */ diff --git a/test/container-extract.test.ts b/test/container-extract.test.ts index f5855fc3b..ffc8ff898 100644 --- a/test/container-extract.test.ts +++ b/test/container-extract.test.ts @@ -23,6 +23,8 @@ import { containerLangOf, containerExtensions, isContainerWarm, + CONTAINER_LANGS, + swapContainerGrammarForTest, } from "../src/graph/container.js"; import { supportedExtensions } from "../src/graph/source-files.js"; import { buildGraph } from "../src/graph/build.js"; @@ -290,3 +292,17 @@ test("container: a clean build of a .vue file checks as in sync", async () => { assert.deepEqual(check.changed, []); assert.equal(check.ok, true, "a clean build checks OK"); }); + +test("a wrapper grammar that throws is a build error, not a clean symbol-less file", async () => { + await warmContainerGrammars(["vue"]); + const vue = CONTAINER_LANGS.find((l) => l.name === "vue")!; + const prev = swapContainerGrammarForTest("vue", new Proxy({}, { get(): never { throw new Error("memory access out of bounds (fake)"); } })); + try { + assert.throws( + () => extractContainer("App.vue", "\n", vue), + /vue grammar threw: memory access out of bounds/, + ); + } finally { + swapContainerGrammarForTest("vue", prev); + } +}); diff --git a/test/generic-extract.test.ts b/test/generic-extract.test.ts index aa7498264..ce8323752 100644 --- a/test/generic-extract.test.ts +++ b/test/generic-extract.test.ts @@ -17,7 +17,12 @@ import { loadWasmLanguage, parseWasm, swapGrammarForTest, + seedWasmParserForTest, + wasmRuntimePoisoned, + clearWasmPoisonForTest, type TsNode, + type Loaded, + type WasmParser, } from "../src/graph/generic.js"; import { resolveEdges } from "../src/graph/resolve.js"; import { buildGraph } from "../src/graph/build.js"; @@ -213,6 +218,26 @@ test("breadth tier: C #include becomes a resolved file→file import (local only assert.deepEqual(targetsOf("src/amb.c"), ["util.h"], "ambiguous include kept external, never guessed"); }); +// A C++ header routed to the C grammar misparses a big brace initializer into +// right-nested comma_expressions: the tree is as deep as the array is long. The +// walks over it must not recurse per level, or V8's stack overflows (RangeError). +test("breadth tier: a C header with a 20,000-element initializer extracts without overflowing the stack", async () => { + await warmGenericGrammars(["c"]); + const n = 20000; + const values = Array.from({ length: n }, (_, i) => String(i + 1)).join(", "); + const src = + '#include "dep.h"\n' + + "#include \n" + + `std::array data = { ${values} };\n` + + "int after(void) { return 0; }\n"; + const { nodes, rawEdges } = extractGeneric("big.h", src, "c"); + assert.ok( + rawEdges.some((e) => e.relation === "imports" && e.specifier === "dep.h"), + "the quoted include before the array is captured", + ); + assert.ok(nodes.some((x) => x.kind === "function" && x.name === "after"), "the function after the array is extracted"); +}); + // Rust `use crate::…` → a file→module import, resolved against the file's crate root // (the lib.rs/main.rs dir). The longest-prefix rule disambiguates a module from an item // and a `foo.rs` from a `foo/mod.rs`; std/super/external/glob are skipped, and an @@ -397,13 +422,18 @@ function namedOfType(root: TsNode, type: string): string[] { async function assertPhpWasmExtractsClassAndMethods(source: string, className: string, label: string): Promise { const language = await loadWasmLanguage("php"); assert.ok(language, "tree-sitter-wasm must ship a php grammar"); - const root = parseWasm(language, source); - assert.ok(root, `${label}: PHP wasm parse must not crash (1.1.4 threw on heredoc/nowdoc)`); - const classes = namedOfType(root, "class_declaration"); - const methods = namedOfType(root, "method_declaration"); - assert.ok(classes.includes(className), `${label}: expected class ${className}, got: ${classes.join(", ") || "(none)"}`); - assert.ok(methods.includes("sql"), `${label}: expected method sql, got: ${methods.join(", ") || "(none)"}`); - assert.ok(methods.includes("other"), `${label}: expected method other, got: ${methods.join(", ") || "(none)"}`); + const tree = parseWasm(language, source); + assert.ok(tree, `${label}: PHP wasm parse must not crash (1.1.4 threw on heredoc/nowdoc)`); + try { + const root = tree.rootNode; + const classes = namedOfType(root, "class_declaration"); + const methods = namedOfType(root, "method_declaration"); + assert.ok(classes.includes(className), `${label}: expected class ${className}, got: ${classes.join(", ") || "(none)"}`); + assert.ok(methods.includes("sql"), `${label}: expected method sql, got: ${methods.join(", ") || "(none)"}`); + assert.ok(methods.includes("other"), `${label}: expected method other, got: ${methods.join(", ") || "(none)"}`); + } finally { + tree.delete(); + } } test("PHP wasm grammar extracts class + methods from a heredoc file (#139)", async () => { @@ -421,12 +451,17 @@ test("PHP wasm grammar extracts class + methods from a nowdoc file (#139)", asyn * clean parse. The real crash is heap-state dependent, so a deterministically * throwing fake grammar is swapped in via the test seam instead. */ -const THROWING_GRAMMAR = { - // web-tree-sitter's setLanguage() inspects the language object, so any - // property access blowing up makes extractGeneric's try block throw exactly - // like a crashing wasm scanner does. - language: new Proxy({}, { get(): never { throw new Error("memory access out of bounds (fake)"); } }), +const THROWING_GRAMMAR: Loaded = { + // The parser is created once per grammar at warm time, so the per-file failure + // an external scanner produces is a throw out of parse(). reset() is what + // extractGeneric calls on the shared parser before rethrowing. + language: {}, query: null, + parser: { + parse(): never { throw new Error("memory access out of bounds (fake)"); }, + reset(): void {}, + delete(): void {}, + }, }; test("extractGeneric rethrows a throwing grammar with the language named (#139)", async () => { @@ -447,7 +482,7 @@ test("extractGeneric rethrows a throwing grammar with the language named (#139)" } }); -test("a throwing grammar is a per-file build error, cached as a failure (#139)", async () => { +test("a throwing grammar is a per-file build error, and the file is retried next build (#139)", async () => { const dir = mkdtempSync(join(tmpdir(), "graft-throwing-grammar-")); writeFileSync(join(dir, "lib.rs"), "pub fn f() {}\n"); await warmGenericGrammars(["rust"]); // so buildGraph's own warm call is a no-op @@ -461,13 +496,236 @@ test("a throwing grammar is a per-file build error, cached as a failure (#139)", assert.ok(g, "graph built"); assert.ok(!g!.nodes.some((n) => n.path === "lib.rs"), "failed file has no file node"); - // The extract cache must remember the failure, not an empty success: an - // incremental rebuild of the unchanged file replays the error. + // The failure is recorded (so the freshness probe is quiet) but NOT replayed: + // the second build parses the unchanged file again and, the grammar still + // throwing, reports the error again as a fresh parse, not a cache hit. const second = await buildGraph(dir, { reuse: true }); - assert.equal(second.parsed, 0, "unchanged file is not re-parsed"); - assert.equal(second.errors.length, 1, `error replayed (got: ${second.errors.join("; ")})`); + assert.equal(second.errors.length, 1, `error reported again (got: ${second.errors.join("; ")})`); assert.match(second.errors[0], /rust grammar threw/); + assert.equal(second.parsed, 1, "the errored file was re-parsed, not replayed"); + assert.equal(second.reused, 0, "an errored entry never counts as reused"); } finally { swapGrammarForTest("rust", prev); } }); + +test("a file that failed for an environmental reason recovers on the next build without a cache delete", async () => { + const dir = mkdtempSync(join(tmpdir(), "graft-recovering-grammar-")); + writeFileSync(join(dir, "lib.rs"), "pub fn f() {}\npub fn g() { f() }\n"); + await warmGenericGrammars(["rust"]); + const prev = swapGrammarForTest("rust", THROWING_GRAMMAR); + try { + const first = await buildGraph(dir, { reuse: false }); + assert.equal(first.errors.length, 1, "first build fails"); + } finally { + swapGrammarForTest("rust", prev); // the "environment" is healthy again + } + const second = await buildGraph(dir, { reuse: true }); + assert.equal(second.errors.length, 0, `second build clean (got: ${second.errors.join("; ")})`); + const g = readGraph(wiringPath(contextDirFor(dir))); + assert.ok(g!.nodes.some((n) => n.id === "lib.rs#f"), "the file's symbols are in the graph now"); +}); + +/** Wrap the real rust grammar so every tree it hands out is counted on the way + * out and on delete(). The parse itself is real; only the bookkeeping is fake. */ +function countingGrammar(real: Loaded, hooks: { created: number; deleted: number; resets: number }): Loaded { + return { + language: real.language, + query: real.query, + parser: { + parse(cb) { + const t = real.parser.parse(cb); + if (!t) return null; + hooks.created++; + return { rootNode: t.rootNode, delete: () => { hooks.deleted++; t.delete(); } }; + }, + reset: () => { hooks.resets++; real.parser.reset(); }, + delete: () => real.parser.delete(), + }, + }; +} + +test("extractGeneric deletes every tree it parses", async () => { + await warmGenericGrammars(["rust"]); + const hooks = { created: 0, deleted: 0, resets: 0 }; + const real = swapGrammarForTest("rust", null)!; + swapGrammarForTest("rust", countingGrammar(real, hooks)); + try { + for (let i = 0; i < 25; i++) extractGeneric("src/lib.rs", `pub fn f${i}() { g() }\n`, "rust"); + assert.equal(hooks.created, 25, "every call parsed"); + assert.equal(hooks.deleted, 25, "every tree deleted"); + assert.equal(hooks.resets, 0, "no reset on the happy path"); + } finally { + swapGrammarForTest("rust", real); + } +}); + +test("extractGeneric deletes the tree even when the walk throws", async () => { + await warmGenericGrammars(["rust"]); + const hooks = { created: 0, deleted: 0, resets: 0 }; + const real = swapGrammarForTest("rust", null)!; + // tagsExtract calls query.matches(root); a query that throws there throws mid-walk. + const grammar = countingGrammar(real, hooks); + grammar.query = { matches(): never { throw new Error("walk boom"); } }; + swapGrammarForTest("rust", grammar); + try { + assert.throws(() => extractGeneric("src/lib.rs", "pub fn f() {}\n", "rust"), /walk boom/); + assert.equal(hooks.created, 1); + assert.equal(hooks.deleted, 1, "the tree was deleted in finally"); + } finally { + swapGrammarForTest("rust", real); + } +}); + +test("a parse() that throws resets the shared parser before rethrowing", async () => { + await warmGenericGrammars(["rust"]); + const hooks = { created: 0, deleted: 0, resets: 0 }; + const real = swapGrammarForTest("rust", null)!; + const grammar = countingGrammar(real, hooks); + grammar.parser = { ...grammar.parser, parse(): never { throw new Error("memory access out of bounds (fake)"); } }; + swapGrammarForTest("rust", grammar); + try { + assert.throws(() => extractGeneric("src/lib.rs", "pub fn f() {}\n", "rust"), /rust grammar threw: memory access out of bounds/); + assert.equal(hooks.resets, 1, "reset() called once"); + assert.equal(wasmRuntimePoisoned(), null, "an ordinary throw does not poison the runtime"); + } finally { + swapGrammarForTest("rust", real); + } +}); + +test("a WebAssembly.RuntimeError poisons the runtime: every later call fails with one distinct message", async () => { + await warmGenericGrammars(["rust"]); + const real = swapGrammarForTest("rust", null)!; + const grammar: Loaded = { + language: real.language, query: real.query, + parser: { parse(): never { throw new WebAssembly.RuntimeError("Aborted(). Build with -sASSERTIONS for more info."); }, reset() {}, delete() {} }, + }; + swapGrammarForTest("rust", grammar); + try { + assert.throws(() => extractGeneric("src/lib.rs", "pub fn f() {}\n", "rust"), /rust grammar threw: Aborted\(\)/); + assert.match(wasmRuntimePoisoned() ?? "", /^Aborted\(\)/); + // The real grammar is back, but the process is poisoned: no parse is attempted. + swapGrammarForTest("rust", real); + assert.throws( + () => extractGeneric("src/lib.rs", "pub fn f() {}\n", "rust"), + /web-tree-sitter aborted earlier in this process .*restart/, + ); + } finally { + clearWasmPoisonForTest(); + swapGrammarForTest("rust", real); + } +}); + +test("a WebAssembly.RuntimeError from the walk poisons the runtime too", async () => { + await warmGenericGrammars(["rust"]); + const real = swapGrammarForTest("rust", null)!; + // query.matches() is WASM-backed, so an abort can surface from the walk, not + // only from parse(). It must poison the runtime the same way a parse abort does, + // and the thrown error must be the original object (its message carries Aborted()). + const grammar: Loaded = { + language: real.language, + query: { matches(): never { throw new WebAssembly.RuntimeError("Aborted(). Build with -sASSERTIONS for more info."); } }, + parser: real.parser, + }; + swapGrammarForTest("rust", grammar); + try { + assert.throws(() => extractGeneric("src/lib.rs", "pub fn f() {}\n", "rust"), /Aborted\(\)/); + assert.match(wasmRuntimePoisoned() ?? "", /^Aborted\(\)/); + // The real grammar is back, but the process is poisoned: no parse is attempted. + swapGrammarForTest("rust", real); + assert.throws( + () => extractGeneric("src/lib.rs", "pub fn f() {}\n", "rust"), + /web-tree-sitter aborted earlier in this process .*restart/, + ); + } finally { + clearWasmPoisonForTest(); + swapGrammarForTest("rust", real); + } +}); + +test("a tree.delete() that throws does not mask the walk's error", async () => { + await warmGenericGrammars(["rust"]); + const hooks = { created: 0, deleted: 0, resets: 0 }; + const real = swapGrammarForTest("rust", null)!; + // Like countingGrammar, but the tree's delete() throws AFTER counting, and the + // walk throws first. The delete throw must be swallowed so the walk's real error + // (not the delete error) is what propagates. + const grammar: Loaded = { + language: real.language, + query: { matches(): never { throw new Error("walk boom"); } }, + parser: { + parse(cb) { + const t = real.parser.parse(cb); + if (!t) return null; + hooks.created++; + return { rootNode: t.rootNode, delete: () => { hooks.deleted++; t.delete(); throw new Error("delete boom"); } }; + }, + reset: () => { hooks.resets++; real.parser.reset(); }, + delete: () => real.parser.delete(), + }, + }; + swapGrammarForTest("rust", grammar); + try { + assert.throws(() => extractGeneric("src/lib.rs", "pub fn f() {}\n", "rust"), /walk boom/); + assert.equal(hooks.deleted, 1, "delete() ran (and threw) but the throw was swallowed"); + } finally { + swapGrammarForTest("rust", real); + } +}); + +test("extractGeneric frees WASM memory: 400 parses of a 40 KB file do not grow RSS", async () => { + await warmGenericGrammars(["rust"]); + assert.ok(isWarm("rust"), "rust grammar warmed"); + const fn = (i: number) => `pub fn f${i}(a: u32, b: u32) -> u32 {\n let c = a + b;\n if c > 10 { helper(c) } else { c }\n}\n`; + const source = Array.from({ length: 400 }, (_, i) => fn(i)).join("\n"); + for (let i = 0; i < 20; i++) extractGeneric("src/lib.rs", source, "rust"); // warm-up + const before = process.memoryUsage().rss; + for (let i = 0; i < 400; i++) extractGeneric("src/lib.rs", source, "rust"); + const grewMb = (process.memoryUsage().rss - before) / 1048576; + // Leaking, each parse retains ~1.9 MB of WASM heap: 400 parses grow RSS by + // ~750 MB (and 1,100 parses would hit the 2 GB abort, so the loop stays at + // 400). Freed, growth is V8 noise. 100 MB is the line. + assert.ok(grewMb < 100, `RSS grew by ${grewMb.toFixed(0)} MB over 400 parses: the tree is not being deleted`); +}); + +// parseWasm has its OWN parse-catch path (generic.ts, distinct from extractGeneric's): +// notePoison → parser.reset() → throw. It is the path the container (.vue) tier hits. +// parseWasm keys a WeakMap by the language object and creates the +// parser itself on first use, so a fake language cannot make parse() throw on its own — +// seedWasmParserForTest pre-seeds the parser parseWasm will use, so a throwing parse() +// can be driven without a real crashing grammar. +test("parseWasm rethrows a throwing parse and resets the shared parser (an ordinary throw does not poison)", async () => { + await warmGenericGrammars(["rust"]); // initialises web-tree-sitter + const language = {}; + let resets = 0; + const parser: WasmParser = { + parse(): never { throw new Error("memory access out of bounds (fake)"); }, + reset(): void { resets++; }, + delete(): void {}, + }; + seedWasmParserForTest(language, parser); + assert.throws(() => parseWasm(language, "x"), /memory access out of bounds/); + assert.equal(resets, 1, "reset() called once before rethrow"); + assert.equal(wasmRuntimePoisoned(), null, "an ordinary throw does not poison the runtime"); +}); + +test("parseWasm poisons the runtime on a WebAssembly.RuntimeError; every later call fails with the aborted-earlier error", async () => { + await warmGenericGrammars(["rust"]); + const language = {}; + // The ES2022 lib has no WebAssembly type; read its RuntimeError constructor off globalThis. + const RuntimeError = (globalThis as { WebAssembly: { RuntimeError: new (m: string) => Error } }).WebAssembly.RuntimeError; + const parser: WasmParser = { + parse(): never { throw new RuntimeError("Aborted()."); }, + reset(): void {}, + delete(): void {}, + }; + seedWasmParserForTest(language, parser); + try { + assert.throws(() => parseWasm(language, "x"), /Aborted\(/); + assert.match(wasmRuntimePoisoned() ?? "", /^Aborted\(/); + // The runtime is poisoned now: a further parseWasm never attempts a parse. + assert.throws(() => parseWasm(language, "x"), /aborted earlier in this process/); + } finally { + clearWasmPoisonForTest(); + } +});