From d615a2609f1eb4f690e4185e295684d0469c233f Mon Sep 17 00:00:00 2001 From: Kane Bruce <54819806+csm-kb@users.noreply.github.com> Date: Thu, 10 Sep 2026 00:49:55 -0600 Subject: [PATCH 1/4] fix(generic): free web-tree-sitter parsers and trees, detect an aborted runtime extractGeneric and parseWasm created a Parser per file and never deleted it or the Tree. Both live in the WASM heap (2 GB cap, no finalizer), so a 65k-file C repo aborted after ~12k files and every later parse failed with Aborted(). Reuse one parser per grammar and delete every tree. After a real abort, fail every later parse with one message instead of one per file. parseWasm now returns the tree for the container tier to free, and rethrows instead of caching a symbol-less file as a success. --- src/graph/container.ts | 117 +++++++++++-------- src/graph/generic.ts | 193 ++++++++++++++++++++++---------- test/container-extract.test.ts | 16 +++ test/generic-extract.test.ts | 199 +++++++++++++++++++++++++++++++-- 4 files changed, 407 insertions(+), 118 deletions(-) diff --git a/src/graph/container.ts b/src/graph/container.ts index 3015069e7..2e1c63ac0 100644 --- a/src/graph/container.ts +++ b/src/graph/container.ts @@ -22,7 +22,7 @@ * fixtures whose true line numbers are known. */ import { extractFile, mintId, type ExtractResult, type Language, type RawEdge } from "./extract.js"; -import { loadWasmLanguage, parseWasm, type TsNode } from "./generic.js"; +import { loadWasmLanguage, parseWasm, type TsNode, type WasmTree } from "./generic.js"; import { contentHash } from "../util/id.js"; import type { NodeV1 } from "./types.js"; @@ -66,6 +66,14 @@ export function containerExtensions(): string[] { const loaded = new Map(); +/** Test seam, same shape as generic.ts's `swapGrammarForTest`. */ +export function swapContainerGrammarForTest(name: string, language: unknown | null): unknown | null { + const prev = loaded.get(name) ?? null; + if (language) loaded.set(name, language); + else loaded.delete(name); + return prev; +} + /** Warm the container grammars this repo needs. Same contract as * `warmGenericGrammars`: await once before the synchronous parse loop, and an * unavailable grammar is skipped rather than fatal (its files then extract to a @@ -144,9 +152,11 @@ function blocks(root: TsNode, lang: ContainerLang): TsNode[] { /** * Extract one container file. Synchronous; needs the grammar pre-warmed. * - * Never throws: a missing grammar, an unparseable SFC or a script block the - * inner extractor chokes on all degrade to "fewer nodes", because a build must - * not fail over one component. + * A missing grammar, or a script block the inner extractor rejects, still + * degrades to "fewer nodes", because a build must not fail over one component. + * But a wrapper grammar that THROWS (a WASM abort, not a per-file syntax error) + * propagates as a build error: swallowing it would cache a clean symbol-less + * file as success, the exact mistake the generic tier used to make. */ export function extractContainer(rel: string, source: string, lang: ContainerLang): ExtractResult { const nodes: NodeV1[] = []; @@ -154,51 +164,66 @@ export function extractContainer(rel: string, source: string, lang: ContainerLan const residuals: string[] = []; const language = loaded.get(lang.name); - const root = language ? parseWasm(language, source) : null; - - if (root) { - // Ids are minted per file by the inner extractor, so two script blocks that - // both define `setup` would collide. Threading one set across the blocks - // makes the second one `path#setup~2`, and the rename is applied to that - // block's edges too so nothing points at an id that no longer exists. - const minted = new Set([rel]); - - for (const body of blocks(root, lang)) { - const script = source.slice(body.startIndex, body.endIndex); - - let inner: ExtractResult; - try { - inner = extractFile(rel, script, lang.inner); - } catch { - continue; // one bad block, not a bad build - } - - // `raw_text` starts immediately after the `>` of the opening tag, so its - // row IS the tag's row and the slice begins with that line's newline. - // Script line 1 is therefore the tail of the tag line, and script line N - // lands on `.vue` line row + N — which is exactly "add the start row to a - // 1-based span". Taking the row from the tag node instead would look - // equivalent and be right only when the tag has no attributes. - const shift = body.startPosition.row; - - // nodes[0] is the script's own file node: it describes the block, not the - // file, so it is dropped and its residual folded into the .vue file node. - const [scriptFile, ...symbols] = inner.nodes; - if (scriptFile?.body_text) residuals.push(scriptFile.body_text); - - const renamed = new Map(); - for (const node of symbols) { - const id = mintId(node.id, minted); - if (id !== node.id) renamed.set(node.id, id); - nodes.push({ ...node, id, span: shiftSpan(node.span, shift) }); - } + let tree: WasmTree | null = null; + if (language) { + try { + tree = parseWasm(language, source); + } catch (err) { + // Same contract as the generic tier: a grammar that throws is a build error + // for this file, never a clean file node cached as success. + throw new Error(`${lang.name} grammar threw: ${err instanceof Error ? err.message : String(err)}`); + } + } - for (const edge of inner.rawEdges) { - const source_ = renamed.get(edge.source) ?? edge.source; - const targetId = edge.targetId === undefined ? undefined : (renamed.get(edge.targetId) ?? edge.targetId); - rawEdges.push({ ...edge, source: source_, ...(targetId === undefined ? {} : { targetId }) }); + try { + if (tree) { + const root = tree.rootNode; + // Ids are minted per file by the inner extractor, so two script blocks that + // both define `setup` would collide. Threading one set across the blocks + // makes the second one `path#setup~2`, and the rename is applied to that + // block's edges too so nothing points at an id that no longer exists. + const minted = new Set([rel]); + + for (const body of blocks(root, lang)) { + const script = source.slice(body.startIndex, body.endIndex); + + let inner: ExtractResult; + try { + inner = extractFile(rel, script, lang.inner); + } catch { + continue; // one bad block, not a bad build + } + + // `raw_text` starts immediately after the `>` of the opening tag, so its + // row IS the tag's row and the slice begins with that line's newline. + // Script line 1 is therefore the tail of the tag line, and script line N + // lands on `.vue` line row + N — which is exactly "add the start row to a + // 1-based span". Taking the row from the tag node instead would look + // equivalent and be right only when the tag has no attributes. + const shift = body.startPosition.row; + + // nodes[0] is the script's own file node: it describes the block, not the + // file, so it is dropped and its residual folded into the .vue file node. + const [scriptFile, ...symbols] = inner.nodes; + if (scriptFile?.body_text) residuals.push(scriptFile.body_text); + + const renamed = new Map(); + for (const node of symbols) { + const id = mintId(node.id, minted); + if (id !== node.id) renamed.set(node.id, id); + nodes.push({ ...node, id, span: shiftSpan(node.span, shift) }); + } + + for (const edge of inner.rawEdges) { + const source_ = renamed.get(edge.source) ?? edge.source; + const targetId = edge.targetId === undefined ? undefined : (renamed.get(edge.targetId) ?? edge.targetId); + rawEdges.push({ ...edge, source: source_, ...(targetId === undefined ? {} : { targetId }) }); + } } } + } finally { + // The wrapper tree is a WASM-heap object; nothing frees it but this. + tree?.delete(); } // Built last so it can carry the residual, but unshifted first so the file node diff --git a/src/graph/generic.ts b/src/graph/generic.ts index b599deb77..9202064c8 100644 --- a/src/graph/generic.ts +++ b/src/graph/generic.ts @@ -83,13 +83,50 @@ const KIND: Record = { object: "class", property: "variable", // Scala object; Swift/Scala property }; +/** The parse-side of web-tree-sitter graft touches. Both are Emscripten-heap + * objects: nothing frees them but an explicit `delete()`. Declared here rather + * than imported so a fake can stand in for tests. `new Parser()` is assignable + * to `WasmParser` without a cast; if an upgrade breaks that, this is where it + * shows. */ +export interface WasmTree { rootNode: TsNode; delete(): void } +export interface WasmParser { + parse(cb: (index: number) => string): WasmTree | null; + reset(): void; + delete(): void; +} // Loaded grammars + compiled tags queries, keyed by graft lang name. Populated by // warmGenericGrammars; read synchronously by extractGeneric. -export interface Loaded { language: unknown; query: unknown | null } +/** One warmed grammar. `parser` is created once per grammar and reused for every + * file: creating one per parse and never deleting it is what exhausted the 2 GB + * WASM heap on a 65k-file repo (`Aborted()` on every file after ~12k). */ +export interface Loaded { language: unknown; query: unknown | null; parser: WasmParser } const loaded = new Map(); let tsMod: typeof import("web-tree-sitter") | null = null; let initPromise: Promise | null = null; +/** The first abort message web-tree-sitter produced in this process, if any. + * Emscripten's abort() sets a process-wide flag and the heap never recovers, so + * after one real abort every further parse is doomed: better one clear error + * than one per remaining file. */ +let poisoned: string | null = null; +export function wasmRuntimePoisoned(): string | null { return poisoned; } +export function clearWasmPoisonForTest(): void { poisoned = null; } +function notePoison(err: unknown): string { + const msg = err instanceof Error ? err.message : String(err); + // WebAssembly is a real Node runtime global but is NOT in this project's TS lib + // (ES2022, no DOM), so read its RuntimeError constructor off globalThis through a + // locally typed view. A `declare global { namespace WebAssembly … }` augmentation + // would conflict with lib.dom's `interface`+`var` for downstream consumers and be + // emitted verbatim into dist/*.d.ts; this keeps the type strictly local. + const RuntimeError = (globalThis as { WebAssembly?: { RuntimeError?: abstract new (...args: never[]) => Error } }) + .WebAssembly?.RuntimeError; + if ((RuntimeError !== undefined && err instanceof RuntimeError) || /^Aborted\(/.test(msg)) poisoned = poisoned ?? msg; + return msg; +} +function poisonedError(langName: string): Error { + return new Error(`${langName} grammar unavailable: web-tree-sitter aborted earlier in this process (${poisoned}); restart the process and rebuild`); +} + function requireWasm(wasm: string): Buffer | null { // Resolve the grammar wasm from the tree-sitter-wasm bundle (its package.json // `exports` maps the bare "/…" subpath to the actual "out//…" file). @@ -141,7 +178,9 @@ export async function warmGenericGrammars(langNames: Iterable): Promise< if (scm) { try { query = new Query(language, scm); } catch { query = null; } } - loaded.set(name, { language, query }); + const parser = new tsMod.Parser(); + parser.setLanguage(language); + loaded.set(name, { language, query, parser }); } catch { /* grammar failed to instantiate — skip; files extract as file-only */ } @@ -189,18 +228,30 @@ export async function loadWasmLanguage(wasm: string): Promise { const PARSE_CHUNK = 16384; // <32KB slices — same tree-sitter limit workaround as extract.ts -/** Parse with an already-loaded grammar. Returns the root node, or null if the - * grammar was never warmed or the parse blew up. Companion to - * `loadWasmLanguage` for callers outside this module. */ -export function parseWasm(language: unknown, source: string): TsNode | null { - if (!tsMod) return null; +/** One shared parser per language object, for callers that loaded a grammar via + * `loadWasmLanguage` (the container tier) rather than `warmGenericGrammars`. */ +const parsersByLanguage = new WeakMap(); + +/** Parse with an already-loaded grammar. Returns the tree, or null only when + * web-tree-sitter was never initialised. Throws when the parse blows up, so a + * WASM abort is a build error and not a silently symbol-less file. The CALLER + * owns the tree and must `delete()` it: it lives in the WASM heap, not V8's. */ +export function parseWasm(language: unknown, source: string): WasmTree | null { + if (!tsMod || typeof language !== "object" || language === null) return null; + if (poisoned) throw poisonedError("wasm"); + let parser = parsersByLanguage.get(language); + if (!parser) { + const p = new tsMod.Parser(); + p.setLanguage(language as never); + parser = p; + parsersByLanguage.set(language, parser); + } try { - const parser = new tsMod.Parser(); - parser.setLanguage(language as never); - const tree = parser.parse((i: number) => source.slice(i, i + PARSE_CHUNK)); - return (tree?.rootNode as TsNode) ?? null; - } catch { - return null; + return parser.parse((i: number) => source.slice(i, i + PARSE_CHUNK)); + } catch (err) { + const msg = notePoison(err); + try { parser.reset(); } catch { /* runtime gone */ } + throw new Error(msg); } } @@ -225,63 +276,85 @@ export function extractGeneric(rel: string, source: string, langName: string): E const entry = loaded.get(langName); if (!entry || !tsMod) return { nodes, rawEdges }; - let tree; + if (poisoned) throw poisonedError(langName); + let tree: WasmTree | null; try { - const parser = new tsMod.Parser(); - parser.setLanguage(entry.language as never); - tree = parser.parse((i: number) => source.slice(i, i + PARSE_CHUNK)); + tree = entry.parser.parse((i: number) => source.slice(i, i + PARSE_CHUNK)); } catch (err) { // A grammar that throws (e.g. a WASM "memory access out of bounds" from an // external scanner) is not a per-file syntax problem: swallowing it here left // the file with only its file node and the build reporting success. Rethrow so - // build.ts records it in `errors`, the CLI prints it, and the extract cache - // remembers the failure instead of caching an empty result as clean. - throw new Error(`${langName} grammar threw: ${err instanceof Error ? err.message : String(err)}`); + // build.ts records it in `errors` and the CLI prints it. + // + // The parser is shared across files, so put it back to a known state first: + // a parse that threw mid-way can leave the stack half-built, and the next + // file would inherit it. + const msg = notePoison(err); + try { entry.parser.reset(); } catch { /* the runtime is already gone */ } + throw new Error(`${langName} grammar threw: ${msg}`); } if (!tree) return { nodes, rawEdges }; - const minted = new Set([rel]); - const lines = source.split("\n"); - const defs: Def[] = []; - // One definition per source span: a grammar's tags.scm can capture the same node - // under two @definition kinds (Swift `func` is method AND function), and the - // walker can revisit a nested match — both would emit near-duplicate nodes. - const spanSeen = new Set(); - // Mint one definition node from a whole-definition tree node. Shared by both - // the tags.scm path and the walker fallback so id/span/signature/body_text are - // built identically. - const mkDef = (name: string, kind: Kind, whole: TsNode): void => { - if (spanSeen.has(whole.startIndex)) return; - spanSeen.add(whole.startIndex); - const idBase = `${rel}#${name}`; - let id = idBase, n = 2; - while (minted.has(id)) id = `${idBase}~${n++}`; - minted.add(id); - const startRow = whole.startPosition.row, endRow = whole.endPosition.row; - const sigLine = (lines[startRow] ?? "").trim().replace(/\s*\{?\s*$/, ""); - nodes.push({ - id, name, kind, path: rel, - span: `L${startRow + 1}-L${endRow + 1}`, - signature: sigLine || null, exported: true, origin: "generic", - body_hash: contentHash(source.slice(whole.startIndex, whole.endIndex)), - body_text: source.slice(whole.startIndex, whole.endIndex).replace(/\s+/g, " ").slice(0, 5000), - summary_state: "pending", summary: null, crux: null, - }); - defs.push({ id, startIndex: whole.startIndex, endIndex: whole.endIndex }); - }; + try { + const minted = new Set([rel]); + const lines = source.split("\n"); + const defs: Def[] = []; + // One definition per source span: a grammar's tags.scm can capture the same node + // under two @definition kinds (Swift `func` is method AND function), and the + // walker can revisit a nested match — both would emit near-duplicate nodes. + const spanSeen = new Set(); + // Mint one definition node from a whole-definition tree node. Shared by both + // the tags.scm path and the walker fallback so id/span/signature/body_text are + // built identically. + const mkDef = (name: string, kind: Kind, whole: TsNode): void => { + if (spanSeen.has(whole.startIndex)) return; + spanSeen.add(whole.startIndex); + const idBase = `${rel}#${name}`; + let id = idBase, n = 2; + while (minted.has(id)) id = `${idBase}~${n++}`; + minted.add(id); + const startRow = whole.startPosition.row, endRow = whole.endPosition.row; + const sigLine = (lines[startRow] ?? "").trim().replace(/\s*\{?\s*$/, ""); + nodes.push({ + id, name, kind, path: rel, + span: `L${startRow + 1}-L${endRow + 1}`, + signature: sigLine || null, exported: true, origin: "generic", + body_hash: contentHash(source.slice(whole.startIndex, whole.endIndex)), + body_text: source.slice(whole.startIndex, whole.endIndex).replace(/\s+/g, " ").slice(0, 5000), + summary_state: "pending", summary: null, crux: null, + }); + defs.push({ id, startIndex: whole.startIndex, endIndex: whole.endIndex }); + }; - if (entry.query) { - tagsExtract(entry.query, tree.rootNode as TsNode, rel, mkDef, defs, rawEdges, langName); - } else { - walkExtract(tree.rootNode as TsNode, mkDef); // no tags.scm → symbols only + if (entry.query) { + tagsExtract(entry.query, tree.rootNode as TsNode, rel, mkDef, defs, rawEdges, langName); + } else { + walkExtract(tree.rootNode as TsNode, mkDef); // no tags.scm → symbols only + } + // The preprocessor is invisible to tags.scm, but in C/C++ a local `#include "x.h"` + // IS the dependency graph — capture it as a file→file import. Likewise a Rust + // `use crate::…` is an in-crate module dependency. + if (langName === "c" || langName === "cpp") extractIncludes(tree.rootNode as TsNode, rel, rawEdges); + else if (langName === "rust") extractUses(tree.rootNode as TsNode, rel, rawEdges); + else if (langName === "php") extractPhpUses(tree.rootNode as TsNode, rel, rawEdges); + return { nodes, rawEdges }; + } catch (err) { + // The walk is WASM-backed too (query.matches() runs in the grammar's runtime), + // so a real abort can surface here, not only from parse(). Route it through + // notePoison so one abort poisons the runtime and every later file gets the + // stable "aborted earlier" error instead of re-attempting a dead runtime. + // Rethrow the ORIGINAL error object unchanged so an ordinary walk failure keeps + // its message (build.ts records it per file). + notePoison(err); + throw err; + } finally { + // Every tree lives in the Emscripten heap until deleted. Without this, a + // 65k-file build leaks ~2 GB and tree-sitter's allocator calls abort(). + // Swallow a throw from delete(): the primary error (from the try or the catch) + // must never be masked by a delete failure, and a delete() that throws means the + // runtime is already gone — there is nothing left to free. + try { tree.delete(); } catch { /* runtime already gone; keep the real error */ } } - // The preprocessor is invisible to tags.scm, but in C/C++ a local `#include "x.h"` - // IS the dependency graph — capture it as a file→file import. Likewise a Rust - // `use crate::…` is an in-crate module dependency. - if (langName === "c" || langName === "cpp") extractIncludes(tree.rootNode as TsNode, rel, rawEdges); - else if (langName === "rust") extractUses(tree.rootNode as TsNode, rel, rawEdges); - else if (langName === "php") extractPhpUses(tree.rootNode as TsNode, rel, rawEdges); - return { nodes, rawEdges }; } /** PHP `use App\Models\User;` → a file→class-file `imports` raw edge, one per imported diff --git a/test/container-extract.test.ts b/test/container-extract.test.ts index f5855fc3b..ffc8ff898 100644 --- a/test/container-extract.test.ts +++ b/test/container-extract.test.ts @@ -23,6 +23,8 @@ import { containerLangOf, containerExtensions, isContainerWarm, + CONTAINER_LANGS, + swapContainerGrammarForTest, } from "../src/graph/container.js"; import { supportedExtensions } from "../src/graph/source-files.js"; import { buildGraph } from "../src/graph/build.js"; @@ -290,3 +292,17 @@ test("container: a clean build of a .vue file checks as in sync", async () => { assert.deepEqual(check.changed, []); assert.equal(check.ok, true, "a clean build checks OK"); }); + +test("a wrapper grammar that throws is a build error, not a clean symbol-less file", async () => { + await warmContainerGrammars(["vue"]); + const vue = CONTAINER_LANGS.find((l) => l.name === "vue")!; + const prev = swapContainerGrammarForTest("vue", new Proxy({}, { get(): never { throw new Error("memory access out of bounds (fake)"); } })); + try { + assert.throws( + () => extractContainer("App.vue", "\n", vue), + /vue grammar threw: memory access out of bounds/, + ); + } finally { + swapContainerGrammarForTest("vue", prev); + } +}); diff --git a/test/generic-extract.test.ts b/test/generic-extract.test.ts index aa7498264..22be35889 100644 --- a/test/generic-extract.test.ts +++ b/test/generic-extract.test.ts @@ -17,7 +17,10 @@ import { loadWasmLanguage, parseWasm, swapGrammarForTest, + wasmRuntimePoisoned, + clearWasmPoisonForTest, type TsNode, + type Loaded, } from "../src/graph/generic.js"; import { resolveEdges } from "../src/graph/resolve.js"; import { buildGraph } from "../src/graph/build.js"; @@ -397,13 +400,18 @@ function namedOfType(root: TsNode, type: string): string[] { async function assertPhpWasmExtractsClassAndMethods(source: string, className: string, label: string): Promise { const language = await loadWasmLanguage("php"); assert.ok(language, "tree-sitter-wasm must ship a php grammar"); - const root = parseWasm(language, source); - assert.ok(root, `${label}: PHP wasm parse must not crash (1.1.4 threw on heredoc/nowdoc)`); - const classes = namedOfType(root, "class_declaration"); - const methods = namedOfType(root, "method_declaration"); - assert.ok(classes.includes(className), `${label}: expected class ${className}, got: ${classes.join(", ") || "(none)"}`); - assert.ok(methods.includes("sql"), `${label}: expected method sql, got: ${methods.join(", ") || "(none)"}`); - assert.ok(methods.includes("other"), `${label}: expected method other, got: ${methods.join(", ") || "(none)"}`); + const tree = parseWasm(language, source); + assert.ok(tree, `${label}: PHP wasm parse must not crash (1.1.4 threw on heredoc/nowdoc)`); + try { + const root = tree.rootNode; + const classes = namedOfType(root, "class_declaration"); + const methods = namedOfType(root, "method_declaration"); + assert.ok(classes.includes(className), `${label}: expected class ${className}, got: ${classes.join(", ") || "(none)"}`); + assert.ok(methods.includes("sql"), `${label}: expected method sql, got: ${methods.join(", ") || "(none)"}`); + assert.ok(methods.includes("other"), `${label}: expected method other, got: ${methods.join(", ") || "(none)"}`); + } finally { + tree.delete(); + } } test("PHP wasm grammar extracts class + methods from a heredoc file (#139)", async () => { @@ -421,12 +429,17 @@ test("PHP wasm grammar extracts class + methods from a nowdoc file (#139)", asyn * clean parse. The real crash is heap-state dependent, so a deterministically * throwing fake grammar is swapped in via the test seam instead. */ -const THROWING_GRAMMAR = { - // web-tree-sitter's setLanguage() inspects the language object, so any - // property access blowing up makes extractGeneric's try block throw exactly - // like a crashing wasm scanner does. - language: new Proxy({}, { get(): never { throw new Error("memory access out of bounds (fake)"); } }), +const THROWING_GRAMMAR: Loaded = { + // The parser is created once per grammar at warm time, so the per-file failure + // an external scanner produces is a throw out of parse(). reset() is what + // extractGeneric calls on the shared parser before rethrowing. + language: {}, query: null, + parser: { + parse(): never { throw new Error("memory access out of bounds (fake)"); }, + reset(): void {}, + delete(): void {}, + }, }; test("extractGeneric rethrows a throwing grammar with the language named (#139)", async () => { @@ -471,3 +484,165 @@ test("a throwing grammar is a per-file build error, cached as a failure (#139)", swapGrammarForTest("rust", prev); } }); + +/** Wrap the real rust grammar so every tree it hands out is counted on the way + * out and on delete(). The parse itself is real; only the bookkeeping is fake. */ +function countingGrammar(real: Loaded, hooks: { created: number; deleted: number; resets: number }): Loaded { + return { + language: real.language, + query: real.query, + parser: { + parse(cb) { + const t = real.parser.parse(cb); + if (!t) return null; + hooks.created++; + return { rootNode: t.rootNode, delete: () => { hooks.deleted++; t.delete(); } }; + }, + reset: () => { hooks.resets++; real.parser.reset(); }, + delete: () => real.parser.delete(), + }, + }; +} + +test("extractGeneric deletes every tree it parses", async () => { + await warmGenericGrammars(["rust"]); + const hooks = { created: 0, deleted: 0, resets: 0 }; + const real = swapGrammarForTest("rust", null)!; + swapGrammarForTest("rust", countingGrammar(real, hooks)); + try { + for (let i = 0; i < 25; i++) extractGeneric("src/lib.rs", `pub fn f${i}() { g() }\n`, "rust"); + assert.equal(hooks.created, 25, "every call parsed"); + assert.equal(hooks.deleted, 25, "every tree deleted"); + assert.equal(hooks.resets, 0, "no reset on the happy path"); + } finally { + swapGrammarForTest("rust", real); + } +}); + +test("extractGeneric deletes the tree even when the walk throws", async () => { + await warmGenericGrammars(["rust"]); + const hooks = { created: 0, deleted: 0, resets: 0 }; + const real = swapGrammarForTest("rust", null)!; + // tagsExtract calls query.matches(root); a query that throws there throws mid-walk. + const grammar = countingGrammar(real, hooks); + grammar.query = { matches(): never { throw new Error("walk boom"); } }; + swapGrammarForTest("rust", grammar); + try { + assert.throws(() => extractGeneric("src/lib.rs", "pub fn f() {}\n", "rust"), /walk boom/); + assert.equal(hooks.created, 1); + assert.equal(hooks.deleted, 1, "the tree was deleted in finally"); + } finally { + swapGrammarForTest("rust", real); + } +}); + +test("a parse() that throws resets the shared parser before rethrowing", async () => { + await warmGenericGrammars(["rust"]); + const hooks = { created: 0, deleted: 0, resets: 0 }; + const real = swapGrammarForTest("rust", null)!; + const grammar = countingGrammar(real, hooks); + grammar.parser = { ...grammar.parser, parse(): never { throw new Error("memory access out of bounds (fake)"); } }; + swapGrammarForTest("rust", grammar); + try { + assert.throws(() => extractGeneric("src/lib.rs", "pub fn f() {}\n", "rust"), /rust grammar threw: memory access out of bounds/); + assert.equal(hooks.resets, 1, "reset() called once"); + assert.equal(wasmRuntimePoisoned(), null, "an ordinary throw does not poison the runtime"); + } finally { + swapGrammarForTest("rust", real); + } +}); + +test("a WebAssembly.RuntimeError poisons the runtime: every later call fails with one distinct message", async () => { + await warmGenericGrammars(["rust"]); + const real = swapGrammarForTest("rust", null)!; + const grammar: Loaded = { + language: real.language, query: real.query, + parser: { parse(): never { throw new WebAssembly.RuntimeError("Aborted(). Build with -sASSERTIONS for more info."); }, reset() {}, delete() {} }, + }; + swapGrammarForTest("rust", grammar); + try { + assert.throws(() => extractGeneric("src/lib.rs", "pub fn f() {}\n", "rust"), /rust grammar threw: Aborted\(\)/); + assert.match(wasmRuntimePoisoned() ?? "", /^Aborted\(\)/); + // The real grammar is back, but the process is poisoned: no parse is attempted. + swapGrammarForTest("rust", real); + assert.throws( + () => extractGeneric("src/lib.rs", "pub fn f() {}\n", "rust"), + /web-tree-sitter aborted earlier in this process .*restart/, + ); + } finally { + clearWasmPoisonForTest(); + swapGrammarForTest("rust", real); + } +}); + +test("a WebAssembly.RuntimeError from the walk poisons the runtime too", async () => { + await warmGenericGrammars(["rust"]); + const real = swapGrammarForTest("rust", null)!; + // query.matches() is WASM-backed, so an abort can surface from the walk, not + // only from parse(). It must poison the runtime the same way a parse abort does, + // and the thrown error must be the original object (its message carries Aborted()). + const grammar: Loaded = { + language: real.language, + query: { matches(): never { throw new WebAssembly.RuntimeError("Aborted(). Build with -sASSERTIONS for more info."); } }, + parser: real.parser, + }; + swapGrammarForTest("rust", grammar); + try { + assert.throws(() => extractGeneric("src/lib.rs", "pub fn f() {}\n", "rust"), /Aborted\(\)/); + assert.match(wasmRuntimePoisoned() ?? "", /^Aborted\(\)/); + // The real grammar is back, but the process is poisoned: no parse is attempted. + swapGrammarForTest("rust", real); + assert.throws( + () => extractGeneric("src/lib.rs", "pub fn f() {}\n", "rust"), + /web-tree-sitter aborted earlier in this process .*restart/, + ); + } finally { + clearWasmPoisonForTest(); + swapGrammarForTest("rust", real); + } +}); + +test("a tree.delete() that throws does not mask the walk's error", async () => { + await warmGenericGrammars(["rust"]); + const hooks = { created: 0, deleted: 0, resets: 0 }; + const real = swapGrammarForTest("rust", null)!; + // Like countingGrammar, but the tree's delete() throws AFTER counting, and the + // walk throws first. The delete throw must be swallowed so the walk's real error + // (not the delete error) is what propagates. + const grammar: Loaded = { + language: real.language, + query: { matches(): never { throw new Error("walk boom"); } }, + parser: { + parse(cb) { + const t = real.parser.parse(cb); + if (!t) return null; + hooks.created++; + return { rootNode: t.rootNode, delete: () => { hooks.deleted++; t.delete(); throw new Error("delete boom"); } }; + }, + reset: () => { hooks.resets++; real.parser.reset(); }, + delete: () => real.parser.delete(), + }, + }; + swapGrammarForTest("rust", grammar); + try { + assert.throws(() => extractGeneric("src/lib.rs", "pub fn f() {}\n", "rust"), /walk boom/); + assert.equal(hooks.deleted, 1, "delete() ran (and threw) but the throw was swallowed"); + } finally { + swapGrammarForTest("rust", real); + } +}); + +test("extractGeneric frees WASM memory: 400 parses of a 40 KB file do not grow RSS", async () => { + await warmGenericGrammars(["rust"]); + assert.ok(isWarm("rust"), "rust grammar warmed"); + const fn = (i: number) => `pub fn f${i}(a: u32, b: u32) -> u32 {\n let c = a + b;\n if c > 10 { helper(c) } else { c }\n}\n`; + const source = Array.from({ length: 400 }, (_, i) => fn(i)).join("\n"); + for (let i = 0; i < 20; i++) extractGeneric("src/lib.rs", source, "rust"); // warm-up + const before = process.memoryUsage().rss; + for (let i = 0; i < 400; i++) extractGeneric("src/lib.rs", source, "rust"); + const grewMb = (process.memoryUsage().rss - before) / 1048576; + // Leaking, each parse retains ~1.9 MB of WASM heap: 400 parses grow RSS by + // ~750 MB (and 1,100 parses would hit the 2 GB abort, so the loop stays at + // 400). Freed, growth is V8 noise. 100 MB is the line. + assert.ok(grewMb < 100, `RSS grew by ${grewMb.toFixed(0)} MB over 400 parses: the tree is not being deleted`); +}); From 79b96c9a9ea021898f867da48cd8b488953e8972 Mon Sep 17 00:00:00 2001 From: Kane Bruce <54819806+csm-kb@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:55:28 -0600 Subject: [PATCH 2/4] fix(build): retry a file whose cached parse failed A cached error was replayed until the file's bytes changed, so one abort pinned 52k files as failed. Skip the cache hit for an errored entry. The fingerprint keeps its hash, so a refresh does not rebuild just to retry it. --- src/graph/build.ts | 16 ++++++++++------ test/generic-extract.test.ts | 29 ++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/graph/build.ts b/src/graph/build.ts index b3094051d..55f300b39 100644 --- a/src/graph/build.ts +++ b/src/graph/build.ts @@ -107,7 +107,7 @@ export interface GraphBuildResult { /** Per-file wiring cards written (Tier-2 passive surface). */ cards: number; files: number; - /** Files re-parsed this run (the rest were replayed from the extraction cache). */ + /** Files parsed this run (the rest were replayed from the extraction cache). Includes files whose previous build recorded a parse error: those are always re-attempted. */ parsed: number; /** Files replayed from the extraction cache. */ reused: number; @@ -244,14 +244,18 @@ export async function buildGraph( } const hash = contentHash(source); - if (cached && hash === cached.hash) { + // A cached FAILURE is never replayed. Most parse failures are not properties + // of the file: a WASM grammar that aborted because the heap was exhausted by + // the files before it fails again only if the heap is exhausted again. Replaying + // the error would pin it to the file until its bytes changed; on a 65k-file + // repo that was 52k files stuck failing after a single abort. Re-parsing a + // genuinely broken file costs one parse per build, which is nothing. (The + // pre-query refresh is unaffected: fingerprint.ts keeps an errored entry with + // a real hash on its stat fast path, so only an explicit build retries.) + if (cached && hash === cached.hash && !cached.error) { entries[rel] = { ...cached, size: f.size, mtimeMs: f.mtimeMs }; sources.set(rel, source); reused++; - if (cached.error) { - errors.push(cached.error); // this file failed to parse last time too - return; - } nodes.push(...cached.nodes); rawEdges.push(...cached.rawEdges); langs.add(label); diff --git a/test/generic-extract.test.ts b/test/generic-extract.test.ts index 22be35889..a129071d2 100644 --- a/test/generic-extract.test.ts +++ b/test/generic-extract.test.ts @@ -460,7 +460,7 @@ test("extractGeneric rethrows a throwing grammar with the language named (#139)" } }); -test("a throwing grammar is a per-file build error, cached as a failure (#139)", async () => { +test("a throwing grammar is a per-file build error, and the file is retried next build (#139)", async () => { const dir = mkdtempSync(join(tmpdir(), "graft-throwing-grammar-")); writeFileSync(join(dir, "lib.rs"), "pub fn f() {}\n"); await warmGenericGrammars(["rust"]); // so buildGraph's own warm call is a no-op @@ -474,17 +474,36 @@ test("a throwing grammar is a per-file build error, cached as a failure (#139)", assert.ok(g, "graph built"); assert.ok(!g!.nodes.some((n) => n.path === "lib.rs"), "failed file has no file node"); - // The extract cache must remember the failure, not an empty success: an - // incremental rebuild of the unchanged file replays the error. + // The failure is recorded (so the freshness probe is quiet) but NOT replayed: + // the second build parses the unchanged file again and, the grammar still + // throwing, reports the error again as a fresh parse, not a cache hit. const second = await buildGraph(dir, { reuse: true }); - assert.equal(second.parsed, 0, "unchanged file is not re-parsed"); - assert.equal(second.errors.length, 1, `error replayed (got: ${second.errors.join("; ")})`); + assert.equal(second.errors.length, 1, `error reported again (got: ${second.errors.join("; ")})`); assert.match(second.errors[0], /rust grammar threw/); + assert.equal(second.parsed, 1, "the errored file was re-parsed, not replayed"); + assert.equal(second.reused, 0, "an errored entry never counts as reused"); } finally { swapGrammarForTest("rust", prev); } }); +test("a file that failed for an environmental reason recovers on the next build without a cache delete", async () => { + const dir = mkdtempSync(join(tmpdir(), "graft-recovering-grammar-")); + writeFileSync(join(dir, "lib.rs"), "pub fn f() {}\npub fn g() { f() }\n"); + await warmGenericGrammars(["rust"]); + const prev = swapGrammarForTest("rust", THROWING_GRAMMAR); + try { + const first = await buildGraph(dir, { reuse: false }); + assert.equal(first.errors.length, 1, "first build fails"); + } finally { + swapGrammarForTest("rust", prev); // the "environment" is healthy again + } + const second = await buildGraph(dir, { reuse: true }); + assert.equal(second.errors.length, 0, `second build clean (got: ${second.errors.join("; ")})`); + const g = readGraph(wiringPath(contextDirFor(dir))); + assert.ok(g!.nodes.some((n) => n.id === "lib.rs#f"), "the file's symbols are in the graph now"); +}); + /** Wrap the real rust grammar so every tree it hands out is counted on the way * out and on delete(). The parse itself is real; only the bookkeeping is fake. */ function countingGrammar(real: Loaded, hooks: { created: number; deleted: number; resets: number }): Loaded { From 9559d0f08871fa8a247658dd2b18cdefc02921d7 Mon Sep 17 00:00:00 2001 From: Kane Bruce <54819806+csm-kb@users.noreply.github.com> Date: Thu, 10 Sep 2026 00:50:13 -0600 Subject: [PATCH 3/4] fix(cli): cap the build error listing at 20 lines; changelog --- CHANGELOG.md | 18 +++++++++++++++ src/cli.ts | 6 ++++- src/graph/build.ts | 5 ++-- src/graph/generic.ts | 4 ++++ test/generic-extract.test.ts | 44 ++++++++++++++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ef248a39..73388d552 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## Unreleased + +### Fixed + +- **`graft build` no longer aborts partway through very large repos.** The + breadth-tier extractor created a web-tree-sitter parser per file and never + freed it or the syntax tree; those live in the WASM heap, which is capped at + 2 GB, so on a C source base of ~65,000 files every parse after ~12,000 failed + with `Aborted()`. One parser per grammar is now reused and every tree is + deleted after extraction, in the container (`.vue`) tier as well as the + breadth tier. If the runtime does abort, the build reports it once and says + to restart, instead of once per remaining file. +- **A parse failure is retried on the next `graft build`** instead of being + replayed from the extract cache until the file's bytes change, so a one-off + failure no longer needs a manual cache delete to clear. +- **A `.vue` wrapper grammar that throws is now a build error** for that file, + not a silently symbol-less file cached as a success. + ## 0.19.0 ### Added diff --git a/src/cli.ts b/src/cli.ts index 54e97a9f2..e7656d563 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -583,7 +583,11 @@ program }, { repo: buildRoot }, ); - for (const e of g.errors) console.error(`✗ ${e}`); + // A poisoned WASM runtime or a broken grammar can fail tens of thousands of + // files identically; the first 20 say everything the rest would. + const MAX_ERRORS_SHOWN = 20; + for (const e of g.errors.slice(0, MAX_ERRORS_SHOWN)) console.error(`✗ ${e}`); + if (g.errors.length > MAX_ERRORS_SHOWN) console.error(`✗ … and ${g.errors.length - MAX_ERRORS_SHOWN} more (${g.errors.length} files failed to parse)`); const rel = relative(process.cwd(), g.contextDir) || "graft"; if (process.env.GRAFT_NO_GITIGNORE) { diff --git a/src/graph/build.ts b/src/graph/build.ts index 55f300b39..cf316d793 100644 --- a/src/graph/build.ts +++ b/src/graph/build.ts @@ -250,8 +250,9 @@ export async function buildGraph( // the error would pin it to the file until its bytes changed; on a 65k-file // repo that was 52k files stuck failing after a single abort. Re-parsing a // genuinely broken file costs one parse per build, which is nothing. (The - // pre-query refresh is unaffected: fingerprint.ts keeps an errored entry with - // a real hash on its stat fast path, so only an explicit build retries.) + // pre-query refresh never rebuilds solely to retry an errored file: + // fingerprint.ts keeps the entry's real hash on its stat fast path. A rebuild + // triggered by other drift re-parses it like any build.) if (cached && hash === cached.hash && !cached.error) { entries[rel] = { ...cached, size: f.size, mtimeMs: f.mtimeMs }; sources.set(rel, source); diff --git a/src/graph/generic.ts b/src/graph/generic.ts index 9202064c8..36ca98f8d 100644 --- a/src/graph/generic.ts +++ b/src/graph/generic.ts @@ -204,6 +204,10 @@ export function swapGrammarForTest(name: string, entry: Loaded | null): Loaded | return prev; } +/** Test seam: pre-seed the parser parseWasm will use for `language`, so a + * throwing parse() can be driven without a real crashing grammar. */ +export function seedWasmParserForTest(language: object, parser: WasmParser): void { parsersByLanguage.set(language, parser); } + /** Load one grammar from the tree-sitter-wasms bundle, initialising * web-tree-sitter on first call. Null when the wasm is missing or won't * instantiate — never throws, so a caller degrades instead of failing the build. diff --git a/test/generic-extract.test.ts b/test/generic-extract.test.ts index a129071d2..554c5ed71 100644 --- a/test/generic-extract.test.ts +++ b/test/generic-extract.test.ts @@ -17,10 +17,12 @@ import { loadWasmLanguage, parseWasm, swapGrammarForTest, + seedWasmParserForTest, wasmRuntimePoisoned, clearWasmPoisonForTest, type TsNode, type Loaded, + type WasmParser, } from "../src/graph/generic.js"; import { resolveEdges } from "../src/graph/resolve.js"; import { buildGraph } from "../src/graph/build.js"; @@ -665,3 +667,45 @@ test("extractGeneric frees WASM memory: 400 parses of a 40 KB file do not grow R // 400). Freed, growth is V8 noise. 100 MB is the line. assert.ok(grewMb < 100, `RSS grew by ${grewMb.toFixed(0)} MB over 400 parses: the tree is not being deleted`); }); + +// parseWasm has its OWN parse-catch path (generic.ts, distinct from extractGeneric's): +// notePoison → parser.reset() → throw. It is the path the container (.vue) tier hits. +// parseWasm keys a WeakMap by the language object and creates the +// parser itself on first use, so a fake language cannot make parse() throw on its own — +// seedWasmParserForTest pre-seeds the parser parseWasm will use, so a throwing parse() +// can be driven without a real crashing grammar. +test("parseWasm rethrows a throwing parse and resets the shared parser (an ordinary throw does not poison)", async () => { + await warmGenericGrammars(["rust"]); // initialises web-tree-sitter + const language = {}; + let resets = 0; + const parser: WasmParser = { + parse(): never { throw new Error("memory access out of bounds (fake)"); }, + reset(): void { resets++; }, + delete(): void {}, + }; + seedWasmParserForTest(language, parser); + assert.throws(() => parseWasm(language, "x"), /memory access out of bounds/); + assert.equal(resets, 1, "reset() called once before rethrow"); + assert.equal(wasmRuntimePoisoned(), null, "an ordinary throw does not poison the runtime"); +}); + +test("parseWasm poisons the runtime on a WebAssembly.RuntimeError; every later call fails with the aborted-earlier error", async () => { + await warmGenericGrammars(["rust"]); + const language = {}; + // The ES2022 lib has no WebAssembly type; read its RuntimeError constructor off globalThis. + const RuntimeError = (globalThis as { WebAssembly: { RuntimeError: new (m: string) => Error } }).WebAssembly.RuntimeError; + const parser: WasmParser = { + parse(): never { throw new RuntimeError("Aborted()."); }, + reset(): void {}, + delete(): void {}, + }; + seedWasmParserForTest(language, parser); + try { + assert.throws(() => parseWasm(language, "x"), /Aborted\(/); + assert.match(wasmRuntimePoisoned() ?? "", /^Aborted\(/); + // The runtime is poisoned now: a further parseWasm never attempts a parse. + assert.throws(() => parseWasm(language, "x"), /aborted earlier in this process/); + } finally { + clearWasmPoisonForTest(); + } +}); From 487513a21f303abe5a8ca3fa03cd4d8a748764ce Mon Sep 17 00:00:00 2001 From: Kane Bruce <54819806+csm-kb@users.noreply.github.com> Date: Sun, 4 Oct 2026 11:24:53 -0600 Subject: [PATCH 4/4] fix(generic): walk syntax trees with an explicit stack A C++ header misparsed by the C grammar can nest one level per initializer element; the recursive walks overflowed the call stack. --- CHANGELOG.md | 4 +++ src/graph/generic.ts | 52 +++++++++++++++++------------------- test/generic-extract.test.ts | 20 ++++++++++++++ 3 files changed, 48 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73388d552..d51a45cff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ failure no longer needs a manual cache delete to clear. - **A `.vue` wrapper grammar that throws is now a build error** for that file, not a silently symbol-less file cached as a success. +- **A deeply nested syntax tree no longer fails with `Maximum call stack size + exceeded`.** A C++ header with a large brace initializer misparses under the + C grammar into a tree one level deep per element; the breadth-tier walks now + use an explicit stack instead of recursing. ## 0.19.0 diff --git a/src/graph/generic.ts b/src/graph/generic.ts index 36ca98f8d..840418652 100644 --- a/src/graph/generic.ts +++ b/src/graph/generic.ts @@ -361,21 +361,32 @@ export function extractGeneric(rel: string, source: string, langName: string): E } } +/** Pre-order DFS over `root` and its named descendants, on an explicit stack: a + * misparse can nest one level per element (a C++ array initializer under the C + * grammar runs thousands deep), which overflows V8's stack if the walk recurses. Children + * are pushed in reverse so the visit order matches a recursive walk. */ +function forEachNamedNode(root: TsNode, fn: (n: TsNode) => void): void { + const stack: TsNode[] = [root]; + while (stack.length) { + const n = stack.pop()!; + fn(n); + for (let i = (n.namedChildCount ?? 0) - 1; i >= 0; i--) { + const c = n.namedChild?.(i); + if (c) stack.push(c); + } + } +} + /** PHP `use App\Models\User;` → a file→class-file `imports` raw edge, one per imported * name (a `{ … }` group expands to several). `use function`/`use const` are skipped — * those name a symbol, not a PSR-4 class file. resolve.ts settles the fully-qualified * name to the in-repo file by namespace suffix, and drops it when it can't. */ function extractPhpUses(root: TsNode, rel: string, rawEdges: RawEdge[]): void { - const visit = (n: TsNode): void => { + forEachNamedNode(root, (n) => { if (n.type === "namespace_use_declaration") { for (const fqn of phpUseNames(n.text)) rawEdges.push({ source: rel, relation: "imports", specifier: fqn, file: rel }); } - for (let i = 0; i < (n.namedChildCount ?? 0); i++) { - const c = n.namedChild?.(i); - if (c) visit(c); - } - }; - visit(root); + }); } /** The fully-qualified class names a PHP `use` declaration imports. Handles a plain @@ -399,17 +410,12 @@ function phpUseNames(text: string): string[] { * `super::`, `self::`, external crates, and globs are skipped — resolve.ts settles the * path against the file's crate root, and drops it when it can't. */ function extractUses(root: TsNode, rel: string, rawEdges: RawEdge[]): void { - const visit = (n: TsNode): void => { + forEachNamedNode(root, (n) => { if (n.type === "use_declaration") { const spec = rustUseModule(n.text); if (spec !== null) rawEdges.push({ source: rel, relation: "imports", specifier: spec ? `crate/${spec}` : "crate", file: rel }); } - for (let i = 0; i < (n.namedChildCount ?? 0); i++) { - const c = n.namedChild?.(i); - if (c) visit(c); - } - }; - visit(root); + }); } /** The crate-relative path a Rust `use crate::…` names (`::`→`/`), or null when it is not @@ -434,7 +440,7 @@ function rustUseModule(text: string): string | null { * in-repo header (relative to the including file, else a unique path-suffix match), and * keeps it as an external string when it cannot — never a guessed edge. */ function extractIncludes(root: TsNode, rel: string, rawEdges: RawEdge[]): void { - const visit = (n: TsNode): void => { + forEachNamedNode(root, (n) => { if (n.type === "preproc_include") { const raw = n.childForFieldName?.("path")?.text ?? ""; if (raw.startsWith('"')) { @@ -442,12 +448,7 @@ function extractIncludes(root: TsNode, rel: string, rawEdges: RawEdge[]): void { if (spec) rawEdges.push({ source: rel, relation: "imports", specifier: spec, file: rel }); } } - for (let i = 0; i < (n.namedChildCount ?? 0); i++) { - const c = n.namedChild?.(i); - if (c) visit(c); - } - }; - visit(root); + }); } /** tags.scm path: @definition. → nodes, @reference.call/@reference.send → @@ -548,18 +549,13 @@ function nodeName(node: TsNode): string | null { /** Walker fallback: DFS every named node, emit a def for each classified one * (symbols only — no call resolution without a query). */ function walkExtract(root: TsNode, mkDef: (name: string, kind: Kind, whole: TsNode) => void): void { - const visit = (n: TsNode): void => { + forEachNamedNode(root, (n) => { const kind = classifyKind(n.type); if (kind) { const name = nodeName(n); if (name) mkDef(name, kind, n); } - for (let i = 0; i < (n.namedChildCount ?? 0); i++) { - const c = n.namedChild?.(i); - if (c) visit(c); - } - }; - visit(root); + }); } /** Minimal structural view of a web-tree-sitter node — shared with container.ts. */ diff --git a/test/generic-extract.test.ts b/test/generic-extract.test.ts index 554c5ed71..ce8323752 100644 --- a/test/generic-extract.test.ts +++ b/test/generic-extract.test.ts @@ -218,6 +218,26 @@ test("breadth tier: C #include becomes a resolved file→file import (local only assert.deepEqual(targetsOf("src/amb.c"), ["util.h"], "ambiguous include kept external, never guessed"); }); +// A C++ header routed to the C grammar misparses a big brace initializer into +// right-nested comma_expressions: the tree is as deep as the array is long. The +// walks over it must not recurse per level, or V8's stack overflows (RangeError). +test("breadth tier: a C header with a 20,000-element initializer extracts without overflowing the stack", async () => { + await warmGenericGrammars(["c"]); + const n = 20000; + const values = Array.from({ length: n }, (_, i) => String(i + 1)).join(", "); + const src = + '#include "dep.h"\n' + + "#include \n" + + `std::array data = { ${values} };\n` + + "int after(void) { return 0; }\n"; + const { nodes, rawEdges } = extractGeneric("big.h", src, "c"); + assert.ok( + rawEdges.some((e) => e.relation === "imports" && e.specifier === "dep.h"), + "the quoted include before the array is captured", + ); + assert.ok(nodes.some((x) => x.kind === "function" && x.name === "after"), "the function after the array is extracted"); +}); + // Rust `use crate::…` → a file→module import, resolved against the file's crate root // (the lib.rs/main.rs dir). The longest-prefix rule disambiguates a module from an item // and a `foo.rs` from a `foo/mod.rs`; std/super/external/glob are skipped, and an