fix(generic): free web-tree-sitter parsers and trees so large builds stop aborting

CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## Unreleased
+
+### Fixed
+
+- **`graft build` no longer aborts partway through very large repos.** The
+  breadth-tier extractor created a web-tree-sitter parser per file and never
+  freed it or the syntax tree; those live in the WASM heap, which is capped at
+  2 GB, so on a C source base of ~65,000 files every parse after ~12,000 failed
+  with `Aborted()`. One parser per grammar is now reused and every tree is
+  deleted after extraction, in the container (`.vue`) tier as well as the
+  breadth tier. If the runtime does abort, the build reports it once and says
+  to restart, instead of once per remaining file.
+- **A parse failure is retried on the next `graft build`** instead of being
+  replayed from the extract cache until the file's bytes change, so a one-off
+  failure no longer needs a manual cache delete to clear.
+- **A `.vue` wrapper grammar that throws is now a build error** for that file,
+  not a silently symbol-less file cached as a success.
+- **A deeply nested syntax tree no longer fails with `Maximum call stack size
+  exceeded`.** A C++ header with a large brace initializer misparses under the
+  C grammar into a tree one level deep per element; the breadth-tier walks now
+  use an explicit stack instead of recursing.
+
 ## 0.19.0
 
 ### Added

src/cli.ts

@@ -583,7 +583,11 @@ program
       },
       { repo: buildRoot },
     );
-    for (const e of g.errors) console.error(`✗ ${e}`);
+    // A poisoned WASM runtime or a broken grammar can fail tens of thousands of
+    // files identically; the first 20 say everything the rest would.
+    const MAX_ERRORS_SHOWN = 20;
+    for (const e of g.errors.slice(0, MAX_ERRORS_SHOWN)) console.error(`✗ ${e}`);
+    if (g.errors.length > MAX_ERRORS_SHOWN) console.error(`✗ … and ${g.errors.length - MAX_ERRORS_SHOWN} more (${g.errors.length} files failed to parse)`);
 
     const rel = relative(process.cwd(), g.contextDir) || "graft";
     if (process.env.GRAFT_NO_GITIGNORE) {

src/graph/build.ts

@@ -107,7 +107,7 @@ export interface GraphBuildResult {
   /** Per-file wiring cards written (Tier-2 passive surface). */
   cards: number;
   files: number;
-  /** Files re-parsed this run (the rest were replayed from the extraction cache). */
+  /** Files parsed this run (the rest were replayed from the extraction cache). Includes files whose previous build recorded a parse error: those are always re-attempted. */
   parsed: number;
   /** Files replayed from the extraction cache. */
   reused: number;
@@ -244,14 +244,19 @@ export async function buildGraph(
     }
 
     const hash = contentHash(source);
-    if (cached && hash === cached.hash) {
+    // A cached FAILURE is never replayed. Most parse failures are not properties
+    // of the file: a WASM grammar that aborted because the heap was exhausted by
+    // the files before it fails again only if the heap is exhausted again. Replaying
+    // the error would pin it to the file until its bytes changed; on a 65k-file
+    // repo that was 52k files stuck failing after a single abort. Re-parsing a
+    // genuinely broken file costs one parse per build, which is nothing. (The
+    // pre-query refresh never rebuilds solely to retry an errored file:
+    // fingerprint.ts keeps the entry's real hash on its stat fast path. A rebuild
+    // triggered by other drift re-parses it like any build.)
+    if (cached && hash === cached.hash && !cached.error) {
       entries[rel] = { ...cached, size: f.size, mtimeMs: f.mtimeMs };
       sources.set(rel, source);
       reused++;
-      if (cached.error) {
-        errors.push(cached.error); // this file failed to parse last time too
-        return;
-      }
       nodes.push(...cached.nodes);
       rawEdges.push(...cached.rawEdges);
       langs.add(label);

src/graph/container.ts

@@ -22,7 +22,7 @@
  * fixtures whose true line numbers are known.
  */
 import { extractFile, mintId, type ExtractResult, type Language, type RawEdge } from "./extract.js";
-import { loadWasmLanguage, parseWasm, type TsNode } from "./generic.js";
+import { loadWasmLanguage, parseWasm, type TsNode, type WasmTree } from "./generic.js";
 import { contentHash } from "../util/id.js";
 import type { NodeV1 } from "./types.js";
 
@@ -66,6 +66,14 @@ export function containerExtensions(): string[] {
 
 const loaded = new Map<string, unknown>();
 
+/** Test seam, same shape as generic.ts's `swapGrammarForTest`. */
+export function swapContainerGrammarForTest(name: string, language: unknown | null): unknown | null {
+  const prev = loaded.get(name) ?? null;
+  if (language) loaded.set(name, language);
+  else loaded.delete(name);
+  return prev;
+}
+
 /** Warm the container grammars this repo needs. Same contract as
  * `warmGenericGrammars`: await once before the synchronous parse loop, and an
  * unavailable grammar is skipped rather than fatal (its files then extract to a
@@ -144,61 +152,78 @@ function blocks(root: TsNode, lang: ContainerLang): TsNode[] {
 /**
  * Extract one container file. Synchronous; needs the grammar pre-warmed.
  *
- * Never throws: a missing grammar, an unparseable SFC or a script block the
- * inner extractor chokes on all degrade to "fewer nodes", because a build must
- * not fail over one component.
+ * A missing grammar, or a script block the inner extractor rejects, still
+ * degrades to "fewer nodes", because a build must not fail over one component.
+ * But a wrapper grammar that THROWS (a WASM abort, not a per-file syntax error)
+ * propagates as a build error: swallowing it would cache a clean symbol-less
+ * file as success, the exact mistake the generic tier used to make.
  */
 export function extractContainer(rel: string, source: string, lang: ContainerLang): ExtractResult {
   const nodes: NodeV1[] = [];
   const rawEdges: RawEdge[] = [];
   const residuals: string[] = [];
 
   const language = loaded.get(lang.name);
-  const root = language ? parseWasm(language, source) : null;
-
-  if (root) {
-    // Ids are minted per file by the inner extractor, so two script blocks that
-    // both define `setup` would collide. Threading one set across the blocks
-    // makes the second one `path#setup~2`, and the rename is applied to that
-    // block's edges too so nothing points at an id that no longer exists.
-    const minted = new Set<string>([rel]);
-
-    for (const body of blocks(root, lang)) {
-      const script = source.slice(body.startIndex, body.endIndex);
-
-      let inner: ExtractResult;
-      try {
-        inner = extractFile(rel, script, lang.inner);
-      } catch {
-        continue; // one bad block, not a bad build
-      }
-
-      // `raw_text` starts immediately after the `>` of the opening tag, so its
-      // row IS the tag's row and the slice begins with that line's newline.
-      // Script line 1 is therefore the tail of the tag line, and script line N
-      // lands on `.vue` line row + N — which is exactly "add the start row to a
-      // 1-based span". Taking the row from the tag node instead would look
-      // equivalent and be right only when the tag has no attributes.
-      const shift = body.startPosition.row;
-
-      // nodes[0] is the script's own file node: it describes the block, not the
-      // file, so it is dropped and its residual folded into the .vue file node.
-      const [scriptFile, ...symbols] = inner.nodes;
-      if (scriptFile?.body_text) residuals.push(scriptFile.body_text);
-
-      const renamed = new Map<string, string>();
-      for (const node of symbols) {
-        const id = mintId(node.id, minted);
-        if (id !== node.id) renamed.set(node.id, id);
-        nodes.push({ ...node, id, span: shiftSpan(node.span, shift) });
-      }
+  let tree: WasmTree | null = null;
+  if (language) {
+    try {
+      tree = parseWasm(language, source);
+    } catch (err) {
+      // Same contract as the generic tier: a grammar that throws is a build error
+      // for this file, never a clean file node cached as success.
+      throw new Error(`${lang.name} grammar threw: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
 
-      for (const edge of inner.rawEdges) {
-        const source_ = renamed.get(edge.source) ?? edge.source;
-        const targetId = edge.targetId === undefined ? undefined : (renamed.get(edge.targetId) ?? edge.targetId);
-        rawEdges.push({ ...edge, source: source_, ...(targetId === undefined ? {} : { targetId }) });
+  try {
+    if (tree) {
+      const root = tree.rootNode;
+      // Ids are minted per file by the inner extractor, so two script blocks that
+      // both define `setup` would collide. Threading one set across the blocks
+      // makes the second one `path#setup~2`, and the rename is applied to that
+      // block's edges too so nothing points at an id that no longer exists.
+      const minted = new Set<string>([rel]);
+
+      for (const body of blocks(root, lang)) {
+        const script = source.slice(body.startIndex, body.endIndex);
+
+        let inner: ExtractResult;
+        try {
+          inner = extractFile(rel, script, lang.inner);
+        } catch {
+          continue; // one bad block, not a bad build
+        }
+
+        // `raw_text` starts immediately after the `>` of the opening tag, so its
+        // row IS the tag's row and the slice begins with that line's newline.
+        // Script line 1 is therefore the tail of the tag line, and script line N
+        // lands on `.vue` line row + N — which is exactly "add the start row to a
+        // 1-based span". Taking the row from the tag node instead would look
+        // equivalent and be right only when the tag has no attributes.
+        const shift = body.startPosition.row;
+
+        // nodes[0] is the script's own file node: it describes the block, not the
+        // file, so it is dropped and its residual folded into the .vue file node.
+        const [scriptFile, ...symbols] = inner.nodes;
+        if (scriptFile?.body_text) residuals.push(scriptFile.body_text);
+
+        const renamed = new Map<string, string>();
+        for (const node of symbols) {
+          const id = mintId(node.id, minted);
+          if (id !== node.id) renamed.set(node.id, id);
+          nodes.push({ ...node, id, span: shiftSpan(node.span, shift) });
+        }
+
+        for (const edge of inner.rawEdges) {
+          const source_ = renamed.get(edge.source) ?? edge.source;
+          const targetId = edge.targetId === undefined ? undefined : (renamed.get(edge.targetId) ?? edge.targetId);
+          rawEdges.push({ ...edge, source: source_, ...(targetId === undefined ? {} : { targetId }) });
+        }
       }
     }
+  } finally {
+    // The wrapper tree is a WASM-heap object; nothing frees it but this.
+    tree?.delete();
   }
 
   // Built last so it can carry the residual, but unshifted first so the file node