perf(build): build very large repos end to end; opt-in parallel parsing with graft build --workers

.env.example

@@ -54,6 +54,10 @@
 # --- Build side effects (opt-out) ---------------------------------------------
 # GRAFT_NO_GITIGNORE=1  — skip writing graft/ into .gitignore (e.g. you already ignore it globally)
 # GRAFT_NO_IGNORE=1     — skip writing .ignore (ripgrep re-admit); only if you manage search yourself
+# GRAFT_PARSE_WORKERS=auto — `graft build` parses in child processes, or a number (capped at the core count and the file count).
+#   auto = min(cores - 1, free memory / 300 MB, files / 200); in-process under 200 files or when that comes to fewer than 2.
+#   Off by default; the pre-query refresh, background sync and workspace builds never fork.
+# Very large repos (tens of thousands of files) also need a bigger V8 heap: NODE_OPTIONS=--max-old-space-size=16384 graft build.
 
 # --- Init (Claude Code) -------------------------------------------------------
 # Skip writing statusLine / subagentStatusLine during `graft init` (same as

CHANGELOG.md

@@ -1,5 +1,50 @@
 # Changelog
 
+## Unreleased
+
+### Changed
+
+- **Very large repos build and load.** The extract cache, wiring.json and the
+  ask index were each one `JSON.stringify` and one `JSON.parse`, which V8 caps
+  at 512 MB; a C source base of ~65,000 files overshoots that on all three.
+  The extract cache is now newline-delimited JSON, and wiring.json and the ask
+  index are written one element per line (still plain JSON for every reader)
+  and read back line by line when they are over the cap. `graft viz` still
+  reads wiring.json as one string and is limited to graphs under 512 MB.
+- **`graft build` holds far less memory on big repos.** Source text is no
+  longer kept for every file across the build; the meaning pass reads a file
+  when it summarizes it, and only if its bytes still hash to what was parsed.
+  A repo this size still needs a larger V8 heap than Node's default
+  (`NODE_OPTIONS=--max-old-space-size=16384` completed the 65k-file build);
+  moving `body_text` out of the retained node set is the next step.
+- **`graft build --workers <n|auto>`** parses a large cold build in child
+  processes and merges their output in file order, so the graph is
+  byte-identical to a single-process build, except that a file whose parse
+  exceeds the 120 s per-file timeout is recorded as an error. Opt-in; the
+  refresh that runs before a query never forks. `GRAFT_PARSE_WORKERS` sets
+  the default; `auto` is min(cores - 1, free memory / 300 MB, files / 200),
+  and stays in-process under 200 files or below 2 workers.
+
+### Fixed
+
+- **`graft build` no longer aborts partway through very large repos.** The
+  breadth-tier extractor created a web-tree-sitter parser per file and never
+  freed it or the syntax tree; those live in the WASM heap, which is capped at
+  2 GB, so on a C source base of ~65,000 files every parse after ~12,000 failed
+  with `Aborted()`. One parser per grammar is now reused and every tree is
+  deleted after extraction, in the container (`.vue`) tier as well as the
+  breadth tier. If the runtime does abort, the build reports it once and says
+  to restart, instead of once per remaining file.
+- **A parse failure is retried on the next `graft build`** instead of being
+  replayed from the extract cache until the file's bytes change, so a one-off
+  failure no longer needs a manual cache delete to clear.
+- **A `.vue` wrapper grammar that throws is now a build error** for that file,
+  not a silently symbol-less file cached as a success.
+- **A deeply nested syntax tree no longer fails with `Maximum call stack size
+  exceeded`.** A C++ header with a large brace initializer misparses under the
+  C grammar into a tree one level deep per element; the breadth-tier walks now
+  use an explicit stack instead of recursing.
+
 ## 0.19.0
 
 ### Added

README.md

@@ -352,6 +352,8 @@ graft build [dir]                    # build graft/ from the code at [dir]: wiri
 graft build --deep                   # add the LLM layer: concept nodes + per-symbol summary/crux (cached)
 graft build --extensions .ts .py     # only include these code extensions
 graft build --no-reuse               # re-parse every file instead of replaying unchanged ones from cache
+graft build --workers auto           # large repo? parse in parallel child processes (opt-in; never used by the pre-query refresh)
+# very large repos (tens of thousands of files) need a bigger V8 heap: NODE_OPTIONS=--max-old-space-size=16384 graft build
 graft build --follow-submodules      # include initialized submodules; persist the choice for builds + MCP refresh
 graft build --no-follow-submodules   # exclude submodules again and persist that choice (the default)
 graft build --follow-nested-repos    # include nested git clones the index doesn't track; persist the choice
@@ -389,6 +391,8 @@ graft check --json                   # print the drift report as JSON
 #   --no-refresh                     # answer from the graph exactly as it is on disk
 #   GRAFT_NO_REFRESH=1               # same, for every command
 #   GRAFT_REFRESH=hash               # hash every file instead of trusting size+mtime
+#   GRAFT_PARSE_WORKERS=auto         # graft build only: parse in child processes (or a number); default in-process
+#                                    #   auto = min(cores-1, free memory / 300 MB, files / 200); in-process under 200 files or below 2 workers
 
 graft trail push [dir]               # read this repo's history into its Trail (the first run signs up in the browser)
 graft trail pull [dir]               # write the changes accepted in Trail into CLAUDE.md, AGENTS.md and the other context files

src/ask/index-file.ts

@@ -19,9 +19,11 @@
  * contribution is folded into the stored `df` at query time (see `ask.ts`),
  * which is why `df` here counts symbol/file nodes only.
  */
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { dirname, join } from "node:path";
+import { constants as bufferConstants } from "node:buffer";
+import { readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
 import type { GraphV1 } from "../graph/types.js";
+import { forEachFileLine, forEachLine, openAtomic, type LineWalker } from "../util/json-stream.js";
 import { CACHE_DIR } from "../context/node-file.js";
 
 /** Words too common/short to carry query intent — dropped before scoring. */
@@ -116,44 +118,104 @@ export function writeAskIndex(outDir: string, graph: GraphV1): string {
     ? docs.reduce((a, d) => a + bagLen(d.body), 0) / docs.length
     : 0;
 
-  const index: AskIndex = {
-    version: 1,
-    avgBodyLen,
-    df: pairs(df),
-    docCount: nodes.length,
-    docs,
-  };
-
   const outPath = askIndexPath(outDir);
-  mkdirSync(dirname(outPath), { recursive: true });
-  writeFileSync(outPath, JSON.stringify(index) + "\n");
+  // Streamed one doc per line: still valid JSON for JSON.parse readers, but the
+  // newline at each doc boundary lets readAskIndexFromBuffer walk a file over
+  // V8's string cap without ever holding it as one string. On a 65k-file repo
+  // the one-shot string measured ~590 MB, over the ~512 MB cap. Key
+  // order matches AskIndex (version, avgBodyLen, df, docCount, docs).
+  const w = openAtomic(outPath);
+  try {
+    // The header is one line, and the whole `df` array lives on it — the streaming
+    // reader walks a line at a time, so this single line must itself fit under
+    // buffer.constants.MAX_STRING_LENGTH (~512 MB). df is the corpus vocabulary
+    // (one entry per distinct token, not per node), so even at 65k files it is a
+    // few MB — far off the ceiling. Only the per-line docs below scale with nodes.
+    w.write(`{"version":1,"avgBodyLen":${JSON.stringify(avgBodyLen)},"df":${JSON.stringify(pairs(df))},"docCount":${nodes.length},"docs":[`);
+    docs.forEach((doc, i) => w.write("\n" + JSON.stringify(doc) + (i < docs.length - 1 ? "," : "")));
+    w.write("\n]}\n");
+    w.commit();
+  } catch (e) {
+    w.abort();
+    throw e;
+  }
   return outPath;
 }
 
-/** Read the ask sidecar. Returns null on a missing file, unparseable JSON, an
- * unrecognized shape, an unknown `version`, or a `docCount` that doesn't match
- * the number of docs actually stored (a corrupted/truncated sidecar would
- * otherwise silently skew IDF) — any of which means the caller should fall
- * back to live tokenization, never crash or trust bad data. */
-export function readAskIndex(outDir: string): AskIndex | null {
+export interface ReadAskIndexOptions { maxStringLength?: number }
+
+/** Validate the shape of a parsed sidecar. Returns null on an unrecognized shape,
+ * an unknown `version`, or a `docCount` that doesn't match the number of docs
+ * actually stored (a corrupted/truncated sidecar would otherwise silently skew
+ * IDF). Runs on the result of both the JSON.parse and the streamed read path. */
+function validateAskIndex(raw: unknown): AskIndex | null {
+  if (!raw || typeof raw !== "object") return null;
+  const r = raw as Record<string, unknown>;
+  if (
+    r.version !== 1 ||
+    typeof r.docCount !== "number" ||
+    typeof r.avgBodyLen !== "number" ||
+    !Array.isArray(r.df) ||
+    !Array.isArray(r.docs) ||
+    r.docCount !== r.docs.length
+  ) {
+    return null;
+  }
+  return raw as AskIndex;
+}
+
+/** Read the ask sidecar. Returns null on a missing file, unparseable JSON, or a
+ * shape {@link validateAskIndex} rejects — any of which means the caller should
+ * fall back to live tokenization, never crash or trust bad data. A file over
+ * V8's string cap is walked line by line ({@link readAskIndexFromBuffer});
+ * `opts.maxStringLength` lets a test force that path. */
+export function readAskIndex(outDir: string, opts: ReadAskIndexOptions = {}): AskIndex | null {
   const path = askIndexPath(outDir);
-  if (!existsSync(path)) return null;
+  let size: number;
+  try { size = statSync(path).size; } catch { return null; }
+  const cap = opts.maxStringLength ?? bufferConstants.MAX_STRING_LENGTH;
+  if (size > cap) {
+    // Chunked: past 2 GiB the file is too large for one Buffer, let alone a string.
+    try { return readAskIndexFromLines((fn) => forEachFileLine(path, fn)); } catch { return null; }
+  }
+  let raw: unknown;
   try {
-    const raw = JSON.parse(readFileSync(path, "utf8"));
-    if (
-      !raw ||
-      typeof raw !== "object" ||
-      raw.version !== 1 ||
-      typeof raw.docCount !== "number" ||
-      typeof raw.avgBodyLen !== "number" ||
-      !Array.isArray(raw.df) ||
-      !Array.isArray(raw.docs) ||
-      raw.docCount !== raw.docs.length
-    ) {
-      return null;
-    }
-    return raw as AskIndex;
+    raw = JSON.parse(readFileSync(path, "utf8"));
   } catch {
     return null;
   }
+  return validateAskIndex(raw);
+}
+
+/** Parse the line-per-doc shape `writeAskIndex` produces without ever holding
+ * the whole file as one string. Null when the buffer is not in that shape or
+ * fails {@link validateAskIndex}. */
+export function readAskIndexFromBuffer(buf: Buffer): AskIndex | null {
+  return readAskIndexFromLines((fn) => forEachLine(buf, fn));
+}
+
+function readAskIndexFromLines(eachLine: LineWalker): AskIndex | null {
+  const DOCS_OPEN = ',"docs":[';
+  const END = "]}";
+  let head: Record<string, unknown> | null = null;
+  const docs: AskIndexDoc[] = [];
+  let closed = false;
+  let bad = false;
+  eachLine((line, i) => {
+    if (bad || closed) { if (line !== "") bad = true; return; }
+    try {
+      if (i === 0) {
+        if (!line.startsWith("{") || !line.endsWith(DOCS_OPEN)) { bad = true; return; }
+        head = JSON.parse(line.slice(0, -DOCS_OPEN.length) + "}") as Record<string, unknown>;
+        return;
+      }
+      if (line === END) { closed = true; return; }
+      const body = line.endsWith(",") ? line.slice(0, -1) : line;
+      docs.push(JSON.parse(body) as AskIndexDoc);
+    } catch {
+      bad = true;
+    }
+  });
+  if (bad || !closed || head === null) return null;
+  return validateAskIndex({ ...(head as Record<string, unknown>), docs });
 }

src/claude/stats.ts

@@ -1,13 +1,13 @@
-import { readFileSync } from 'node:fs';
-import { join } from 'node:path';
 import type { GraphV1 } from '../graph/types.js';
+import { readGraph, wiringPath, type ReadGraphOptions } from '../graph/write.js';
 import type { Stats } from './state.js';
 import { resolveContextDir } from '../util/state.js';
 
-export function readWiring(projectDir: string): GraphV1 | null {
-  try {
-    return JSON.parse(readFileSync(join(resolveContextDir(projectDir), '.graph', 'wiring.json'), 'utf8')) as GraphV1;
-  } catch { return null; }
+/** The project's wiring graph, or null when it is missing or unreadable. Through
+ * `readGraph`, so a graph over V8's string cap still reads (the statusline, the
+ * hooks and the background sync all start here); `opts` lets a test force that. */
+export function readWiring(projectDir: string, opts: ReadGraphOptions = {}): GraphV1 | null {
+  return readGraph(wiringPath(resolveContextDir(projectDir)), opts);
 }
 
 export function computeStats(

src/claude/sync-run.ts

@@ -9,9 +9,11 @@ function realBuild(dir: string): void {
   // GRAFT_TEST_CLI is the same seam hooks.ts's graftJson uses, so a test can
   // point this at a stub and inspect the exact argv it was invoked with.
   const cliPath = process.env.GRAFT_TEST_CLI ?? graftCliPath();
-  const args = [cliPath, 'build', '.'];
-  // Mirrors `withContextDirArg` in hooks.ts: a no-op unless GRAFT_DIR is set, so an
-  // unconfigured repo's rebuild sees byte-identical argv to before this existed.
+  // `--workers 0`: a background sync never forks a parser pool. The flag outranks
+  // GRAFT_PARSE_WORKERS, which the child would otherwise inherit from this env or
+  // load from the repo's own .env through dotenv.
+  const args = [cliPath, 'build', '.', '--workers', '0'];
+  // Mirrors `withContextDirArg` in hooks.ts: a no-op unless GRAFT_DIR is set.
   if (process.env.GRAFT_DIR) args.push('--dir', resolveContextDir(dir));
   execFileSync(process.execPath, args, { cwd: dir, stdio: 'ignore', timeout: 120000 });
 }

src/cli.ts

@@ -365,6 +365,7 @@ program
   .option("--deep", "run the LLM pass: concept nodes (graft/*.md) + per-symbol summary/crux")
   .option("-e, --extensions <exts...>", 'code extensions to include (e.g. ".ts" ".py"); an extension with no parser is ignored with a warning that lists the supported set')
   .option("-j, --concurrency <n>", "files summarized in parallel during --deep (default 5)")
+  .option("--workers <n>", "parse in <n> child processes for a large cold build (or 'auto'); default in-process. GRAFT_PARSE_WORKERS sets the default")
   .option("--no-reuse", "re-parse every file instead of replaying unchanged ones from the extraction cache")
   .option("--lsp", "add compiler-grade call edges via a language server if one is installed (opt-in, slower; e.g. rust-analyzer, clangd)")
   .option("--allow-partial", "with --deep: exit 0 even when some files' summaries failed (default: a degraded meaning tier exits 1)")
@@ -408,6 +409,7 @@ program
       deep?: boolean;
       extensions?: string[];
       concurrency?: string;
+      workers?: string;
       reuse?: boolean;
       lsp?: boolean;
       allowPartial?: boolean;
@@ -428,6 +430,17 @@ program
       console.error(`✗ --concurrency must be a number, got "${opts.concurrency}"`);
       process.exit(1);
     }
+    // Opt-in only. The env var is read HERE, by the one command a human runs,
+    // never by buildGraph: the pre-query refresh must not fork under a query.
+    // An empty GRAFT_PARSE_WORKERS= line in a .env reads as unset, not as invalid.
+    const workersArg = (opts.workers as string | undefined) ?? (process.env.GRAFT_PARSE_WORKERS?.trim() || undefined);
+    if (workersArg !== undefined && workersArg !== "auto" && !/^\d+$/.test(workersArg)) {
+      const source = opts.workers !== undefined ? "--workers" : "GRAFT_PARSE_WORKERS";
+      console.error(`✗ ${source} must be a non-negative integer or "auto", got "${workersArg}"`);
+      process.exit(1);
+    }
+    const parseWorkers: number | "auto" | undefined =
+      workersArg === undefined ? undefined : workersArg === "auto" ? "auto" : Number(workersArg);
     warnUnsupportedExtensions(opts.extensions);
     // Persisted BEFORE the build itself runs, so this invocation's walks (and
     // every later no-flag build / hooks refresh) see it identically — the
@@ -507,6 +520,10 @@ program
     const buildRoot = resolve(dir);
     const buildGlobalDir = program.opts<GlobalOpts>().dir;
     if (isWorkspaceBuildRoot(buildRoot, buildGlobalDir)) {
+      // Children build in-thread; `0` asks for exactly that, so only a pool request is ignored.
+      if (parseWorkers !== undefined && parseWorkers !== 0) {
+        console.error(`⚠ --workers / GRAFT_PARSE_WORKERS is ignored for workspace builds; each repo parses in-process`);
+      }
       await runWorkspaceBuild(buildRoot, {
         deep: !!deep,
         extensions: opts.extensions,
@@ -549,6 +566,7 @@ program
     const g = await engine.graph(dir, {
       llm: deep,
       concurrency,
+      parseWorkers,
       reuse: opts.reuse,
       lsp: opts.lsp,
       onlyDirs,
@@ -563,6 +581,8 @@ program
     process.stderr.write("\n");
     console.log(`✓ wiring: ${g.nodes} nodes (${fmt(g.byKind)}), ${g.edges} edges, ${g.cards} cards [${g.languages.join(", ")}]`);
     console.log(`  parsed: ${g.parsed} of ${g.files} files (${g.reused} replayed from cache)`);
+    if (g.parseWorkers > 0 && g.poolFallback) console.log(`  workers: ${g.parseWorkers} parser processes, until the pool could not keep them alive; the rest parsed in-process`);
+    else if (g.parseWorkers > 0) console.log(`  workers: ${g.parseWorkers} parser processes`);
     // Worth one line: this build started from a graph the user never built *here*.
     if (g.seededFrom) console.log(`  seeded: copied a starting graph from ${g.seededFrom} (git worktree)`);
     if (deep) {
@@ -583,7 +603,11 @@ program
       },
       { repo: buildRoot },
     );
-    for (const e of g.errors) console.error(`✗ ${e}`);
+    // A poisoned WASM runtime or a broken grammar can fail tens of thousands of
+    // files identically; the first 20 say everything the rest would.
+    const MAX_ERRORS_SHOWN = 20;
+    for (const e of g.errors.slice(0, MAX_ERRORS_SHOWN)) console.error(`✗ ${e}`);
+    if (g.errors.length > MAX_ERRORS_SHOWN) console.error(`✗ … and ${g.errors.length - MAX_ERRORS_SHOWN} more (${g.errors.length} files failed to parse)`);
 
     const rel = relative(process.cwd(), g.contextDir) || "graft";
     if (process.env.GRAFT_NO_GITIGNORE) {

src/engine.ts

@@ -49,6 +49,9 @@ export interface GraphRunOptions {
   lsp?: boolean;
   /** Repo-relative directory prefixes to limit the build to (`--only-dir`). */
   onlyDirs?: string[];
+  /** Parser child processes for a large cold build (`graft build --workers`);
+   * `"auto"` sizes the pool. Absent/0 parses in-thread. */
+  parseWorkers?: number | "auto";
   onProgress?: GraphBuildOptions["onProgress"];
 }
 
@@ -93,6 +96,7 @@ export class Graft {
       contextDir: this.cfg.contextDir,
       summarizer: opts.llm ? this.cruxSummarizer() : undefined,
       concurrency: opts.concurrency,
+      parseWorkers: opts.parseWorkers,
       reuse: opts.reuse,
       lsp: opts.lsp,
       onlyDirs: opts.onlyDirs,