perf: faster graph export — bundle and impact facts, same answers

gemini-extension.json

@@ -1,6 +1,6 @@
 {
   "name": "axiomcode",
-  "version": "0.1.9",
+  "version": "0.1.10",
   "description": "Ask your repository how its code connects: who calls this, what breaks if I change it, which tests an edit reaches, how A reaches B. Answers come from a resolved call graph and are verified against it; nothing is guessed.",
   "contextFileName": "plugins/axiomcode/AGENTS.md",
   "mcpServers": {

graph/bundle/build.ts

@@ -10,9 +10,19 @@
 import * as fs from 'fs';
 import * as path from 'path';
 
-import { Header, readHeader, readRaw, readRfc4180 } from '@/bundle/csv';
+import { Header, type Pick, readHeader, readRaw, readRfc4180 } from '@/bundle/csv';
 import type { LanguageAdapter, RawSource } from '@/bundle/languages';
 
+/** AXIOM_BUNDLE_PROFILE=1: one stderr line per stage with its wall time, for measuring the bundle. */
+const PROFILE = !!process.env.AXIOM_BUNDLE_PROFILE;
+let lapAt = Date.now();
+export function lap(what: string): void {
+  if (!PROFILE) return;
+  const now = Date.now();
+  process.stderr.write(`  [bundle-profile] ${((now - lapAt) / 1000).toFixed(2)}s ${what}\n`);
+  lapAt = now;
+}
+
 export type Cell = string | number | null;
 export type Row = Cell[];
 
@@ -114,9 +124,10 @@ async function libSource(inp: BuildInputs, file: string): Promise<EntitySource |
   return { path: facts, header: new Header(h.names, `${facts} (header from ${headerFile})`), hasHeaderRow: false };
 }
 
-async function* rowsOf(src: EntitySource): AsyncGenerator<string[]> {
+async function* rowsOf(src: EntitySource, pick?: Pick): AsyncGenerator<string[]> {
   let first = src.hasHeaderRow;
-  for await (const r of readRfc4180(src.path)) {
+  // the reader never filters a file's first row: here that is the header, and a headerless file is read whole
+  for await (const r of readRfc4180(src.path, src.hasHeaderRow ? pick : undefined)) {
     if (first) { first = false; continue; }
     yield r;
   }
@@ -136,6 +147,7 @@ export async function buildCore(inp: BuildInputs): Promise<CoreTables> {
   const rawFieldAccess = A.raw.fieldAccess ? await readSource(rawDir, A.raw.fieldAccess) : [];
   const rawTypeUse = A.raw.typeUse ? await readSource(rawDir, A.raw.typeUse) : [];
   const rawConfigBinding = A.raw.configBinding ? await readSource(rawDir, A.raw.configBinding) : [];
+  lap('read raw relations');
   log(`  raw: ${rawEdges.length} edge rows, ${rawAncestors.length} ancestor rows, ${rawOverrides.length} override rows`);
 
   // call_edges: split the raw ToMethod into a method key or a label by provenance
@@ -179,6 +191,7 @@ export async function buildCore(inp: BuildInputs): Promise<CoreTables> {
     return [ref!, nul(type), context!, int(depth) ?? 0, ownerKind!, owner!, nul(enclMethod), nul(enclType), nul(prov), tier!];
   });
 
+  lap('transform raw rows (call_edges, field_access, type_use)');
   // ── 2. which ids the bundle must name ─────────────────────────────────────
   const wantMethods = new Set<string>();
   const wantTypes = new Set<string>();
@@ -211,6 +224,7 @@ export async function buildCore(inp: BuildInputs): Promise<CoreTables> {
     if (r[2] === 'field' && r[3] !== null) wantFields.add(r[3] as string);
   }
 
+  lap('collect wanted ids');
   // ── 3. client entities (all of them — the client is the subject) ──────────
   // The IR spells visibility differently per entity file (PUBLIC on methods/types, PUBLIC_ACCESS on
   // fields); normalise to one vocabulary so a rule can match on it. '' when the language has none.
@@ -367,6 +381,7 @@ export async function buildCore(inp: BuildInputs): Promise<CoreTables> {
     }
   }
 
+  lap('read client modules/methods/types/fields IR');
   // ── 4. library entities — only the referenced ones ────────────────────────
   const missingMethods = new Set([...wantMethods].filter((id) => !methods.has(id)));
   const missingTypesEarly = [...wantTypes].some((id) => !types.has(id));
@@ -447,6 +462,7 @@ export async function buildCore(inp: BuildInputs): Promise<CoreTables> {
   if (FI?.ownerQualifiedNameFromType) fillOwnerNames(fields, 3, 4);
   // A library module's file path is not staged; a library site never occurs (edges start in the client).
 
+  lap('read library entities + generated/external');
   // ── 5. call sites: name + position for every site the edges mention ───────
   // row: id, caller_id, kind, callee_name, file_path, start_line, start_column, end_line, end_column
   const sites = new Map<string, Row>();
@@ -466,7 +482,7 @@ export async function buildCore(inp: BuildInputs): Promise<CoreTables> {
       const h = src.header;
       const ce = h.col(C.expressionId), cn = h.col(C.calleeName), cl = h.col(C.startLine), cc = h.col(C.startColumn);
       const cel = C.endLine ? h.col(C.endLine) : -1;
-      for await (const r of rowsOf(src)) {
+      for await (const r of rowsOf(src, { column: ce, has: (id) => sites.has(id) })) {
         const row = sites.get(r[ce] ?? '');
         if (!row) continue;
         fill(row, 3, nul(r[cn])); fill(row, 4, fileOf(C.fileVia, h, r));
@@ -475,6 +491,7 @@ export async function buildCore(inp: BuildInputs): Promise<CoreTables> {
       }
     }
   }
+  lap('position sites from call-site IR');
   {
     const E = A.ir.expressions;
     const src = await clientSource(inp.clientIrDir, E.file);
@@ -483,7 +500,7 @@ export async function buildCore(inp: BuildInputs): Promise<CoreTables> {
       const ci = h.col(E.id), ck = h.col(E.kind), cl = h.col(E.startLine), cc = h.col(E.startColumn), cel = h.col(E.endLine), cec = h.col(E.endColumn);
       const cname = E.calleeName ? h.col(E.calleeName.column) : -1;
       const nameKinds = new Set(E.calleeName?.kinds ?? []);
-      for await (const r of rowsOf(src)) {
+      for await (const r of rowsOf(src, { column: ci, has: (id) => sites.has(id) || fieldSites.has(id) })) {
         const id = r[ci] ?? '';
         // A FIELD ACCESS site is an expression too, and this is the one pass over a table that
         // can be GB-scale, so both are positioned here rather than streaming it twice.
@@ -501,6 +518,7 @@ export async function buildCore(inp: BuildInputs): Promise<CoreTables> {
       }
     }
   }
+  lap('position sites from expressions IR');
   if (A.ir.decorators) {
     const D = A.ir.decorators;
     const src = await clientSource(inp.clientIrDir, D.file);
@@ -572,6 +590,7 @@ export async function buildCore(inp: BuildInputs): Promise<CoreTables> {
   const unplaced = [...sites.values()].filter((r) => r[5] === null).length;
   if (unplaced > 0) log(`  ! ${unplaced} of ${sites.size} call sites have no position in the IR`);
 
+  lap('position sites from decorators/locals/field sites + fallbacks');
   // ── 5b. what the parser did NOT read ──────────────────────────────────────
   // A file the parser skipped has no row in any table above, and until #1148 nothing in the
   // bundle said so: a consumer could not tell a file with no calls from a file that was
@@ -599,9 +618,10 @@ export async function buildCore(inp: BuildInputs): Promise<CoreTables> {
   }
   if (skipped.length > 0) log(`  skipped: ${skipped.length} file(s) the parser did not read`);
 
+  lap('read skipped');
   // ── 6. assemble ───────────────────────────────────────────────────────────
   const run: Row[] = Object.entries(inp.meta).map(([k, v]) => [k, v]);
-  return {
+  const core = {
     run,
     methods: [...methods.values()],
     types: [...types.values()],
@@ -619,6 +639,8 @@ export async function buildCore(inp: BuildInputs): Promise<CoreTables> {
     type_use,
     skipped: dedupe(skipped),
   };
+  lap('assemble + dedupe');
+  return core;
 }
 
 function dedupe(rows: Row[]): Row[] {

graph/bundle/catalog.ts

@@ -16,6 +16,21 @@ export interface ExtRelation {
   description: string;
 }
 
+/**
+ * The ext relations the query layer reads (plugins/, hooks/: impact, path, context, test-impact, the MCP server).
+ * Only these get the c0 index: an index on a relation nothing queries is time and disk at every build, and the
+ * schema promises the table, not an index on it. Not a filter on what is loaded — every manifest relation is still
+ * an ext table. graph/test/tools/ext-queried-test.sh fails when a script names an ext relation missing here.
+ */
+export const EXT_QUERIED: ReadonlySet<string> = new Set([
+  'bean_def', 'call_runs_edge', 'call_site_unresolved', 'config_affects_method', 'config_binding', 'config_key_ref',
+  'config_unresolved', 'ctor_implicit_type', 'decorated_name_target', 'di_edge', 'fn_value_call', 'framework_edge',
+  'inject_point', 'lib_callback_edge', 'lib_callback_site', 'library_receiver', 'method_returns_method',
+  'method_returns_param', 'persistence_field', 'remote_edge', 'remote_unsent', 'remote_unserved', 'return_value',
+  'state_call_alloc', 'state_call_open', 'state_gate', 'state_gate_alloc', 'state_world_of_gated',
+  'type_base_unresolved', 'unresolved_value_callee',
+]);
+
 /** relation<TAB>file per line; comments and blanks skipped. */
 export function readManifest(langDir: string): { relation: string; file: string }[] {
   const p = path.join(langDir, 'souffle', 'export_manifest.tsv');

graph/bundle/cli.ts

@@ -17,7 +17,7 @@ import * as path from 'path';
 process.removeAllListeners('warning');
 process.on('warning', (w) => { if (w.name !== 'ExperimentalWarning') console.error(w.stack ?? w.message); });
 
-import { buildCore } from '@/bundle/build';
+import { buildCore, lap } from '@/bundle/build';
 import { catalogExt, readLibMap } from '@/bundle/catalog';
 import { adapterFor } from '@/bundle/languages';
 import { renderSchemaMarkdown, SCHEMA_VERSION } from '@/bundle/schema';
@@ -91,7 +91,9 @@ export async function main(argv = process.argv.slice(2)): Promise<void> {
 
   const dbPath = path.join(a.out!, 'graph.sqlite');
   if (haveSqlite) {
-    await writeSqlite({ dbPath, language: adapter.language, core, ext: catalogExt(langDir), rawDir: a.raw!, log });
+    const ext = catalogExt(langDir);
+    lap('lift ext docs (catalogExt)');
+    await writeSqlite({ dbPath, language: adapter.language, core, ext, rawDir: a.raw!, log });
     log(`▶ wrote ${dbPath}${a.debug ? ' (+ csv/*.csv, --debug)' : ''}`);
   } else {
     fs.rmSync(dbPath, { force: true });

graph/bundle/csv.ts

@@ -14,43 +14,120 @@ import * as fs from 'fs';
 
 const TAB = '\t';
 
-/** One RFC 4180 (tab-delimited) file as an async stream of rows. A zero-byte file yields nothing. */
-export async function* readRfc4180(path: string): AsyncGenerator<string[]> {
+/** One RFC 4180 (tab-delimited) file as an async stream of rows. A zero-byte file yields nothing.
+ *
+ *  A LINE WITH NO QUOTE IN IT IS A WHOLE ROW, and nearly every line is one, so it is split on tabs as it
+ *  stands: a quote is the only thing that can make a field hold a tab or a newline. Only a line that holds
+ *  a quote, and the lines a quoted field carries it across, goes through the character-by-character reader
+ *  (`Rfc4180Row`), which is the definition. Reading every character that way cost most of the bundle's time
+ *  on a GB-scale expression table. */
+export async function* readRfc4180(path: string, pick?: Pick): AsyncGenerator<string[]> {
   const stream = fs.createReadStream(path, { encoding: 'utf8', highWaterMark: 1 << 20 });
-  let field = '';
-  let row: string[] = [];
-  let quoted = false; // inside a quoted field
-  let afterQuote = false; // just saw a quote inside a quoted field — next char decides
-  let atFieldStart = true;
+  const slow = new Rfc4180Row();
+  let rest = '';
+  let first = true; // the first row, a header in a headered file, is never filtered
   for await (const chunk of stream as AsyncIterable<string>) {
-    for (let i = 0; i < chunk.length; i++) {
+    const text = rest.length > 0 ? rest + chunk : chunk;
+    let at = 0;
+    let quote = text.indexOf('"'); // the next quote at or after `at`, -1 when the chunk holds no more
+    for (;;) {
+      const nl = text.indexOf('\n', at);
+      if (nl < 0) break;
+      if (quote >= 0 && quote < at) quote = text.indexOf('"', at);
+      if (slow.open || (quote >= 0 && quote < nl)) {
+        // the line holds a quote, or a quoted field opened on an earlier line runs on into it
+        const row = slow.feed(text, at, nl + 1);
+        if (row) yield row;
+      } else if (first || !pick || pick.has(fieldAt(text, at, nl, pick.column))) {
+        yield splitLine(text, at, nl);
+      }
+      first = false;
+      at = nl + 1;
+    }
+    rest = text.slice(at);
+  }
+  // final row without a trailing newline
+  if (slow.open || rest.indexOf('"') >= 0) {
+    slow.feed(rest, 0, rest.length);
+    const row = slow.end();
+    if (row) yield row;
+  } else if (rest.length > 0 && (first || !pick || pick.has(fieldAt(rest, 0, rest.length, pick.column)))) {
+    yield splitLine(rest, 0, rest.length);
+  }
+}
+
+/** Rows the caller will drop, skipped before they are split: a row is read only when `has` holds for its
+ *  `column`th field ('' when the row is shorter, as `row[column] ?? ''` reads it). A row a quote touches is
+ *  always returned, so the caller must still test every row it gets; this only saves the split of the rest. */
+export interface Pick { column: number; has: (value: string) => boolean }
+
+/** The `col`th tab-separated field of text[a, b), a line holding no quote; a trailing \r on the last field dropped. */
+function fieldAt(text: string, a: number, b: number, col: number): string {
+  let start = a;
+  for (let i = 0; i < col; i++) {
+    const t = text.indexOf(TAB, start);
+    if (t < 0 || t >= b) return '';
+    start = t + 1;
+  }
+  const t = text.indexOf(TAB, start);
+  if (t >= 0 && t < b) return text.slice(start, t);
+  const v = text.slice(start, b);
+  return v.endsWith('\r') ? v.slice(0, -1) : v;
+}
+
+/** text[a, b) — one line holding no quote, without its newline — as a row: split on tabs, a trailing \r dropped. */
+function splitLine(text: string, a: number, b: number): string[] {
+  const line = text.slice(a, b);
+  const row = line.split(TAB);
+  const last = row[row.length - 1]!;
+  if (last.endsWith('\r')) row[row.length - 1] = last.slice(0, -1);
+  return row;
+}
+
+/** The RFC 4180 reader, a character at a time: fed whole lines, it returns a row when one ends on a newline outside quotes. */
+class Rfc4180Row {
+  private field = '';
+  private row: string[] = [];
+  private quoted = false; // inside a quoted field
+  private afterQuote = false; // just saw a quote inside a quoted field — next char decides
+  private atFieldStart = true;
+  /** a row has begun and not ended: a quoted field crossed a newline */
+  get open(): boolean { return this.quoted || this.afterQuote || this.row.length > 0 || this.field.length > 0 || !this.atFieldStart; }
+  feed(chunk: string, from: number, to: number): string[] | null {
+    let out: string[] | null = null;
+    for (let i = from; i < to; i++) {
       const ch = chunk[i]!;
-      if (quoted) {
-        if (afterQuote) {
-          afterQuote = false;
-          if (ch === '"') { field += '"'; continue; } // doubled quote → literal quote
-          quoted = false; // the closing quote — fall through to unquoted handling
+      if (this.quoted) {
+        if (this.afterQuote) {
+          this.afterQuote = false;
+          if (ch === '"') { this.field += '"'; continue; } // doubled quote → literal quote
+          this.quoted = false; // the closing quote — fall through to unquoted handling
         } else {
-          if (ch === '"') afterQuote = true; else field += ch;
+          if (ch === '"') this.afterQuote = true; else this.field += ch;
           continue;
         }
       }
-      if (ch === '"' && atFieldStart) { quoted = true; atFieldStart = false; continue; }
-      atFieldStart = false;
-      if (ch === TAB) { row.push(field); field = ''; atFieldStart = true; continue; }
+      if (ch === '"' && this.atFieldStart) { this.quoted = true; this.atFieldStart = false; continue; }
+      this.atFieldStart = false;
+      if (ch === TAB) { this.row.push(this.field); this.field = ''; this.atFieldStart = true; continue; }
       if (ch === '\n') {
-        row.push(field.endsWith('\r') ? field.slice(0, -1) : field);
-        field = ''; atFieldStart = true;
-        yield row; row = [];
+        this.row.push(this.field.endsWith('\r') ? this.field.slice(0, -1) : this.field);
+        this.field = ''; this.atFieldStart = true;
+        out = this.row; this.row = [];
         continue;
       }
-      field += ch;
+      this.field += ch;
     }
+    return out;
   }
-  // final row without a trailing newline
-  if (field.length > 0 || row.length > 0) {
-    row.push(field.endsWith('\r') ? field.slice(0, -1) : field);
-    yield row;
+  /** end of input: the row in progress, if any */
+  end(): string[] | null {
+    if (this.field.length > 0 || this.row.length > 0) {
+      this.row.push(this.field.endsWith('\r') ? this.field.slice(0, -1) : this.field);
+      const r = this.row; this.row = []; this.field = '';
+      return r;
+    }
+    return null;
   }
 }