diff --git a/gemini-extension.json b/gemini-extension.json index ebb25063d..4e8cef176 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,6 +1,6 @@ { "name": "axiomcode", - "version": "0.1.9", + "version": "0.1.10", "description": "Ask your repository how its code connects: who calls this, what breaks if I change it, which tests an edit reaches, how A reaches B. Answers come from a resolved call graph and are verified against it; nothing is guessed.", "contextFileName": "plugins/axiomcode/AGENTS.md", "mcpServers": { diff --git a/graph/bundle/build.ts b/graph/bundle/build.ts index 84c9fcb9e..d066ead75 100644 --- a/graph/bundle/build.ts +++ b/graph/bundle/build.ts @@ -10,9 +10,19 @@ import * as fs from 'fs'; import * as path from 'path'; -import { Header, readHeader, readRaw, readRfc4180 } from '@/bundle/csv'; +import { Header, type Pick, readHeader, readRaw, readRfc4180 } from '@/bundle/csv'; import type { LanguageAdapter, RawSource } from '@/bundle/languages'; +/** AXIOM_BUNDLE_PROFILE=1: one stderr line per stage with its wall time, for measuring the bundle. */ +const PROFILE = !!process.env.AXIOM_BUNDLE_PROFILE; +let lapAt = Date.now(); +export function lap(what: string): void { + if (!PROFILE) return; + const now = Date.now(); + process.stderr.write(` [bundle-profile] ${((now - lapAt) / 1000).toFixed(2)}s ${what}\n`); + lapAt = now; +} + export type Cell = string | number | null; export type Row = Cell[]; @@ -114,9 +124,10 @@ async function libSource(inp: BuildInputs, file: string): Promise { +async function* rowsOf(src: EntitySource, pick?: Pick): AsyncGenerator { let first = src.hasHeaderRow; - for await (const r of readRfc4180(src.path)) { + // the reader never filters a file's first row: here that is the header, and a headerless file is read whole + for await (const r of readRfc4180(src.path, src.hasHeaderRow ? pick : undefined)) { if (first) { first = false; continue; } yield r; } @@ -136,6 +147,7 @@ export async function buildCore(inp: BuildInputs): Promise { const rawFieldAccess = A.raw.fieldAccess ? await readSource(rawDir, A.raw.fieldAccess) : []; const rawTypeUse = A.raw.typeUse ? await readSource(rawDir, A.raw.typeUse) : []; const rawConfigBinding = A.raw.configBinding ? await readSource(rawDir, A.raw.configBinding) : []; + lap('read raw relations'); log(` raw: ${rawEdges.length} edge rows, ${rawAncestors.length} ancestor rows, ${rawOverrides.length} override rows`); // call_edges: split the raw ToMethod into a method key or a label by provenance @@ -179,6 +191,7 @@ export async function buildCore(inp: BuildInputs): Promise { return [ref!, nul(type), context!, int(depth) ?? 0, ownerKind!, owner!, nul(enclMethod), nul(enclType), nul(prov), tier!]; }); + lap('transform raw rows (call_edges, field_access, type_use)'); // ── 2. which ids the bundle must name ───────────────────────────────────── const wantMethods = new Set(); const wantTypes = new Set(); @@ -211,6 +224,7 @@ export async function buildCore(inp: BuildInputs): Promise { if (r[2] === 'field' && r[3] !== null) wantFields.add(r[3] as string); } + lap('collect wanted ids'); // ── 3. client entities (all of them — the client is the subject) ────────── // The IR spells visibility differently per entity file (PUBLIC on methods/types, PUBLIC_ACCESS on // fields); normalise to one vocabulary so a rule can match on it. '' when the language has none. @@ -367,6 +381,7 @@ export async function buildCore(inp: BuildInputs): Promise { } } + lap('read client modules/methods/types/fields IR'); // ── 4. library entities — only the referenced ones ──────────────────────── const missingMethods = new Set([...wantMethods].filter((id) => !methods.has(id))); const missingTypesEarly = [...wantTypes].some((id) => !types.has(id)); @@ -447,6 +462,7 @@ export async function buildCore(inp: BuildInputs): Promise { if (FI?.ownerQualifiedNameFromType) fillOwnerNames(fields, 3, 4); // A library module's file path is not staged; a library site never occurs (edges start in the client). + lap('read library entities + generated/external'); // ── 5. call sites: name + position for every site the edges mention ─────── // row: id, caller_id, kind, callee_name, file_path, start_line, start_column, end_line, end_column const sites = new Map(); @@ -466,7 +482,7 @@ export async function buildCore(inp: BuildInputs): Promise { const h = src.header; const ce = h.col(C.expressionId), cn = h.col(C.calleeName), cl = h.col(C.startLine), cc = h.col(C.startColumn); const cel = C.endLine ? h.col(C.endLine) : -1; - for await (const r of rowsOf(src)) { + for await (const r of rowsOf(src, { column: ce, has: (id) => sites.has(id) })) { const row = sites.get(r[ce] ?? ''); if (!row) continue; fill(row, 3, nul(r[cn])); fill(row, 4, fileOf(C.fileVia, h, r)); @@ -475,6 +491,7 @@ export async function buildCore(inp: BuildInputs): Promise { } } } + lap('position sites from call-site IR'); { const E = A.ir.expressions; const src = await clientSource(inp.clientIrDir, E.file); @@ -483,7 +500,7 @@ export async function buildCore(inp: BuildInputs): Promise { const ci = h.col(E.id), ck = h.col(E.kind), cl = h.col(E.startLine), cc = h.col(E.startColumn), cel = h.col(E.endLine), cec = h.col(E.endColumn); const cname = E.calleeName ? h.col(E.calleeName.column) : -1; const nameKinds = new Set(E.calleeName?.kinds ?? []); - for await (const r of rowsOf(src)) { + for await (const r of rowsOf(src, { column: ci, has: (id) => sites.has(id) || fieldSites.has(id) })) { const id = r[ci] ?? ''; // A FIELD ACCESS site is an expression too, and this is the one pass over a table that // can be GB-scale, so both are positioned here rather than streaming it twice. @@ -501,6 +518,7 @@ export async function buildCore(inp: BuildInputs): Promise { } } } + lap('position sites from expressions IR'); if (A.ir.decorators) { const D = A.ir.decorators; const src = await clientSource(inp.clientIrDir, D.file); @@ -572,6 +590,7 @@ export async function buildCore(inp: BuildInputs): Promise { const unplaced = [...sites.values()].filter((r) => r[5] === null).length; if (unplaced > 0) log(` ! ${unplaced} of ${sites.size} call sites have no position in the IR`); + lap('position sites from decorators/locals/field sites + fallbacks'); // ── 5b. what the parser did NOT read ────────────────────────────────────── // A file the parser skipped has no row in any table above, and until #1148 nothing in the // bundle said so: a consumer could not tell a file with no calls from a file that was @@ -599,9 +618,10 @@ export async function buildCore(inp: BuildInputs): Promise { } if (skipped.length > 0) log(` skipped: ${skipped.length} file(s) the parser did not read`); + lap('read skipped'); // ── 6. assemble ─────────────────────────────────────────────────────────── const run: Row[] = Object.entries(inp.meta).map(([k, v]) => [k, v]); - return { + const core = { run, methods: [...methods.values()], types: [...types.values()], @@ -619,6 +639,8 @@ export async function buildCore(inp: BuildInputs): Promise { type_use, skipped: dedupe(skipped), }; + lap('assemble + dedupe'); + return core; } function dedupe(rows: Row[]): Row[] { diff --git a/graph/bundle/catalog.ts b/graph/bundle/catalog.ts index 3276e855b..9e216e7d6 100644 --- a/graph/bundle/catalog.ts +++ b/graph/bundle/catalog.ts @@ -16,6 +16,21 @@ export interface ExtRelation { description: string; } +/** + * The ext relations the query layer reads (plugins/, hooks/: impact, path, context, test-impact, the MCP server). + * Only these get the c0 index: an index on a relation nothing queries is time and disk at every build, and the + * schema promises the table, not an index on it. Not a filter on what is loaded — every manifest relation is still + * an ext table. graph/test/tools/ext-queried-test.sh fails when a script names an ext relation missing here. + */ +export const EXT_QUERIED: ReadonlySet = new Set([ + 'bean_def', 'call_runs_edge', 'call_site_unresolved', 'config_affects_method', 'config_binding', 'config_key_ref', + 'config_unresolved', 'ctor_implicit_type', 'decorated_name_target', 'di_edge', 'fn_value_call', 'framework_edge', + 'inject_point', 'lib_callback_edge', 'lib_callback_site', 'library_receiver', 'method_returns_method', + 'method_returns_param', 'persistence_field', 'remote_edge', 'remote_unsent', 'remote_unserved', 'return_value', + 'state_call_alloc', 'state_call_open', 'state_gate', 'state_gate_alloc', 'state_world_of_gated', + 'type_base_unresolved', 'unresolved_value_callee', +]); + /** relationfile per line; comments and blanks skipped. */ export function readManifest(langDir: string): { relation: string; file: string }[] { const p = path.join(langDir, 'souffle', 'export_manifest.tsv'); diff --git a/graph/bundle/cli.ts b/graph/bundle/cli.ts index ad975fad2..36a26c0c0 100644 --- a/graph/bundle/cli.ts +++ b/graph/bundle/cli.ts @@ -17,7 +17,7 @@ import * as path from 'path'; process.removeAllListeners('warning'); process.on('warning', (w) => { if (w.name !== 'ExperimentalWarning') console.error(w.stack ?? w.message); }); -import { buildCore } from '@/bundle/build'; +import { buildCore, lap } from '@/bundle/build'; import { catalogExt, readLibMap } from '@/bundle/catalog'; import { adapterFor } from '@/bundle/languages'; import { renderSchemaMarkdown, SCHEMA_VERSION } from '@/bundle/schema'; @@ -91,7 +91,9 @@ export async function main(argv = process.argv.slice(2)): Promise { const dbPath = path.join(a.out!, 'graph.sqlite'); if (haveSqlite) { - await writeSqlite({ dbPath, language: adapter.language, core, ext: catalogExt(langDir), rawDir: a.raw!, log }); + const ext = catalogExt(langDir); + lap('lift ext docs (catalogExt)'); + await writeSqlite({ dbPath, language: adapter.language, core, ext, rawDir: a.raw!, log }); log(`▶ wrote ${dbPath}${a.debug ? ' (+ csv/*.csv, --debug)' : ''}`); } else { fs.rmSync(dbPath, { force: true }); diff --git a/graph/bundle/csv.ts b/graph/bundle/csv.ts index 43fc7ef97..befee56b5 100644 --- a/graph/bundle/csv.ts +++ b/graph/bundle/csv.ts @@ -14,43 +14,120 @@ import * as fs from 'fs'; const TAB = '\t'; -/** One RFC 4180 (tab-delimited) file as an async stream of rows. A zero-byte file yields nothing. */ -export async function* readRfc4180(path: string): AsyncGenerator { +/** One RFC 4180 (tab-delimited) file as an async stream of rows. A zero-byte file yields nothing. + * + * A LINE WITH NO QUOTE IN IT IS A WHOLE ROW, and nearly every line is one, so it is split on tabs as it + * stands: a quote is the only thing that can make a field hold a tab or a newline. Only a line that holds + * a quote, and the lines a quoted field carries it across, goes through the character-by-character reader + * (`Rfc4180Row`), which is the definition. Reading every character that way cost most of the bundle's time + * on a GB-scale expression table. */ +export async function* readRfc4180(path: string, pick?: Pick): AsyncGenerator { const stream = fs.createReadStream(path, { encoding: 'utf8', highWaterMark: 1 << 20 }); - let field = ''; - let row: string[] = []; - let quoted = false; // inside a quoted field - let afterQuote = false; // just saw a quote inside a quoted field — next char decides - let atFieldStart = true; + const slow = new Rfc4180Row(); + let rest = ''; + let first = true; // the first row, a header in a headered file, is never filtered for await (const chunk of stream as AsyncIterable) { - for (let i = 0; i < chunk.length; i++) { + const text = rest.length > 0 ? rest + chunk : chunk; + let at = 0; + let quote = text.indexOf('"'); // the next quote at or after `at`, -1 when the chunk holds no more + for (;;) { + const nl = text.indexOf('\n', at); + if (nl < 0) break; + if (quote >= 0 && quote < at) quote = text.indexOf('"', at); + if (slow.open || (quote >= 0 && quote < nl)) { + // the line holds a quote, or a quoted field opened on an earlier line runs on into it + const row = slow.feed(text, at, nl + 1); + if (row) yield row; + } else if (first || !pick || pick.has(fieldAt(text, at, nl, pick.column))) { + yield splitLine(text, at, nl); + } + first = false; + at = nl + 1; + } + rest = text.slice(at); + } + // final row without a trailing newline + if (slow.open || rest.indexOf('"') >= 0) { + slow.feed(rest, 0, rest.length); + const row = slow.end(); + if (row) yield row; + } else if (rest.length > 0 && (first || !pick || pick.has(fieldAt(rest, 0, rest.length, pick.column)))) { + yield splitLine(rest, 0, rest.length); + } +} + +/** Rows the caller will drop, skipped before they are split: a row is read only when `has` holds for its + * `column`th field ('' when the row is shorter, as `row[column] ?? ''` reads it). A row a quote touches is + * always returned, so the caller must still test every row it gets; this only saves the split of the rest. */ +export interface Pick { column: number; has: (value: string) => boolean } + +/** The `col`th tab-separated field of text[a, b), a line holding no quote; a trailing \r on the last field dropped. */ +function fieldAt(text: string, a: number, b: number, col: number): string { + let start = a; + for (let i = 0; i < col; i++) { + const t = text.indexOf(TAB, start); + if (t < 0 || t >= b) return ''; + start = t + 1; + } + const t = text.indexOf(TAB, start); + if (t >= 0 && t < b) return text.slice(start, t); + const v = text.slice(start, b); + return v.endsWith('\r') ? v.slice(0, -1) : v; +} + +/** text[a, b) — one line holding no quote, without its newline — as a row: split on tabs, a trailing \r dropped. */ +function splitLine(text: string, a: number, b: number): string[] { + const line = text.slice(a, b); + const row = line.split(TAB); + const last = row[row.length - 1]!; + if (last.endsWith('\r')) row[row.length - 1] = last.slice(0, -1); + return row; +} + +/** The RFC 4180 reader, a character at a time: fed whole lines, it returns a row when one ends on a newline outside quotes. */ +class Rfc4180Row { + private field = ''; + private row: string[] = []; + private quoted = false; // inside a quoted field + private afterQuote = false; // just saw a quote inside a quoted field — next char decides + private atFieldStart = true; + /** a row has begun and not ended: a quoted field crossed a newline */ + get open(): boolean { return this.quoted || this.afterQuote || this.row.length > 0 || this.field.length > 0 || !this.atFieldStart; } + feed(chunk: string, from: number, to: number): string[] | null { + let out: string[] | null = null; + for (let i = from; i < to; i++) { const ch = chunk[i]!; - if (quoted) { - if (afterQuote) { - afterQuote = false; - if (ch === '"') { field += '"'; continue; } // doubled quote → literal quote - quoted = false; // the closing quote — fall through to unquoted handling + if (this.quoted) { + if (this.afterQuote) { + this.afterQuote = false; + if (ch === '"') { this.field += '"'; continue; } // doubled quote → literal quote + this.quoted = false; // the closing quote — fall through to unquoted handling } else { - if (ch === '"') afterQuote = true; else field += ch; + if (ch === '"') this.afterQuote = true; else this.field += ch; continue; } } - if (ch === '"' && atFieldStart) { quoted = true; atFieldStart = false; continue; } - atFieldStart = false; - if (ch === TAB) { row.push(field); field = ''; atFieldStart = true; continue; } + if (ch === '"' && this.atFieldStart) { this.quoted = true; this.atFieldStart = false; continue; } + this.atFieldStart = false; + if (ch === TAB) { this.row.push(this.field); this.field = ''; this.atFieldStart = true; continue; } if (ch === '\n') { - row.push(field.endsWith('\r') ? field.slice(0, -1) : field); - field = ''; atFieldStart = true; - yield row; row = []; + this.row.push(this.field.endsWith('\r') ? this.field.slice(0, -1) : this.field); + this.field = ''; this.atFieldStart = true; + out = this.row; this.row = []; continue; } - field += ch; + this.field += ch; } + return out; } - // final row without a trailing newline - if (field.length > 0 || row.length > 0) { - row.push(field.endsWith('\r') ? field.slice(0, -1) : field); - yield row; + /** end of input: the row in progress, if any */ + end(): string[] | null { + if (this.field.length > 0 || this.row.length > 0) { + this.row.push(this.field.endsWith('\r') ? this.field.slice(0, -1) : this.field); + const r = this.row; this.row = []; this.field = ''; + return r; + } + return null; } } diff --git a/graph/bundle/write.ts b/graph/bundle/write.ts index d00899fee..6898777b5 100644 --- a/graph/bundle/write.ts +++ b/graph/bundle/write.ts @@ -7,8 +7,8 @@ import * as fs from 'fs'; import * as path from 'path'; import { readRaw, writeTable } from '@/bundle/csv'; -import type { CoreTables, Row } from '@/bundle/build'; -import type { ExtRelation } from '@/bundle/catalog'; +import { lap, type CoreTables, type Row } from '@/bundle/build'; +import { EXT_QUERIED, type ExtRelation } from '@/bundle/catalog'; import { CATALOG_TABLES, CORE_TABLES, GUIDE, LANGUAGES, NOTES, QUERIES, SCHEMA_VERSION, VOCAB, type Language, type TableSpec } from '@/bundle/schema'; export function writeCoreCsv(graphDir: string, core: CoreTables, log: (s: string) => void): void { @@ -48,6 +48,7 @@ export interface SqliteInputs { // VALUES list amortizes both; 128 rows per statement keeps the flattened argument array well // under sqlite's bind limit at the widest table. const BATCH = Math.max(1, Number(process.env.AXIOM_BUNDLE_BATCH ?? 128) || 128); +const PAGE_SIZE = [4096, 8192, 16384, 32768, 65536].includes(Number(process.env.AXIOM_BUNDLE_PAGE_SIZE)) ? Number(process.env.AXIOM_BUNDLE_PAGE_SIZE) : 16384; type DbLike = { prepare(sql: string): { run(...a: (string | number | null)[]): unknown } }; function batchInsert(db: DbLike, table: string, arity: number, verb: string): { push(r: (string | number | null)[]): void; flush(): void } { const one = `(${Array.from({ length: arity }, () => '?').join(', ')})`; @@ -72,12 +73,35 @@ function batchInsert(db: DbLike, table: string, arity: number, verb: string): { }; } +/** + * A keyed table's rows in key order, for loading. A WITHOUT ROWID table is stored in key order whatever order its + * rows arrive in, so this changes nothing stored; but rows arriving in hash order land all over the B-tree and + * split its pages again and again, while rows in key order are appended. The sort is stable, so of two rows with + * one key the first is still the one INSERT OR IGNORE keeps. A copy is sorted: the caller's array is not reordered. + */ +function inKeyOrder(t: TableSpec, rows: Row[]): Row[] { + const keys = t.columns.flatMap((c, i) => (c.key ? [i] : [])); + if (keys.length === 0 || rows.length < 2) return rows; + const str = (v: Row[number] | undefined) => (v === null || v === undefined ? '' : String(v)); + return rows.slice().sort((a, b) => { + for (const i of keys) { + const x = str(a[i]), y = str(b[i]); + if (x !== y) return x < y ? -1 : 1; + } + return 0; + }); +} + export async function writeSqlite(inp: SqliteInputs): Promise { // Loaded lazily so the CSV path works on a Node without the module. const { DatabaseSync } = await import('node:sqlite'); fs.rmSync(inp.dbPath, { force: true }); fs.rmSync(inp.dbPath + '-journal', { force: true }); const db = new DatabaseSync(inp.dbPath); + // 16 KiB PAGES, set before the first table (AXIOM_BUNDLE_PAGE_SIZE overrides). The bundle is a bulk load of a file + // that reaches GBs, and at SQLite's default 4 KiB every table and index B-tree has four times the pages to split + // and write; the queries over the graph read it no slower. The page size changes no row and no answer. + db.exec(`PRAGMA page_size = ${PAGE_SIZE};`); db.exec('PRAGMA journal_mode = OFF; PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;'); try { // ── core ──────────────────────────────────────────────────────────────── @@ -86,11 +110,13 @@ export async function writeSqlite(inp: SqliteInputs): Promise { db.exec(createSql(t)); const rows = (inp.core as unknown as Record)[t.name] ?? []; const ins = batchInsert(db, t.name, t.columns.length, 'INSERT OR IGNORE'); - for (const r of rows) ins.push(r as (string | number | null)[]); + for (const r of inKeyOrder(t, rows)) ins.push(r as (string | number | null)[]); ins.flush(); inp.log(` sqlite ${t.name}: ${rows.length} rows`); + lap(`insert ${t.name} (${rows.length} rows)`); } db.exec('COMMIT;'); + lap('commit core'); // ── ext ───────────────────────────────────────────────────────────────── db.exec('BEGIN;'); @@ -107,8 +133,10 @@ export async function writeSqlite(inp: SqliteInputs): Promise { } ins.flush(); if (n > 0) inp.log(` sqlite ${name}: ${n} rows`); + lap(`insert ${name} (${n} rows)`); } db.exec('COMMIT;'); + lap('commit ext'); // ── catalog ───────────────────────────────────────────────────────────── db.exec('BEGIN;'); @@ -165,14 +193,17 @@ export async function writeSqlite(inp: SqliteInputs): Promise { const query = db.prepare('INSERT INTO schema_queries VALUES (?, ?, ?, ?)'); for (const q of QUERIES) query.run(q.name, q.question, q.params, q.sql); db.exec('COMMIT;'); + lap('catalog + vocab'); // ── indexes, after the load ───────────────────────────────────────────── db.exec('BEGIN;'); for (const t of CORE_TABLES) { - for (const cc of t.columns) if (cc.indexed) db.exec(`CREATE INDEX idx_${t.name}_${cc.name} ON ${t.name}(${cc.name});`); + for (const cc of t.columns) if (cc.indexed) { db.exec(`CREATE INDEX idx_${t.name}_${cc.name} ON ${t.name}(${cc.name});`); lap(`index ${t.name}.${cc.name}`); } } - for (const e of inp.ext) if (e.arity > 0) db.exec(`CREATE INDEX idx_ext_${e.relation}_c0 ON ext_${e.relation}(c0);`); + // an ext table the tools query keeps its c0 index; the rest are loaded and left unindexed (catalog.ts EXT_QUERIED) + for (const e of inp.ext) if (e.arity > 0 && EXT_QUERIED.has(e.relation)) { db.exec(`CREATE INDEX idx_ext_${e.relation}_c0 ON ext_${e.relation}(c0);`); lap(`index ext_${e.relation}.c0`); } db.exec('COMMIT;'); + lap('commit indexes'); db.exec(`PRAGMA user_version = ${Number(SCHEMA_VERSION)};`); } finally { db.close(); diff --git a/graph/test/tools/bundle-test.sh b/graph/test/tools/bundle-test.sh index 06ead93d8..8f3245bdf 100755 --- a/graph/test/tools/bundle-test.sh +++ b/graph/test/tools/bundle-test.sh @@ -310,6 +310,30 @@ if ! diff -q <(BUNDLE --print-schema) "$ROOT/graph/bundle/SCHEMA.md" >/dev/null; bad "graph/bundle/SCHEMA.md is stale — run: npm run schema-doc" fi +# 8. every ext relation a script queries is in EXT_QUERIED, so it keeps its c0 index (catalog.ts). The others are +# loaded unindexed: a new query on one of them would still answer, only slower, which no other check would see. +missing="$(python3 - "$ROOT" <<'PY' +import os, re, sys +root = sys.argv[1] +rels = set() +for lang in os.listdir(os.path.join(root, 'graph')): + m = os.path.join(root, 'graph', lang, 'souffle', 'export_manifest.tsv') + if os.path.isfile(m): + rels |= {l.split('\t')[0].strip() for l in open(m) if l.strip() and not l.lstrip().startswith('#')} +cat = open(os.path.join(root, 'graph', 'bundle', 'catalog.ts')).read() +queried = set(re.findall(r"'(\w+)'", cat[cat.index('EXT_QUERIED'):cat.index(']);', cat.index('EXT_QUERIED'))])) +used = set() +for d in ('plugins', 'hooks', 'bin'): + for dp, dns, fns in os.walk(os.path.join(root, d)): + dns[:] = [x for x in dns if x not in ('node_modules', 'dist', '.git')] + for fn in fns: + try: text = open(os.path.join(dp, fn), errors='replace').read() + except OSError: continue + used |= {r for r in re.findall(r'\bext_(\w+)', text) if r in rels} +print(' '.join(sorted(used - queried))) +PY +)" +[ -z "$missing" ] || bad "a script queries ext relation(s) missing from EXT_QUERIED in graph/bundle/catalog.ts (no c0 index): $missing" # ── the DEFAULT writes the database and nothing else ──────────────────────── # The deliverable is graph.sqlite. csv/*.csv carries the same core tables, so writing @@ -323,7 +347,7 @@ if [ -d "$d/csv" ] && [ -n "$(ls -A "$d/csv" 2>/dev/null)" ]; then fi if [ "$fail" -eq 0 ]; then - echo "bundle: ok (3 languages$( [ "$HAVE_SQLITE" = 1 ] && echo ', one schema across them, canonical queries run' || echo ', csv only — no sqlite3 CLI'), schema doc current)" + echo "bundle: ok (3 languages$( [ "$HAVE_SQLITE" = 1 ] && echo ', one schema across them, canonical queries run' || echo ', csv only — no sqlite3 CLI'), schema doc current, queried ext relations indexed)" else echo "bundle: $fail failure(s)"; exit 1 fi diff --git a/package.json b/package.json index 448fe5753..fbff87117 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@axiomcode/code-graph", - "version": "0.1.9", + "version": "0.1.10", "description": "AxiomCode Graph: a resolved call graph of your codebase grounded in formal methods, so you and your coding agents can see who calls what, what a change breaks, and which tests it reaches.", "repository": { "type": "git", @@ -31,11 +31,11 @@ ], "license": "FSL-1.1-Apache-2.0", "optionalDependencies": { - "@axiomcode/engine-darwin-arm64": "0.1.9", - "@axiomcode/engine-darwin-x64": "0.1.9", - "@axiomcode/engine-linux-x64": "0.1.9", - "@axiomcode/engine-linux-arm64": "0.1.9", - "@axiomcode/engine-win32-x64": "0.1.9" + "@axiomcode/engine-darwin-arm64": "0.1.10", + "@axiomcode/engine-darwin-x64": "0.1.10", + "@axiomcode/engine-linux-x64": "0.1.10", + "@axiomcode/engine-linux-arm64": "0.1.10", + "@axiomcode/engine-win32-x64": "0.1.10" }, "bin": { "axiomcode": "bin/axiomcode.js" diff --git a/plugins/axiomcode/.claude-plugin/plugin.json b/plugins/axiomcode/.claude-plugin/plugin.json index 1c32d1929..98067cc05 100644 --- a/plugins/axiomcode/.claude-plugin/plugin.json +++ b/plugins/axiomcode/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "axiomcode", "description": "Ask your repository how its code connects: who calls this, what breaks if I change it, which tests an edit reaches, how A reaches B. Answers come from a resolved call graph and are verified against it; nothing is guessed.", - "version": "0.1.9", + "version": "0.1.10", "author": { "name": "AxiomCode" } diff --git a/plugins/axiomcode/.codex-plugin/plugin.json b/plugins/axiomcode/.codex-plugin/plugin.json index 720690425..3940024ae 100644 --- a/plugins/axiomcode/.codex-plugin/plugin.json +++ b/plugins/axiomcode/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "axiomcode", - "version": "0.1.9", + "version": "0.1.10", "description": "Ask your repository how its code connects: who calls this, what breaks if I change it, which tests an edit reaches, how A reaches B. Answers come from a resolved call graph and are verified against it; nothing is guessed.", "author": { "name": "AxiomCode", "url": "https://github.com/AxiomCodeAI/axiomcodegraph" }, "homepage": "https://github.com/AxiomCodeAI/axiomcodegraph", diff --git a/plugins/axiomcode/.cursor-plugin/plugin.json b/plugins/axiomcode/.cursor-plugin/plugin.json index 3e3941cf7..ada8f9aab 100644 --- a/plugins/axiomcode/.cursor-plugin/plugin.json +++ b/plugins/axiomcode/.cursor-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "axiomcode", - "version": "0.1.9", + "version": "0.1.10", "description": "Ask your repository how its code connects: who calls this, what breaks if I change it, which tests an edit reaches, how A reaches B. Answers come from a resolved call graph and are verified against it; nothing is guessed. Java, TypeScript, Python, JavaScript.", "author": { "name": "AxiomCode" }, "homepage": "https://github.com/AxiomCodeAI/axiomcodegraph", diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_registration.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_registration.py index 0167e2c42..299232f86 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_registration.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_registration.py @@ -49,7 +49,7 @@ MAX_SITE_SPAN = 200 # a "call" spanning a whole file is a parse artefact, not a registration -def registrations(q, site_file=None): +def registrations(q, site_file=None, with_own_lines=False): """[(file, line, kind, key, why)] — every line that hands a declaration to something outside the graph. `kind` is always "route"; `key` is the string the framework dispatches on, VERBATIM and never normalised @@ -57,9 +57,11 @@ def registrations(q, site_file=None): client (`supertest.get('/orders/1')`, `TestClient(app).post('/orders')`) be joined to the handler it reaches: the two ends spell the same string and nothing else connects them. Normalising it belongs in the join, not here — `:id`, `{order_id}` and `` are three frameworks' spellings of one hole. + + `with_own_lines`: (rows, route_site_lines(q, site_file)) — the same walk computes both. """ if not _has(q, 'call_sites'): - return [] + return ([], {}) if with_own_lines else [] sf = site_file or (lambda x: x) lits = {} if _has(q, 'literals'): @@ -83,7 +85,7 @@ def registrations(q, site_file=None): # the cap and joins two unrelated tests. The DEPENDENT row is still true and still printed — a test that mounts a # handler does depend on it — so only the key is withheld. tf = {f for (f,) in q("SELECT DISTINCT file FROM symbols WHERE is_test = 1 AND file IS NOT NULL")} if _has(q, 'symbols') else set() - out, verbs, _own = _route_links(q, sf, lits, tf) + out, verbs, own = _route_links(q, sf, lits, tf) # `own` does not depend on tf, which only withholds a key # `router.route('/').get(h)` is two route-shaped calls on one line, and `route` said nothing about the method: the # line's HTTP verbs name it, in the order written. A chain registering two (`.get(a).post(b)`) names both, because # a line is all a reference records and it cannot say which argument list the handler sat in @@ -92,7 +94,8 @@ def registrations(q, site_file=None): if k == 'route' and vs: m = re.match(r'registered as a \w+ route (".*?") here', w) if m: out[(f, l)] = (k, key, f"registered as a {'/'.join(sorted(vs, key=vs.get))} route {m.group(1)} here — the router calls it, no call site does") - return sorted((ref_at.get((f, l), ''), f, l, k, key, w) for (f, l), (k, key, w) in out.items()) + rows = sorted((ref_at.get((f, l), ''), f, l, k, key, w) for (f, l), (k, key, w) in out.items()) + return (rows, own) if with_own_lines else rows def route_site_lines(q, site_file=None): @@ -289,6 +292,7 @@ class _Handlers: def __init__(self, q, read, sf): self.q, self.read, self.sf, self.names, self.consts, self.held = q, read, sf, {}, {}, {} + self.tables = None # const_value_tables, read on the first const asked about if _has(q, 'symbols'): for n, k, f, l in q("SELECT name, kind, file, line FROM symbols WHERE name IS NOT NULL"): self.names.setdefault(n, {}).setdefault(k or '', set()).add(f) @@ -310,7 +314,8 @@ def holds_fn(self, n, files=None): # a module a const requires may export the handler itself: `const h = require('./h'); app.get('/x', h)` required = any(re.search(rf'(? 0"""): callee_at.setdefault((sf(fp), l, col), set()).add(m) + return fns, rets, callee_at + + +def const_values(q, code, decls, site_file=None, tables=None): + """For each (file, line, name) const declaration: ({(file, line)} that hold a function, {(file, line, wrapper)} the + client callable its initializer calls, {(file, line, name)} the function it is an alias of). `tables` is + const_value_tables(q, site_file), when the caller already has it.""" + callable_, wrap, alias = set(), set(), set() + if not decls or code is None: + return callable_, wrap, alias + fns, rets, callee_at = tables if tables is not None else const_value_tables(q, site_file) def call_value(text, i, f, a, starts, depth=0): """(callable?, wrappers) for the expression at text[i:]""" diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact index 48079480f..be9e94efc 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact @@ -256,8 +256,17 @@ class Impact: triple = py or file.endswith(('.java', '.kt', '.scala', '.groovy')) # a Java text block is """ too js = file.endswith(('.js', '.jsx', '.mjs', '.cjs', '.ts', '.tsx')) prev = '' # the last code character seen + # only these characters can open a comment, a string or a regex literal: a run of any others is code, copied whole + special = self._CODE_SPECIAL_PY if py else self._CODE_SPECIAL while i < n: c = t[i] + if c not in '/"\'`#': + m = special.search(t, i); j = m.start() if m else n + out.append(t[i:j]) + k = j - 1 # `prev`: the run's last non-space, or its last newline + while k >= i and t[k].isspace() and t[k] != '\n': k -= 1 + if k >= i: prev = t[k] if not t[k].isspace() else '\n' + i = j; continue if js and c == '/' and i + 1 < n and t[i + 1] not in '/*' and prev in '(,=:[!&|?{};+-*%~^<>' + '\n': j = i + 1 # a regex literal: /…/flags, never a division while j < n and t[j] not in '/\n': @@ -284,8 +293,9 @@ class Impact: if not c.isspace(): prev = c elif c == '\n': prev = '\n' i += 1; continue - out.append(''.join('\n' if ch == '\n' else ' ' for ch in t[i:j])); prev = 'x'; i = j + out.append(self._BLANK.sub(' ', t[i:j])); prev = 'x'; i = j self.src[('code', file)] = ''.join(out).split('\n'); return self.src[('code', file)] + _CODE_SPECIAL = re.compile(r'[/"\'`]'); _CODE_SPECIAL_PY = re.compile(r'[/"\'`#]'); _BLANK = re.compile(r'[^\n]') # ── targets ────────────────────────────────────────────────────────────────────────────────────────────────── @@ -1835,13 +1845,18 @@ class Impact: # `Accessors = ...`): if the declaration's own source names a generator, it generates the same members gen_table = {d: set(ws) for d, ws in GENERATED.items()} used = {r[1].split('.')[-1] for r in g.q("SELECT owner_id, name FROM decorations")} if g.has('decorations') else set() - for d in used - set(gen_table): + # the first declaration of each such name with a file and a line, in the symbol table's order: found in one pass + # rather than one pass over every symbol per decorator name + wanted = used - set(gen_table); first_decl = {} + if wanted: for i, sy in g.sym.items(): - if sy['name'] != d or not sy.get('file') or not sy.get('line'): continue + if sy['name'] in wanted and sy['name'] not in first_decl and sy.get('file') and sy.get('line'): first_decl[sy['name']] = sy + for d in wanted: + sy = first_decl.get(d) + if sy is not None: body = '\n'.join(self.code(sy['file'])[sy['line'] - 1: (sy['end_line'] or sy['line']) + 1]) hit = {w for k, ws in GENERATED.items() if re.search(rf'\b{re.escape(k)}\b', body) for w in ws} if hit: gen_table[d] = hit - break # A BASE CLASS BUILT BY A FACTORY: `Base = declarative_base()`, then `class Customer(Base)`. The written # base name is `Base`, which discriminates nothing, and the alias rule cannot help because an assignment # is not a declaration with a body to read. What identifies it is the assignment itself, in the source the @@ -1855,6 +1870,7 @@ class Impact: pat = re.compile(rf'^\s*{re.escape(b)}\s*(?::[^=]+)?=\s*(\w+)\s*\(') for (fl,) in g.q("SELECT DISTINCT file FROM type_refs WHERE name = ? AND file IS NOT NULL", b): for ln in (self.code(fl) or []): + if b not in ln: continue # the pattern starts with the name itself m = pat.match(ln) if m and m.group(1) in GENERATED: gen_table[b] = set(GENERATED[m.group(1)]); break @@ -1869,9 +1885,10 @@ class Impact: W('cert_tier', sorted({(t, ax_edges.direct_cert(t), ax_edges.direct_why(t)) for t in {r[0] for r in g.q("SELECT DISTINCT tier FROM call_edges")} | {ax_edges.STUB_TIER, 'overload'}})) # a call written inside a mock's stub or verification keeps its row and loses its tier: "stub" (ax_edges.stub_sites) - stubs = ax_edges.stub_sites(lambda s, p: g.q(s, *p)) if g.has('call_sites') else set() + stubs = g.stub_sites() # a chained route link's edge sits on the line of its own verb and path, where `registration` labels it - link_line = ax_registration.route_site_lines(g.q, g.site_file) + # the registrations come out of the same walk over the route calls, so they are kept for `registration` below + regs, link_line = ax_registration.registrations(g.q, g.site_file, with_own_lines=True) # LEADS A LINK REJECTED (ax_links.py): a site's by-name match or set member someone read and found wrong is not walked; # the engine's row stays in graph.sqlite. `rejected` feeds the by-name rules, the set members leave `calls` here. import ax_links @@ -1970,7 +1987,7 @@ class Impact: L = self.code(r['file']); text = L[r['line'] - 1] if r['line'] <= len(L) else '' for qn in re.findall(rf'([A-Za-z_$][\w$]*)\s*\.\s*{re.escape(r["name"])}\b', text): quals.append((r['file'], r['line'], r['name'], qn)) W('ref', refs); W('qualifier', sorted(set(quals))) - regs = ax_registration.registrations(g.q, g.site_file) # computed once: reg_key_fact below reads it too + # `regs`, computed once above with link_line: reg_key_fact below reads it too W('registration', regs) trs = []; trf = set() # a reference on a module-level type alias's own lines belongs to the alias, not the module (#784) @@ -2331,7 +2348,19 @@ class Impact: # These are edges no call site carries — without them a change to a key or a bean has no visible consumer at all. cfg = []; keys = set(); beans = []; inj = []; factories = [] if g.has('ext_config_affects_method'): - for r in g.q("SELECT a.c0, a.c1, a.c2 FROM ext_config_affects_method a JOIN symbols s ON s.method_id = a.c1"): cfg.append((ckey(r[0]), r[1], r[2] or 'bound')) + # the rows whose method the index names, once per symbol row that names it: the join + # `ext_config_affects_method a JOIN symbols s ON s.method_id = a.c1`, in a's order, without probing the + # symbols index once per row (millions of rows over a few keys) + per_method = collections.Counter(m for (m,) in g.q("SELECT method_id FROM symbols WHERE method_id IS NOT NULL")) + ck = {} + for r in g.q("SELECT c0, c1, c2 FROM ext_config_affects_method"): + n = per_method.get(r[1], 0) + if not n: continue + k = ck.get(r[0]) + if k is None: k = ck[r[0]] = ckey(r[0]) + row = (k, r[1], r[2] or 'bound') + cfg.append(row) + for _ in range(n - 1): cfg.append(row) keys |= {ckey(r[0]) for r in g.q("SELECT DISTINCT c0 FROM ext_config_affects_method")} for t in ('ext_config_binding', 'ext_config_key_ref', 'ext_config_unresolved'): if g.has(t): keys |= {ckey(r[0]) for r in g.q(f"SELECT DISTINCT c0 FROM {t}")} diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path index 8a79bdd54..741164a7f 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path @@ -293,6 +293,13 @@ class G: self.con.executemany("INSERT INTO decorations VALUES (?,?,?,?,?)", out) self.decorations_synthesised = len(out) def q(self, s, *p): return self.con.execute(s, p).fetchall() + def stub_sites(self): + """ax_edges.stub_sites of this graph, read once per connection: both exports (path's and impact's) ask for it. + The set is shared — a caller must not change it.""" + if getattr(self, '_stubs_con', None) is not self.con: + self._stubs = ax_edges.stub_sites(lambda s, p: self.q(s, *p)) if self.has('call_sites') else set() + self._stubs_con = self.con + return self._stubs def has(self, t): return bool(self.q("SELECT 1 FROM sqlite_master WHERE name=?", t)) def loc(self, i): r = self.sym.get(i) or self.all_sym.get(i) @@ -1050,7 +1057,7 @@ class G: # …except a call written inside a mocking library's stub or verification (`when(repo.find(1))`, # `mock.Setup(r => r.Find(1))`): the receiver is a mock, so the real body never runs there, and walked as a call # it made every test that stubs a method a `sound` route to it. impact lists those sites as "stubs it". - stubs = ax_edges.stub_sites(lambda s, p: self.q(s, *p)) if self.has('call_sites') else set() + stubs = self.stub_sites() # leads an asserted-links rejection takes out (ax_links.suppressed): a set member every site of which was rejected import ax_links _rej, rej_edges = ax_links.suppressed(self.q, self.site_file)