Fix traversal cycles, both() filter, statement reuse; WAL default; unmask CI

.github/workflows/ci.yml

@@ -32,7 +32,7 @@ jobs:
       run: npm run build
 
     - name: Run tests with coverage
-      run: npm run test:coverage -- --maxWorkers=1 --passWithNoTests || true
+      run: npm run test:coverage -- --maxWorkers=1 --passWithNoTests
       env:
         NODE_OPTIONS: --max_old_space_size=6144
 

README.md

@@ -84,7 +84,7 @@ git clone https://github.com/michaeloboyle/sqlite-graph.git
 cd sqlite-graph
 npm install
 npm run build
-npm test  # 294 tests passing
+npm test  # 415 of 417 passing; 2 known failures: #17, #18
 ```
 
 When published (target: late December 2025):
@@ -234,6 +234,19 @@ All performance goals met ✅
 
 See [BENCHMARKS.md](docs/BENCHMARKS.md) for detailed performance analysis, methodology, and hardware specifications.
 
+### Scale benchmark (2026-11, 100k nodes / 500k edges)
+
+Run with `npm run build && node benchmarks/scale-benchmark.js`. Apple M2 Pro, Node 22, medians:
+
+| Operation | sqlite-graph | Note |
+|---|---|---|
+| getNode | 0.006ms | parity with raw better-sqlite3 |
+| `where({ email })`, no index | 22.4ms | full scan, grows linearly |
+| `where({ email })` after `createPropertyIndex('Person', 'email')` | 0.022ms | ~1000x; index any property you filter on |
+| `traverse().out().unique().maxDepth(3)` (~155 nodes) | 3.79ms | before #12 fixes; raw SQLite 1.10ms |
+| `shortestPath` between random nodes | ~2.5s | unidirectional BFS, see #16 |
+| single `createNode` autocommit, WAL (default since #13) | 4,405/s internal SSD, 21,277/s external drive | 1,491/s and 1,202/s with `{ wal: false }` (median of 5) |
+
 ## Contributing
 
 Contributions are welcome! We follow the [Contributor Covenant Code of Conduct](CODE_OF_CONDUCT.md).
@@ -268,8 +281,8 @@ MIT License - see [LICENSE](LICENSE) for details
 
 **Status:** 🟡 Alpha - Phase 1-2 Complete, Phase 3 Partial | Phase 4 (Browser Support) In Progress
 **Current Phase:** Building v1.0 with universal browser support (6-week timeline)
-**Test Status:** 294 tests passing | 2 suites with memory issues | ~75-80% coverage (estimated)
-**Not Production Ready:** Memory crashes in coverage tests, missing npm publish config
+**Test Status:** 415 of 417 passing (known failures: #17 nested transactions, #18 traverse() on a missing node) | ~75-80% coverage (estimated)
+**Not Production Ready:** shortestPath is slow on large graphs (#16), missing npm publish config
 
 **What Works:**
 - ✅ Complete CRUD operations with transactions

benchmarks/scale-benchmark.js

@@ -0,0 +1,157 @@
+// Scale benchmark: sqlite-graph vs raw better-sqlite3 doing the same work.
+// Usage: npm run build && node benchmarks/scale-benchmark.js
+// Env: SCALES=1000,10000,100000  OUT=<dir for temp DB files, default os tmpdir>
+const path = require('path');
+const fs = require('fs');
+const os = require('os');
+const { GraphDatabase } = require('../dist');
+const Database = require('better-sqlite3');
+
+const OUT = process.env.OUT || os.tmpdir();
+const SCALES = (process.env.SCALES || '1000,10000,100000').split(',').map(Number);
+const DEG = 5;
+
+// mulberry32: a plain LCG mod 2^31 produced correlated query pairs (all at distance 2)
+function mulberry32(a) {
+  return () => {
+    a |= 0; a = (a + 0x6d2b79f5) | 0;
+    let t = Math.imul(a ^ (a >>> 15), 1 | a);
+    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+const rnd = mulberry32(42);
+const ri = (n) => 1 + Math.floor(rnd() * n);
+
+function time(fn, iters) {
+  const t = [];
+  for (let i = 0; i < iters; i++) {
+    const s = process.hrtime.bigint();
+    fn(i);
+    t.push(Number(process.hrtime.bigint() - s) / 1e6);
+  }
+  t.sort((a, b) => a - b);
+  return { med: t[Math.floor(t.length / 2)], p95: t[Math.floor(t.length * 0.95)], n: iters };
+}
+const fmt = (r) => `${r.med.toFixed(3)}ms (p95 ${r.p95.toFixed(3)}, n=${r.n})`;
+const results = [];
+function rec(scale, op, impl, r, extra = {}) {
+  results.push({ scale, op, impl, ...r, ...extra });
+  console.log(`[${scale}] ${op.padEnd(34)} ${impl.padEnd(10)} ${fmt(r)} ${Object.keys(extra).length ? JSON.stringify(extra) : ''}`);
+}
+
+function buildGraph(file, N) {
+  if (fs.existsSync(file)) fs.unlinkSync(file);
+  const g = new GraphDatabase(file);
+  const raw = g.getRawDb();
+  const cities = ['Scranton', 'Boston', 'Austin', 'Denver', 'Miami', 'Seattle', 'Chicago', 'Portland'];
+  const insN = raw.prepare('INSERT INTO nodes (type, properties) VALUES (?, ?)');
+  const insE = raw.prepare('INSERT INTO edges (type, from_id, to_id) VALUES (?, ?, ?)');
+  raw.transaction(() => {
+    for (let i = 1; i <= N; i++) {
+      insN.run(i % 10 === 0 ? 'Company' : 'Person',
+        JSON.stringify({ name: `n${i}`, age: 18 + (i % 60), city: cities[i % cities.length], email: `u${i}@x.io` }));
+    }
+    for (let i = 1; i <= N; i++) for (let d = 0; d < DEG; d++) insE.run('KNOWS', i, ri(N));
+  })();
+  return g;
+}
+
+// --- baselines in raw SQL with cached statements ---
+function rawBfs(raw, start, maxDepth, hyd) {
+  const nb = raw.prepare("SELECT to_id AS id FROM edges WHERE from_id = ? AND type = 'KNOWS'").pluck();
+  const seen = new Set([start]); let frontier = [start]; const out = [];
+  for (let d = 0; d < maxDepth; d++) {
+    const next = [];
+    for (const id of frontier) for (const n of nb.all(id)) if (!seen.has(n)) { seen.add(n); next.push(n); out.push(hyd ? JSON.parse(hyd.get(n).properties) : n); }
+    frontier = next;
+  }
+  return out.length;
+}
+function cteBfs(raw, start, maxDepth) {
+  return raw.prepare(`WITH RECURSIVE r(id, d) AS (SELECT ?, 0 UNION SELECT e.to_id, r.d + 1 FROM edges e JOIN r ON e.from_id = r.id WHERE e.type='KNOWS' AND r.d < ?)
+    SELECT COUNT(DISTINCT id) - 1 AS c FROM r`).get(start, maxDepth).c;
+}
+function rawShortest(raw, a, b) {
+  const nb = raw.prepare('SELECT to_id AS id FROM edges WHERE from_id = ? UNION SELECT from_id FROM edges WHERE to_id = ?').pluck();
+  const parent = new Map([[a, 0]]); let q = [a], h = 0;
+  while (h < q.length) { const c = q[h++]; if (c === b) return true; for (const n of nb.all(c, c)) if (!parent.has(n)) { parent.set(n, c); q.push(n); } }
+  return false;
+}
+
+for (const N of SCALES) {
+  const file = path.join(OUT, `bench-${N}.db`);
+  const t0 = Date.now();
+  const g = buildGraph(file, N);
+  const raw = g.getRawDb();
+  console.log(`\n=== N=${N} nodes, ${N * DEG} edges, build ${Date.now() - t0}ms, journal=${raw.pragma('journal_mode', { simple: true })} sync=${raw.pragma('synchronous', { simple: true })}`);
+  const it = N >= 100000 ? 200 : 500;
+
+  // Point lookup
+  rec(N, 'getNode', 'lib', time(() => g.getNode(ri(N)), 2000));
+  const gn = raw.prepare('SELECT * FROM nodes WHERE id = ?');
+  rec(N, 'getNode', 'raw', time(() => { const r = gn.get(ri(N)); JSON.parse(r.properties); }, 2000));
+
+  // Property filter, unindexed then indexed
+  const city = () => ['Scranton', 'Boston', 'Austin'][ri(3) - 1];
+  const em = () => `u${ri(N)}@x.io`;
+  rec(N, 'where(email) unindexed', 'lib', time(() => g.nodes('Person').where({ email: em() }).exec(), 50));
+  rec(N, 'where(city).limit(20) unindexed', 'lib', time(() => g.nodes('Person').where({ city: city() }).limit(20).exec(), 200));
+  rec(N, 'where(city).count() unindexed', 'lib', time(() => g.nodes('Person').where({ city: city() }).count(), 50));
+  g.createPropertyIndex('Person', 'email');
+  const plan = raw.prepare(`EXPLAIN QUERY PLAN SELECT n.* FROM nodes n WHERE n.type = ? AND json_extract(n.properties, '$.email') = ?`).all('Person', 'x').map(r => r.detail).join(' | ');
+  rec(N, 'where(email) indexed', 'lib', time(() => g.nodes('Person').where({ email: em() }).exec(), 500), { plan });
+
+  // Traversal depth 2 and 3
+  for (const depth of [2, 3]) {
+    const its = depth === 3 && N >= 100000 ? 50 : (depth === 3 ? 100 : it);
+    let cnt = 0; const st = Array.from({ length: its }, () => ri(N));
+    rec(N, `traverse.out.unique d=${depth}`, 'lib', time((i) => { cnt = g.traverse(st[i]).out('KNOWS').unique().maxDepth(depth).toArray().length; }, its), { reached: cnt });
+    rec(N, `traverse.out.unique d=${depth}`, 'raw-js', time((i) => { cnt = rawBfs(raw, st[i], depth); }, its), { reached: cnt });
+    rec(N, `traverse.out.unique d=${depth}`, 'raw+hydr', time((i) => { cnt = rawBfs(raw, st[i], depth, gn); }, its), { reached: cnt });
+    if (N <= 10000) rec(N, `traverse.out.unique d=${depth}`, 'raw-cte', time((i) => { cnt = cteBfs(raw, st[i], depth); }, Math.min(its, 50)), { reached: cnt });
+  }
+  // Non-unique traversal (default): result inflation on a cyclic graph
+  {
+    let cnt = 0; const s = ri(N);
+    rec(N, 'traverse.out NON-unique d=3', 'lib', time(() => { cnt = g.traverse(s).out('KNOWS').maxDepth(3).toArray().length; }, 20), { returned: cnt, distinct: g.traverse(s).out('KNOWS').unique().maxDepth(3).toArray().length });
+  }
+
+  // Shortest path between random pairs
+  const pairs = Array.from({ length: 30 }, () => [ri(N), ri(N)]);
+  rec(N, 'shortestPath (any edge)', 'lib', time((i) => g.traverse(pairs[i][0]).shortestPath(pairs[i][1]), pairs.length));
+  rec(N, 'shortestPath (any edge)', 'raw-js', time((i) => rawShortest(raw, pairs[i][0], pairs[i][1]), pairs.length));
+
+  // Statement prepare overhead in isolation
+  rec(N, 'prepare+get per call', 'lib-style', time(() => raw.prepare('SELECT * FROM nodes WHERE id = ?').get(ri(N)), 5000));
+  rec(N, 'cached stmt get', 'raw', time(() => gn.get(ri(N)), 5000));
+
+  g.close();
+  for (const f of [file, file + '-wal', file + '-shm']) if (fs.existsSync(f)) fs.unlinkSync(f);
+}
+
+// --- Write path: journal mode x transaction ---
+{
+  const N = 2000;
+  for (const mode of ['no-wal', 'wal']) {
+    const file = path.join(OUT, `w-${mode}.db`);
+    for (const f of [file, file + '-wal', file + '-shm']) if (fs.existsSync(f)) fs.unlinkSync(f);
+    const g = new GraphDatabase(file, { wal: mode === 'wal' });
+    let s = Date.now();
+    for (let i = 0; i < N; i++) g.createNode('Person', { name: `p${i}` });
+    const auto = Date.now() - s;
+    s = Date.now();
+    g.transaction(() => { for (let i = 0; i < N; i++) g.createNode('Person', { name: `q${i}` }); });
+    const tx = Date.now() - s;
+    s = Date.now();
+    for (let i = 0; i < N; i++) g.createEdge(i + 1, 'KNOWS', N + i + 1);
+    const edges = Date.now() - s;
+    console.log(`[write ${mode}] createNode x${N} autocommit ${auto}ms (${(N / auto * 1000).toFixed(0)}/s) | in transaction ${tx}ms (${(N / tx * 1000).toFixed(0)}/s) | createEdge autocommit ${edges}ms`);
+    results.push({ scale: N, op: 'write', impl: mode, autocommit_ms: auto, tx_ms: tx, edge_autocommit_ms: edges });
+    g.close();
+    for (const f of [file, file + '-wal', file + '-shm']) if (fs.existsSync(f)) fs.unlinkSync(f);
+  }
+}
+
+fs.writeFileSync(path.join(OUT, 'sqlite-graph-scale-results.json'), JSON.stringify(results, null, 2));
+console.log('\nwrote', path.join(OUT, 'sqlite-graph-scale-results.json'));

docs/API.md

@@ -781,6 +781,10 @@ toArray(): Node[]
 
 **Returns:** Array of nodes reached during traversal
 
+**Depth and duplicates:**
+- Without `maxDepth()`, each defined step is followed once (`.out('X')` is one hop), extended to `minDepth()` when that is deeper.
+- Without `unique()`, a node is returned once per distinct path. A path never revisits one of its own nodes, so cycles terminate, but result size can grow exponentially with depth on dense graphs; use `unique()` for reachability.
+
 **Example:**
 
 ```typescript

src/core/Database.ts

@@ -63,6 +63,7 @@
   public readonly db: Database.Database;
   private schema?: GraphSchema;
   private preparedStatements: Map<string, Database.Statement>;
+  private transactionDepth = 0;
 
   /**
    * Creates a new graph database instance.
@@ -102,6 +103,13 @@
     this.schema = options?.schema;
     this.preparedStatements = new Map();
 
+    // WAL + synchronous=NORMAL for file databases unless opted out.
+    // Autocommit writes avoid a full fsync per statement (~22x in benchmarks).
+    if (options?.wal !== false && !options?.readonly && !this.db.memory) {
+      this.db.pragma('journal_mode = WAL');
+      this.db.pragma('synchronous = NORMAL');
+    }
+
     // Initialize database schema
     initializeSchema(this.db);
 
@@ -178,7 +186,7 @@
     validateNodeProperties(type, properties, this.schema);
 
     const stmt = this.preparedStatements.get('insertNode')!;
     const row = stmt.get(type, serialize(properties)) as any;
 
     return {
       id: row.id,
@@ -209,7 +217,7 @@
     validateNodeId(id);
 
     const stmt = this.preparedStatements.get('getNode')!;
     const row = stmt.get(id) as any;
 
     if (!row) return null;
 
@@ -250,7 +258,7 @@
 
     const merged = { ...existing.properties, ...properties };
     const stmt = this.preparedStatements.get('updateNode')!;
     const row = stmt.get(serialize(merged), id) as any;
 
     return {
       id: row.id,
@@ -333,7 +341,7 @@
       from,
       to,
       properties ? serialize(properties) : null
     ) as any;
 
     return {
       id: row.id,
@@ -363,7 +371,7 @@
     validateNodeId(id);
 
     const stmt = this.preparedStatements.get('getEdge')!;
     const row = stmt.get(id) as any;
 
     if (!row) return null;
 
@@ -507,10 +515,12 @@
    * ```
    */
   transaction<T>(fn: (ctx: TransactionContext) => T): T {
-    // Start transaction
-    this.db.prepare('BEGIN').run();
+    // Inside an open transaction, nest with a savepoint instead of BEGIN (#17)
+    const nestedName = this.db.inTransaction ? `sg_tx_${this.transactionDepth}` : undefined;
+    this.db.prepare(nestedName ? `SAVEPOINT "${nestedName}"` : 'BEGIN').run();
+    this.transactionDepth++;
 
-    const ctx = new TransactionContext(this.db);
+    const ctx = new TransactionContext(this.db, nestedName);
 
     try {
       const result = fn(ctx);
@@ -527,6 +537,8 @@
         ctx.rollback();
       }
       throw error;
+    } finally {
+      this.transactionDepth--;
     }
   }
 
@@ -545,7 +557,7 @@
     const nodesStmt = this.db.prepare('SELECT * FROM nodes ORDER BY id');
     const edgesStmt = this.db.prepare('SELECT * FROM edges ORDER BY id');
 
     const nodes = nodesStmt.all().map((row: any) => ({
       id: row.id,
       type: row.type,
       properties: deserialize(row.properties),
@@ -553,7 +565,7 @@
       updatedAt: timestampToDate(row.updated_at)
     }));
 
     const edges = edgesStmt.all().map((row: any) => ({
       id: row.id,
       type: row.type,
       from: row.from_id,
@@ -687,10 +699,10 @@
         SELECT * FROM nodes
         WHERE type = ? AND ${whereConditions.join(' AND ')}
       `;
       const matchValues = matchKeys.map((key) => (matchProperties as any)[key]);
 
       const stmt = this.db.prepare(sql);
       const rows = stmt.all(type, ...matchValues) as any[];
 
       if (rows.length > 1) {
         const nodes = rows.map((row) => ({
@@ -713,7 +725,7 @@
         validateNodeProperties(type, mergedProps as T, this.schema);
 
         const updateStmt = this.preparedStatements.get('updateNode')!;
         const updatedRow = updateStmt.get(serialize(mergedProps), existing.id) as any;
 
         return {
           node: {

src/core/Transaction.ts

@@ -40,8 +40,15 @@ export class TransactionContext {
    * @param db - SQLite database instance
    * @internal
    */
-  constructor(db: Database.Database) {
+  private nestedName?: string;
+
+  /**
+   * @param nestedName - set when this context is a savepoint inside an outer
+   *   transaction; commit/rollback then release or roll back that savepoint only
+   */
+  constructor(db: Database.Database, nestedName?: string) {
     this.db = db;
+    this.nestedName = nestedName;
   }
 
   /**
@@ -62,7 +69,7 @@ export class TransactionContext {
     if (this.finalized) {
       throw new TransactionAlreadyFinalizedError();
     }
-    this.db.prepare('COMMIT').run();
+    this.db.prepare(this.nestedName ? `RELEASE "${this.nestedName}"` : 'COMMIT').run();
     this.finalized = true;
   }
 
@@ -88,7 +95,13 @@ export class TransactionContext {
     if (this.finalized) {
       throw new TransactionAlreadyFinalizedError();
     }
-    this.db.prepare('ROLLBACK').run();
+    if (this.nestedName) {
+      // Undo the inner work, then close the savepoint; the outer transaction continues
+      this.db.prepare(`ROLLBACK TO "${this.nestedName}"`).run();
+      this.db.prepare(`RELEASE "${this.nestedName}"`).run();
+    } else {
+      this.db.prepare('ROLLBACK').run();
+    }
     this.finalized = true;
   }