From e4c45dfa1b0bdff52504b60e57e92c03c5841e50 Mon Sep 17 00:00:00 2001 From: githubscum <65476409+githubscum@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:01:36 -0500 Subject: [PATCH] CORE (signing sitting): chain checkpoints and inclusion proofs WO-CARTRIDGE-SEAL part B, queued as a patch. Nothing under src/ changes on this branch. - proposals/chain-checkpoints.patch, generated by proposals/make-chain-checkpoints-patch.mjs (git diff --no-index over temp copies; never typed). Adds src/chain/merkle.js (RFC 9162 hashing, inclusion and consistency proofs), src/chain/checkpoint.js (C2SP signed-note and tlog-checkpoint, pure verifiers), src/store/checkpoints.js (sidecar receipts/checkpoints.jsonl, segment sealing every 1024 entries, incremental and full sealed verify, proofs), verifyRange in src/chain/index.js (verifyChain becomes verifyRange from genesis, results unchanged), and seal/verifyIncremental/verifySealed/proveInclusion/proveConsistency on the store. chain.jsonl is never rewritten. - proposals/chain-checkpoints/: the new files' sources, the N measurement script. proposals/chain-checkpoints.md: design, preflight, measurement, sitting steps, draft KNOWN-LIMITS entries. - test/chain-checkpoints.test.js: rehearsal. Live cases are TODO and fail until the patch lands; the queued-patch block applies it in a temp repo and runs every case. Co-Authored-By: Claude Opus 5.5 --- proposals/chain-checkpoints.md | 123 ++ proposals/chain-checkpoints.patch | 1236 +++++++++++++++++ proposals/chain-checkpoints/checkpoint.js | 285 ++++ proposals/chain-checkpoints/measure-n.mjs | 45 + proposals/chain-checkpoints/merkle.js | 263 ++++ .../chain-checkpoints/store-checkpoints.js | 487 +++++++ proposals/make-chain-checkpoints-patch.mjs | 303 ++++ test/chain-checkpoints.test.js | 564 ++++++++ 8 files changed, 3306 insertions(+) create mode 100644 proposals/chain-checkpoints.md create mode 100644 proposals/chain-checkpoints.patch create mode 100644 proposals/chain-checkpoints/checkpoint.js create mode 100644 proposals/chain-checkpoints/measure-n.mjs create mode 100644 proposals/chain-checkpoints/merkle.js create mode 100644 proposals/chain-checkpoints/store-checkpoints.js create mode 100644 proposals/make-chain-checkpoints-patch.mjs create mode 100644 test/chain-checkpoints.test.js diff --git a/proposals/chain-checkpoints.md b/proposals/chain-checkpoints.md new file mode 100644 index 0000000..b0c2ac5 --- /dev/null +++ b/proposals/chain-checkpoints.md @@ -0,0 +1,123 @@ +# Chain checkpoints and inclusion proofs (WO-CARTRIDGE-SEAL part B) + +Status: CORE, queued for the signing sitting. Patch: `proposals/chain-checkpoints.patch`, +generated by `node proposals/make-chain-checkpoints-patch.mjs` (never typed, KNOWN-LIMITS 80). +Sources of the three new core files live in `proposals/chain-checkpoints/` until the sitting. +Rehearsal: `test/chain-checkpoints.test.js`. + +## What it adds + +Today a cartridge carries the chain head, and checking it needs the whole chain. `verify`, +`loadChain` and queries read from genesis every time. This patch adds: + +- **Segments.** Every 1024 entries (measured below) the store can seal a segment: a signed + checkpoint over the RFC 9162 Merkle tree of every entry so far. `chain.jsonl` is never + rewritten. Checkpoints live in a sidecar, `receipts/checkpoints.jsonl`. +- **Signed checkpoints in the published format.** C2SP `tlog-checkpoint` notes (origin, tree + size, base64 root), signed as C2SP `signed-note` Ed25519 (type 0x01) with the chain key. Key + name = origin = `lotor.local/chain/`. No extension lines. +- **Linking.** Each checkpoint is linked to the previous one the standard way: the newer tree + extends the older one, and `proveConsistency()` produces the RFC 9162 consistency proof a + stranger checks with `verifyConsistencyProof()`. +- **Incremental verify.** `store.verifyIncremental()` starts from the newest checkpoint, or from + one the caller already trusts (`{ trusted: }`), and reads only the entries after it. + It also checks every later checkpoint against the tree it recomputes, so a fork after a kept + checkpoint shows. `store.verify()` (full chain) is unchanged; `store.verifySealed()` is the full + read of the chain plus every checkpoint. +- **Inclusion proofs.** `store.proveInclusion(seq) -> { leaf, index, size, path, checkpoint }` + and the pure `verifyInclusionProof(proof, publicKey)` in `src/chain/checkpoint.js`, which needs + only the proof and the chain's public key. +- **Head checkpoint on demand.** `store.seal({ atHead: true })` also seals the partial segment at + the head, so a receipt can be proven right away (a prediction signed before its outcome needs + this). The next segment seal still ends on a multiple of 1024. +- **verifyRange.** `src/chain/index.js` gains `verifyRange(entries, key, { startSeq, prevHash })`; + `verifyChain` becomes `verifyRange` from genesis. Same checks, same order, same reason strings; + the rehearsal asserts deep equality of results against today's live `verifyChain` on seven + chains (clean, payload tamper, sig flip, broken link, seq skip, genesis offset, empty). + +Leaves are the existing entry hashes (`SHA-256(0x00 || 32-byte entry hash)`). The entry hash +already commits to seq, timestamp, nonce, prevHash and payload, so a verifier needs no JSON +canonicalisation and a cartridge can cite a receipt by hash. + +## Preflight (existing solutions) + +- **Formats:** C2SP tlog-checkpoint and signed-note (read 2026-10-26, editor's copies dated + 2026-10-07), RFC 9162 section 2.1 hashing, inclusion (2.1.3) and consistency (2.1.4) proofs. + Nothing invented. The C2SP signature line starts with U+2014, which this repo bans from its + text; the code builds it with `String.fromCharCode(0x2014)`, and the rehearsal asserts the + patch carries no U+2014 byte. +- **tlog-tiles / Trillian Tessera:** tiles is an HTTP layout for serving a log's tree to the + public, and Tessera is the Go library that writes it. Lotor has no server and its log is a + local file, so neither is adopted here. The tree and checkpoint formats are the ones tiles + serves, so a tiles export can be added later without changing anything signed. +- **npm libraries:** none adopted. Candidates found on 2026-10-26: `ctjs` (last published 2018), + and several 2026 release-candidate packages with no track record. The algorithms are about 150 + lines of RFC text, and this code lands in protected core, where a dependency is supply-chain + surface the gate cannot see. Correctness is pinned instead: the RFC 6962 reference roots for + sizes 1..8, the C2SP signed-note spec example (vkey `example.com/foo+530d903a+...`), and an + exhaustive round-trip of every inclusion and consistency proof up to tree size 40. + +## Choosing N (measured 2026-10-26, node 24.18, Windows 11, this machine) + +`node proposals/chain-checkpoints/measure-n.mjs` over a 20,000-entry chain with ~640-byte +lines, two runs (a third run while writing this: 33.4 / 80.0 / 2.9 / 0.75 us): + +| cost per entry | run 1 | run 2 | +|---|---|---| +| append (sign) | 33.8 us | 34.0 us | +| verifyChain (hash + Ed25519 verify + links) | 81.2 us | 78.1 us | +| Merkle leaf + tree hashing | 2.9 us | 2.8 us | +| JSON.parse of a line | 0.75 us | 0.73 us | + +Verification dominates by ~30x, so N sets the worst-case work an incremental verify does past the +newest checkpoint: N - 1 entries at ~80 us. A budget of about 100 ms gives N <= 1250. N = 1024 +(power of two, so a segment ending on a multiple of N is a perfect subtree of the RFC tree) puts +the worst case at ~82 ms. A full verify of 40,000 entries is ~3.2 s by comparison. Sealing one +segment holds the chain lock for about that 82 ms; the lock is taken per segment, never across a +migration, because the lock goes stale after 30 s and appends give up after 5 s (KNOWN-LIMITS 7). + +## Measured: inclusion proofs in a cartridge (NOT built here) + +The non-core follow-on (the cartridge carries proofs for the receipts it cites) is not in this +patch. Measured on the 40,000-entry rehearsal chain: path 16 hashes; the signed note is 219 B. +A compact binary encoding (index u32, size u32, path, root 32, signature 64; the origin and key +are derivable from the pubkey the manifest already carries) is 616 B for the first proof and +516 B for each further proof against the same checkpoint. Against the 2,808 B useful PAP budget +that is at most 5 proofs with an empty spine, about 3 next to a 1 KB compressed spine, before +sealed-mode overhead. The path grows by 32 B each time the chain doubles (20 hashes at ~1M +entries). Proofs for nearby receipts share upper nodes; deduplicating them is not done. + +## Sitting instructions + +1. `git apply --check proposals/chain-checkpoints.patch` (clean on `337358f`). +2. Apply with every target named on the command line (KNOWN-LIMITS 82), for example one + signature for one command: + `git apply --include=src/chain/index.js --include=src/chain/merkle.js --include=src/chain/checkpoint.js --include=src/store/index.js --include=src/store/checkpoints.js proposals/chain-checkpoints.patch` +3. `npm run limits-pin -- --stamp`, then `npm test`. In the rehearsal the patched tree ran the + full suite with the cases un-todo'd; see the PR body for the count. +4. Append the KNOWN-LIMITS entries below with the next free numbers, then re-stamp. + +## Draft KNOWN-LIMITS entries (numbered at landing) + +**Checkpoints are signed by the chain key, so they bind a history only for someone who kept +one.** The key that signs entries also signs checkpoints, and it is plaintext at rest (limits 8 +and 48). Its holder can rewrite the chain, re-seal, and `verifySealed()` passes on the result. +What catches the rewrite is an earlier checkpoint kept elsewhere: `verifyIncremental({ trusted })` +reports a fork, and no consistency proof from the kept checkpoint exists. A cosigner or witness +(C2SP tlog-cosignature) or an external anchor would remove the need to keep one; neither is in +this patch, and OpenTimestamps anchoring is out of scope until the notary is back. + +**Incremental verify does not read sealed segments.** From the newest checkpoint, an edit inside +an older sealed segment is out of sight until a full read (`verifySealed()` or `store.verify()`). +That is the trade that makes it incremental, and the rehearsal pins it as documented behavior. + +**Nothing seals automatically.** No hook or CLI calls `store.seal()` in this patch. Until one does +(session-end hook, which is core, or `pap-export`, which is not), a chain has checkpoints only when +something asks for them. Sealing inside the append path was rejected: it would add work and a new +failure mode to the unattended hook path. + +Not limits, recorded so nobody rediscovers them: sealing refuses a chain line that does not parse, +where `loadChain` silently drops it, so a chain with an inserted garbage line can pass +`store.verify()` and still refuse to seal. Re-sealing from scratch yields byte-identical notes +(Ed25519 is deterministic and the note text is fixed by size and root), so losing the sidecar +loses nothing a stranger kept. diff --git a/proposals/chain-checkpoints.patch b/proposals/chain-checkpoints.patch new file mode 100644 index 0000000..42bbd7f --- /dev/null +++ b/proposals/chain-checkpoints.patch @@ -0,0 +1,1236 @@ +diff --git a/src/chain/checkpoint.js b/src/chain/checkpoint.js +new file mode 100644 +--- /dev/null ++++ b/src/chain/checkpoint.js +@@ -0,0 +1,285 @@ ++import crypto from 'node:crypto'; ++import { leafHash, verifyInclusion, verifyConsistency } from './merkle.js'; ++ ++/** ++ * src/chain/checkpoint.js ++ * ++ * Signed checkpoints for the receipt chain, in the published formats, not a ++ * new one: ++ * ++ * - C2SP signed-note (c2sp.org/signed-note): note text, a blank line, then ++ * signature lines " base64(keyID4 || signature)". ++ * Ed25519, signature type 0x01, key ID = SHA-256(name || 0x0A || 0x01 || ++ * pubkey32)[:4]. ++ * - C2SP tlog-checkpoint (c2sp.org/tlog-checkpoint): note text is exactly ++ * three lines, origin, tree size, base64 RFC 6962 root. No extension ++ * lines are written (the spec does not recommend them). ++ * ++ * The signature-line marker is U+2014, which this repository otherwise bans ++ * from its text. It is built with String.fromCharCode below so that the ban ++ * stays checkable by a byte grep and the spec stays exact. ++ * ++ * Leaves are the chain's own entry hashes (32 raw bytes each, the SHA-256 of ++ * the canonical entry). The entry hash already commits to seq, timestamp, ++ * nonce, prevHash and payload, so a verifier needs no JSON canonicalisation ++ * to check a proof, and a cartridge can cite a receipt by its hash alone. ++ * ++ * Everything here is pure: no file I/O, no clock. A stranger needs only a ++ * proof object and the chain's raw 32-byte Ed25519 public key. ++ */ ++ ++const SIGNATURE_MARK = String.fromCharCode(0x2014); ++const NL = String.fromCharCode(10); ++const SIG_TYPE_ED25519 = 0x01; ++const ORIGIN_PREFIX = 'lotor.local/chain/'; ++ ++/** SPKI DER prefix for a raw Ed25519 public key (RFC 8410). */ ++const ED25519_SPKI_PREFIX = Buffer.from('302a300506032b6570032100', 'hex'); ++ ++function rawPublicKey(key) { ++ if (Buffer.isBuffer(key) && key.length === 32) return key; ++ const obj = crypto.createPublicKey(key); ++ return obj.export({ format: 'der', type: 'spki' }).subarray(-32); ++} ++ ++function publicKeyObject(raw32) { ++ return crypto.createPublicKey({ key: Buffer.concat([ED25519_SPKI_PREFIX, raw32]), format: 'der', type: 'spki' }); ++} ++ ++/** The log origin (and key name) for a chain key: fixed, derived, offline. */ ++function originFor(publicKey) { ++ const raw = rawPublicKey(publicKey); ++ return ORIGIN_PREFIX + crypto.createHash('sha256').update(raw).digest('hex').slice(0, 16); ++} ++ ++function keyId(name, raw32) { ++ return crypto.createHash('sha256') ++ .update(Buffer.from(name, 'utf8')) ++ .update(Buffer.from([0x0a, SIG_TYPE_ED25519])) ++ .update(raw32) ++ .digest() ++ .subarray(0, 4); ++} ++ ++/** C2SP verifier key string: name+hex(keyID)+base64(0x01 || pubkey). */ ++function vkey(name, publicKey) { ++ const raw = rawPublicKey(publicKey); ++ return `${name}+${keyId(name, raw).toString('hex')}+${Buffer.concat([Buffer.from([SIG_TYPE_ED25519]), raw]).toString('base64')}`; ++} ++ ++/** Parse a C2SP vkey into { name, keyId, publicKeyRaw }. Throws on anything else. */ ++function parseVkey(s) { ++ const parts = String(s).split('+'); ++ if (parts.length !== 3) throw new Error('vkey must have three +-separated parts'); ++ const [name, idHex, b64] = parts; ++ const material = decodeCanonicalBase64(b64); ++ if (material === null || material.length !== 33 || material[0] !== SIG_TYPE_ED25519) { ++ throw new Error('vkey is not an Ed25519 (type 0x01) key'); ++ } ++ const raw = material.subarray(1); ++ if (keyId(name, raw).toString('hex') !== idHex) throw new Error('vkey key ID does not match its name and key'); ++ return { name, keyId: Buffer.from(idHex, 'hex'), publicKeyRaw: Buffer.from(raw) }; ++} ++ ++function decodeCanonicalBase64(s) { ++ if (typeof s !== 'string' || s.length === 0 || s.length % 4 !== 0) return null; ++ if (!/^[A-Za-z0-9+/]*={0,2}$/.test(s)) return null; ++ const buf = Buffer.from(s, 'base64'); ++ return buf.toString('base64') === s ? buf : null; ++} ++ ++function isValidKeyName(name) { ++ return typeof name === 'string' && name.length > 0 && !/[\s+]/u.test(name); ++} ++ ++/** ++ * Sign note text (which must end in a newline) as a C2SP signed note. ++ * @param {string} text ++ * @param {string} name - key name ++ * @param {crypto.KeyObject|string} privateKey - Ed25519 private key ++ * @param {crypto.KeyObject|string|Buffer} publicKey - matching public key ++ */ ++function signNote(text, name, privateKey, publicKey) { ++ if (typeof text !== 'string' || !text.endsWith(NL)) throw new Error('note text must end in a newline'); ++ if (!isValidKeyName(name)) throw new Error(`invalid key name: ${name}`); ++ const raw = rawPublicKey(publicKey); ++ const sig = crypto.sign(null, Buffer.from(text, 'utf8'), privateKey); ++ const blob = Buffer.concat([keyId(name, raw), sig]).toString('base64'); ++ return `${text}${NL}${SIGNATURE_MARK} ${name} ${blob}${NL}`; ++} ++ ++/** ++ * Split a signed note into its text and signature lines. Returns null for ++ * anything that is not a well-formed note. ++ */ ++function parseNote(note) { ++ if (typeof note !== 'string' || !note.endsWith(NL)) return null; ++ for (let i = 0; i < note.length; i++) { ++ const c = note.charCodeAt(i); ++ if (c < 0x20 && c !== 0x0a) return null; ++ } ++ const sep = note.lastIndexOf(NL + NL); ++ if (sep < 0) return null; ++ const text = note.slice(0, sep + 1); ++ const sigLines = note.slice(sep + 2).split(NL); ++ sigLines.pop(); // the note ends in a newline, so the last split part is '' ++ if (sigLines.length === 0) return null; ++ const signatures = []; ++ for (const line of sigLines) { ++ const prefix = SIGNATURE_MARK + ' '; ++ if (!line.startsWith(prefix)) return null; ++ const rest = line.slice(prefix.length); ++ const space = rest.lastIndexOf(' '); ++ if (space <= 0) return null; ++ const name = rest.slice(0, space); ++ const blob = decodeCanonicalBase64(rest.slice(space + 1)); ++ if (!isValidKeyName(name) || blob === null || blob.length < 5) return null; ++ signatures.push({ name, keyId: blob.subarray(0, 4), signature: blob.subarray(4) }); ++ } ++ return { text, signatures }; ++} ++ ++/** ++ * Verify a signed note against one known Ed25519 key. Per the spec, signatures ++ * from unknown keys are ignored, a known key whose signature fails rejects the ++ * note, and a note with no verified known signature is rejected. ++ * @returns {{ ok: true, text: string } | { ok: false, reason: string }} ++ */ ++function verifyNote(note, name, publicKey) { ++ const parsed = parseNote(note); ++ if (parsed === null) return { ok: false, reason: 'not a well-formed signed note' }; ++ let raw; ++ try { ++ raw = rawPublicKey(publicKey); ++ } catch (e) { ++ return { ok: false, reason: 'unusable public key' }; ++ } ++ const id = keyId(name, raw); ++ const keyObj = publicKeyObject(raw); ++ let verified = false; ++ for (const s of parsed.signatures) { ++ if (s.name !== name || !s.keyId.equals(id)) continue; ++ const ok = s.signature.length === 64 && ++ crypto.verify(null, Buffer.from(parsed.text, 'utf8'), keyObj, s.signature); ++ if (!ok) return { ok: false, reason: `signature by ${name} does not verify` }; ++ verified = true; ++ } ++ if (!verified) return { ok: false, reason: `no signature by ${name} (key ${id.toString('hex')})` }; ++ return { ok: true, text: parsed.text }; ++} ++ ++/** Checkpoint note text (unsigned) for a tree. */ ++function checkpointText(origin, size, root) { ++ if (!Number.isSafeInteger(size) || size < 0) throw new Error(`tree size must be a non-negative safe integer, got ${size}`); ++ if (!Buffer.isBuffer(root) || root.length !== 32) throw new Error('root must be 32 bytes'); ++ return `${origin}${NL}${size}${NL}${root.toString('base64')}${NL}`; ++} ++ ++/** Parse verified checkpoint text into { origin, size, root }. Null if malformed. */ ++function parseCheckpointText(text) { ++ const lines = text.split(NL); ++ lines.pop(); ++ if (lines.length < 3) return null; ++ const [origin, sizeStr, rootB64, ...extensions] = lines; ++ if (!origin || !/^(0|[1-9][0-9]*)$/.test(sizeStr)) return null; ++ const size = Number(sizeStr); ++ if (!Number.isSafeInteger(size)) return null; ++ const root = decodeCanonicalBase64(rootB64); ++ if (root === null || root.length !== 32) return null; ++ if (extensions.some((l) => l.length === 0)) return null; ++ return { origin, size, root, extensions }; ++} ++ ++/** Sign a checkpoint for a chain key. Key name = origin, per the spec. */ ++function signCheckpoint({ size, root }, privateKey, publicKey) { ++ const origin = originFor(publicKey); ++ return signNote(checkpointText(origin, size, root), origin, privateKey, publicKey); ++} ++ ++/** ++ * Verify a checkpoint against a chain public key. ++ * @returns {{ ok: true, origin, size, root } | { ok: false, reason }} ++ */ ++function verifyCheckpoint(note, publicKey) { ++ let origin; ++ try { ++ origin = originFor(publicKey); ++ } catch (e) { ++ return { ok: false, reason: 'unusable public key' }; ++ } ++ const v = verifyNote(note, origin, publicKey); ++ if (!v.ok) return v; ++ const cp = parseCheckpointText(v.text); ++ if (cp === null) return { ok: false, reason: 'checkpoint text is malformed' }; ++ if (cp.origin !== origin) return { ok: false, reason: `checkpoint origin ${cp.origin} is not this chain (${origin})` }; ++ if (cp.extensions.length !== 0) return { ok: false, reason: 'checkpoint carries extension lines; Lotor writes none' }; ++ return { ok: true, origin: cp.origin, size: cp.size, root: cp.root }; ++} ++ ++const HEX64 = /^[0-9a-f]{64}$/; ++ ++function hexList(list) { ++ if (!Array.isArray(list)) return null; ++ const out = []; ++ for (const h of list) { ++ if (typeof h !== 'string' || !HEX64.test(h)) return null; ++ out.push(Buffer.from(h, 'hex')); ++ } ++ return out; ++} ++ ++/** ++ * Verify an inclusion proof as produced by the store's proveInclusion(): ++ * { leaf: , index: , size, path: [hex...], checkpoint: } ++ * Needs only the proof and the chain's public key. ++ * @returns {{ ok: true, index, size, root } | { ok: false, reason }} ++ */ ++function verifyInclusionProof(proof, publicKey) { ++ if (proof === null || typeof proof !== 'object') return { ok: false, reason: 'proof is not an object' }; ++ const cp = verifyCheckpoint(proof.checkpoint, publicKey); ++ if (!cp.ok) return { ok: false, reason: `checkpoint: ${cp.reason}` }; ++ if (proof.size !== cp.size) return { ok: false, reason: `proof is for tree size ${proof.size}, checkpoint is size ${cp.size}` }; ++ if (typeof proof.leaf !== 'string' || !HEX64.test(proof.leaf)) return { ok: false, reason: 'leaf must be a 64-hex entry hash' }; ++ const path = hexList(proof.path); ++ if (path === null) return { ok: false, reason: 'path must be a list of 64-hex hashes' }; ++ const ok = verifyInclusion(proof.index, cp.size, leafHash(Buffer.from(proof.leaf, 'hex')), path, cp.root); ++ if (!ok) return { ok: false, reason: `entry ${proof.leaf} is not at index ${proof.index} of the tree this checkpoint signs` }; ++ return { ok: true, index: proof.index, size: cp.size, root: cp.root }; ++} ++ ++/** ++ * Verify that a newer checkpoint extends an older one: ++ * { from: , to: , path: [hex...] } ++ * @returns {{ ok: true, from, to } | { ok: false, reason }} ++ */ ++function verifyConsistencyProof(proof, publicKey) { ++ if (proof === null || typeof proof !== 'object') return { ok: false, reason: 'proof is not an object' }; ++ const a = verifyCheckpoint(proof.from, publicKey); ++ if (!a.ok) return { ok: false, reason: `older checkpoint: ${a.reason}` }; ++ const b = verifyCheckpoint(proof.to, publicKey); ++ if (!b.ok) return { ok: false, reason: `newer checkpoint: ${b.reason}` }; ++ const path = hexList(proof.path); ++ if (path === null) return { ok: false, reason: 'path must be a list of 64-hex hashes' }; ++ if (!verifyConsistency(a.size, b.size, a.root, b.root, path)) { ++ return { ok: false, reason: `the size-${b.size} tree does not extend the size-${a.size} tree` }; ++ } ++ return { ok: true, from: a.size, to: b.size }; ++} ++ ++export { ++ SIGNATURE_MARK, ++ originFor, ++ keyId, ++ vkey, ++ parseVkey, ++ signNote, ++ parseNote, ++ verifyNote, ++ checkpointText, ++ parseCheckpointText, ++ signCheckpoint, ++ verifyCheckpoint, ++ verifyInclusionProof, ++ verifyConsistencyProof, ++ rawPublicKey ++}; +diff --git a/src/chain/index.js b/src/chain/index.js +--- a/src/chain/index.js ++++ b/src/chain/index.js +@@ -100,16 +100,44 @@ function verifyChain(entries, publicKey) { + return { ok: true }; + } + ++ return verifyRange(entries, publicKey, { startSeq: 0, prevHash: GENESIS_PREV_HASH }); ++} ++ ++/** ++ * Verify a contiguous range of entries: the first carries seq startSeq and ++ * links to prevHash (the hash of entry startSeq - 1, or the genesis constant ++ * when startSeq is 0). Same checks, same order and same reasons as a full ++ * verifyChain; entry numbers in brokenAt and in the reasons are absolute seqs, ++ * so from startSeq 0 the result is exactly what verifyChain returned before ++ * this function existed. A caller that already trusts the chain up to ++ * startSeq (a verified checkpoint, src/store/checkpoints.js) verifies only ++ * what came after it. ++ * @param {Array} entries - Array of entry objects, in seq order ++ * @param {crypto.KeyObject} publicKey - Ed25519 public key ++ * @param {Object} [from] - { startSeq = 0, prevHash = GENESIS_PREV_HASH } ++ * @returns {Object} { ok: boolean, brokenAt?: number, reason?: string } ++ */ ++function verifyRange(entries, publicKey, from = {}) { ++ const startSeq = from.startSeq === undefined ? 0 : from.startSeq; ++ const startPrevHash = from.prevHash === undefined ? GENESIS_PREV_HASH : from.prevHash; ++ if (!Number.isSafeInteger(startSeq) || startSeq < 0) { ++ throw new Error(`verifyRange: startSeq must be a non-negative safe integer, got ${startSeq}`); ++ } ++ if (!entries || entries.length === 0) { ++ return { ok: true }; ++ } ++ + for (let i = 0; i < entries.length; i++) { + const entry = entries[i]; ++ const at = startSeq + i; + + // Verify hash recomputation + const expectedHash = computeHash(entry); + if (expectedHash !== entry.hash) { + return { + ok: false, +- brokenAt: i, +- reason: `hash mismatch at entry ${i}: expected ${expectedHash}, got ${entry.hash}` ++ brokenAt: at, ++ reason: `hash mismatch at entry ${at}: expected ${expectedHash}, got ${entry.hash}` + }; + } + +@@ -121,39 +149,40 @@ function verifyChain(entries, publicKey) { + if (!sigOk) { + return { + ok: false, +- brokenAt: i, +- reason: `signature verification failed at entry ${i}` ++ brokenAt: at, ++ reason: `signature verification failed at entry ${at}` + }; + } + +- // Verify seq continuity. verifyChain already requires a full chain from +- // genesis, so entry i must carry seq i. The hash covers seq, so a wrong seq +- // can only be signed in at write time; this is where it is caught. +- if (!Number.isSafeInteger(entry.seq) || entry.seq !== i) { ++ // Verify seq continuity. Entry i of a range that starts at startSeq must ++ // carry seq startSeq + i (seq i for a full chain from genesis). The hash ++ // covers seq, so a wrong seq can only be signed in at write time; this is ++ // where it is caught. ++ if (!Number.isSafeInteger(entry.seq) || entry.seq !== at) { + return { + ok: false, +- brokenAt: i, +- reason: `seq discontinuity at entry ${i}: expected ${i}, got ${JSON.stringify(entry.seq)}` ++ brokenAt: at, ++ reason: `seq discontinuity at entry ${at}: expected ${at}, got ${JSON.stringify(entry.seq)}` + }; + } + + // Verify prevHash linkage +- if (i === 0) { ++ if (at === 0) { + // Genesis entry: prevHash must be genesis constant + if (entry.prevHash !== GENESIS_PREV_HASH) { + return { + ok: false, +- brokenAt: i, +- reason: `genesis entry prevHash mismatch at entry ${i}: expected ${GENESIS_PREV_HASH}, got ${entry.prevHash}` ++ brokenAt: at, ++ reason: `genesis entry prevHash mismatch at entry ${at}: expected ${GENESIS_PREV_HASH}, got ${entry.prevHash}` + }; + } + } else { +- const prevEntry = entries[i - 1]; +- if (entry.prevHash !== prevEntry.hash) { ++ const expectedPrev = i > 0 ? entries[i - 1].hash : startPrevHash; ++ if (entry.prevHash !== expectedPrev) { + return { + ok: false, +- brokenAt: i, +- reason: `prevHash link broken at entry ${i}: expected ${prevEntry.hash}, got ${entry.prevHash}` ++ brokenAt: at, ++ reason: `prevHash link broken at entry ${at}: expected ${expectedPrev}, got ${entry.prevHash}` + }; + } + } +@@ -161,7 +190,6 @@ function verifyChain(entries, publicKey) { + + return { ok: true }; + } +- + /** + * Generate an Ed25519 keypair for chain signing. + * @returns {Object} { publicKey: KeyObject, privateKey: KeyObject } +@@ -176,6 +204,7 @@ function generateKeyPair() { + export { + createChain, + verifyChain, ++ verifyRange, + generateKeyPair, + GENESIS_PREV_HASH + }; +diff --git a/src/chain/merkle.js b/src/chain/merkle.js +new file mode 100644 +--- /dev/null ++++ b/src/chain/merkle.js +@@ -0,0 +1,263 @@ ++import crypto from 'node:crypto'; ++ ++/** ++ * src/chain/merkle.js ++ * ++ * RFC 9162 (Certificate Transparency v2) section 2.1 Merkle tree hashing, the ++ * same tree as RFC 6962 and the one C2SP tlog-checkpoint names for its root ++ * hash. Node builtins only. ++ * ++ * leaf hash = SHA-256(0x00 || leaf data) ++ * node hash = SHA-256(0x01 || left || right) ++ * MTH({}) = SHA-256() of the empty string ++ * MTH(D[n]) = node(MTH(D[0:k]), MTH(D[k:n])), k the largest power of two ++ * strictly less than n ++ * ++ * Every function here takes and returns 32-byte Buffers. Tree sizes and ++ * indexes are plain non-negative safe integers; halving uses Math.floor, not ++ * the 32-bit bit operators, so sizes above 2^31 stay correct. ++ * ++ * Why no dependency: the algorithms are ~150 lines of RFC text, the npm ++ * candidates found on 2026-10-26 (ctjs, last published 2018, plus several ++ * release-candidate packages from 2026) are unmaintained or unproven, and this ++ * file lands in protected core, where a dependency is a supply-chain surface ++ * the gate cannot see. The RFC 6962 reference roots are pinned in the tests. ++ */ ++ ++const LEAF_PREFIX = Buffer.from([0x00]); ++const NODE_PREFIX = Buffer.from([0x01]); ++ ++function sha256(...parts) { ++ const h = crypto.createHash('sha256'); ++ for (const p of parts) h.update(p); ++ return h.digest(); ++} ++ ++/** MTH of the empty tree. */ ++const EMPTY_ROOT = sha256(Buffer.alloc(0)); ++ ++function leafHash(data) { ++ return sha256(LEAF_PREFIX, data); ++} ++ ++function nodeHash(left, right) { ++ return sha256(NODE_PREFIX, left, right); ++} ++ ++function isOdd(n) { ++ return n % 2 === 1; ++} ++ ++function half(n) { ++ return Math.floor(n / 2); ++} ++ ++/** Largest power of two strictly less than n (n >= 2). */ ++function splitPoint(n) { ++ let k = 1; ++ while (k * 2 < n) k *= 2; ++ return k; ++} ++ ++function requireSize(n, name) { ++ if (!Number.isSafeInteger(n) || n < 0) { ++ throw new Error(`${name} must be a non-negative safe integer, got ${n}`); ++ } ++} ++ ++/** ++ * MTH over leaf hashes[start:end]. ++ * @param {Buffer[]} hashes - leaf hashes (already leafHash()ed) ++ */ ++function rootFromLeafHashes(hashes, start = 0, end = hashes.length) { ++ const n = end - start; ++ if (n === 0) return EMPTY_ROOT; ++ if (n === 1) return hashes[start]; ++ const k = splitPoint(n); ++ return nodeHash(rootFromLeafHashes(hashes, start, start + k), rootFromLeafHashes(hashes, start + k, end)); ++} ++ ++/** ++ * RFC 9162 2.1.3.1 PATH(m, D[n]), leaf-to-root order. ++ * @param {number} m - leaf index, 0 <= m < n ++ * @param {Buffer[]} hashes - all n leaf hashes of the tree ++ */ ++function inclusionPath(m, hashes, start = 0, end = hashes.length) { ++ const n = end - start; ++ if (!(m >= 0 && m < n)) { ++ throw new Error(`leaf index ${m} out of range for tree size ${n}`); ++ } ++ if (n === 1) return []; ++ const k = splitPoint(n); ++ if (m < k) { ++ return [...inclusionPath(m, hashes, start, start + k), rootFromLeafHashes(hashes, start + k, end)]; ++ } ++ return [...inclusionPath(m - k, hashes, start + k, end), rootFromLeafHashes(hashes, start, start + k)]; ++} ++ ++/** ++ * RFC 9162 2.1.3.2: verify an inclusion proof. Pure; returns a boolean and ++ * never throws on hostile input. ++ */ ++function verifyInclusion(leafIndex, treeSize, leaf, path, root) { ++ if (!Number.isSafeInteger(leafIndex) || !Number.isSafeInteger(treeSize)) return false; ++ if (leafIndex < 0 || leafIndex >= treeSize) return false; ++ if (!Array.isArray(path) || !Buffer.isBuffer(leaf) || !Buffer.isBuffer(root)) return false; ++ let fn = leafIndex; ++ let sn = treeSize - 1; ++ let r = leaf; ++ for (const p of path) { ++ if (!Buffer.isBuffer(p) || p.length !== 32) return false; ++ if (sn === 0) return false; ++ if (isOdd(fn) || fn === sn) { ++ r = nodeHash(p, r); ++ if (!isOdd(fn)) { ++ while (!isOdd(fn) && fn !== 0) { ++ fn = half(fn); ++ sn = half(sn); ++ } ++ } ++ } else { ++ r = nodeHash(r, p); ++ } ++ fn = half(fn); ++ sn = half(sn); ++ } ++ return sn === 0 && r.equals(root); ++} ++ ++/** ++ * RFC 9162 2.1.4.1 SUBPROOF(m, D[n], b). ++ */ ++function subproof(m, hashes, start, end, b) { ++ const n = end - start; ++ if (m === n) { ++ return b ? [] : [rootFromLeafHashes(hashes, start, end)]; ++ } ++ const k = splitPoint(n); ++ if (m <= k) { ++ return [...subproof(m, hashes, start, start + k, b), rootFromLeafHashes(hashes, start + k, end)]; ++ } ++ return [...subproof(m - k, hashes, start + k, end, false), rootFromLeafHashes(hashes, start, start + k)]; ++} ++ ++/** ++ * RFC 9162 2.1.4.1 PROOF(m, D[n]): consistency proof that the tree of the ++ * first m leaves is a prefix of the tree of all n leaves. Empty when m is 0 ++ * or m equals n. ++ */ ++function consistencyProof(m, hashes) { ++ const n = hashes.length; ++ requireSize(m, 'old tree size'); ++ if (m > n) throw new Error(`old tree size ${m} exceeds new tree size ${n}`); ++ if (m === 0 || m === n) return []; ++ return subproof(m, hashes, 0, n, true); ++} ++ ++/** ++ * RFC 9162 2.1.4.2: verify a consistency proof. Pure; returns a boolean. ++ */ ++function verifyConsistency(first, second, firstHash, secondHash, proof) { ++ if (!Number.isSafeInteger(first) || !Number.isSafeInteger(second)) return false; ++ if (first < 0 || first > second) return false; ++ if (!Array.isArray(proof) || !Buffer.isBuffer(firstHash) || !Buffer.isBuffer(secondHash)) return false; ++ for (const p of proof) { ++ if (!Buffer.isBuffer(p) || p.length !== 32) return false; ++ } ++ if (first === second) return proof.length === 0 && firstHash.equals(secondHash); ++ if (first === 0) return proof.length === 0; ++ if (proof.length === 0) return false; ++ ++ const path = isPowerOfTwo(first) ? [firstHash, ...proof] : proof.slice(); ++ let fn = first - 1; ++ let sn = second - 1; ++ while (isOdd(fn)) { ++ fn = half(fn); ++ sn = half(sn); ++ } ++ let fr = path[0]; ++ let sr = path[0]; ++ for (let i = 1; i < path.length; i++) { ++ const c = path[i]; ++ if (sn === 0) return false; ++ if (isOdd(fn) || fn === sn) { ++ fr = nodeHash(c, fr); ++ sr = nodeHash(c, sr); ++ if (!isOdd(fn)) { ++ while (!isOdd(fn) && fn !== 0) { ++ fn = half(fn); ++ sn = half(sn); ++ } ++ } ++ } else { ++ sr = nodeHash(sr, c); ++ } ++ fn = half(fn); ++ sn = half(sn); ++ } ++ return sn === 0 && fr.equals(firstHash) && sr.equals(secondHash); ++} ++ ++function isPowerOfTwo(n) { ++ if (n < 1) return false; ++ let x = n; ++ while (x > 1) { ++ if (isOdd(x)) return false; ++ x = half(x); ++ } ++ return true; ++} ++ ++/** ++ * Compact range ("frontier") of a tree: the roots of the perfect subtrees ++ * that make up its first `size` leaves, one per set bit of size, left to ++ * right. It is enough to extend the tree leaf by leaf and to compute the ++ * root, without the leaves themselves. Folding it right to left gives ++ * exactly MTH for every size, because MTH's split point is always the ++ * largest perfect subtree on the left. ++ */ ++function frontierAppend(frontier, size, leaf) { ++ const f = frontier.slice(); ++ let carry = leaf; ++ let s = size; ++ while (isOdd(s)) { ++ carry = nodeHash(f.pop(), carry); ++ s = half(s); ++ } ++ f.push(carry); ++ return f; ++} ++ ++function frontierRoot(frontier) { ++ if (frontier.length === 0) return EMPTY_ROOT; ++ let r = frontier[frontier.length - 1]; ++ for (let i = frontier.length - 2; i >= 0; i--) { ++ r = nodeHash(frontier[i], r); ++ } ++ return r; ++} ++ ++/** Number of set bits: the frontier length a tree of `size` must have. */ ++function popcount(size) { ++ let c = 0; ++ let s = size; ++ while (s > 0) { ++ if (isOdd(s)) c++; ++ s = half(s); ++ } ++ return c; ++} ++ ++export { ++ EMPTY_ROOT, ++ leafHash, ++ nodeHash, ++ rootFromLeafHashes, ++ inclusionPath, ++ verifyInclusion, ++ consistencyProof, ++ verifyConsistency, ++ frontierAppend, ++ frontierRoot, ++ popcount ++}; +diff --git a/src/store/checkpoints.js b/src/store/checkpoints.js +new file mode 100644 +--- /dev/null ++++ b/src/store/checkpoints.js +@@ -0,0 +1,487 @@ ++import fs from 'node:fs'; ++import path from 'node:path'; ++import crypto from 'node:crypto'; ++import { verifyRange, GENESIS_PREV_HASH } from '../chain/index.js'; ++import { ++ leafHash, ++ rootFromLeafHashes, ++ inclusionPath, ++ verifyInclusion, ++ consistencyProof, ++ frontierAppend, ++ frontierRoot, ++ popcount ++} from '../chain/merkle.js'; ++import { signCheckpoint, verifyCheckpoint, verifyInclusionProof, verifyConsistencyProof } from '../chain/checkpoint.js'; ++import { withLock } from './lock.js'; ++ ++/** ++ * src/store/checkpoints.js ++ * ++ * Chain segments, signed checkpoints and inclusion proofs (WO-CARTRIDGE-SEAL ++ * part B). chain.jsonl is never rewritten: every checkpoint lives in a ++ * sidecar, receipts/checkpoints.jsonl, one JSON record per line: ++ * ++ * { v: 1, size, offset, lastHash, lastPath: [hex], frontier: [hex], ++ * segmentRoot, note } ++ * ++ * size entries covered (seq 0 .. size-1); a multiple of the segment ++ * size, except a checkpoint sealed at the head on demand ++ * ({ atHead: true }), after which the next segment seal fills ++ * up to the next multiple again ++ * offset byte offset in chain.jsonl just past entry size-1's newline ++ * lastHash entry size-1's hash; prevHash of entry `size` must equal it ++ * lastPath RFC 9162 inclusion path of lastHash at index size-1. For the ++ * last leaf this is the frontier of size-1, reversed. ++ * frontier the compact range of the tree at `size`: enough to extend it ++ * without reading the entries before `size` ++ * segmentRoot MTH of the leaves this record's segment added (informational: ++ * it is signed only through the cumulative root) ++ * note the C2SP tlog-checkpoint signed note for (origin, size, root) ++ * ++ * Only `note` is signed. Everything else is a cache that the readers below ++ * check against the signed root before using it (frontier folds to the root; ++ * lastHash is proven into it by lastPath), so a tampered sidecar fails closed ++ * instead of being trusted. ++ * ++ * Locking: each segment is sealed under its own withLock() acquisition, never ++ * a whole migration under one, because the lock is considered stale after 30 ++ * s (src/store/lock.js) and appends give up after 5 s (KNOWN-LIMITS 7). One ++ * segment of 1024 entries verifies in about 80 ms (measured 2026-10-26). ++ * ++ * Signing uses the chain key (KNOWN-LIMITS 8 and 48 apply unchanged): a ++ * checkpoint shows that the key holder committed to a history. It catches a ++ * later rewrite only for someone who kept an earlier checkpoint, because the ++ * same key can sign a fresh set for a rewritten chain. ++ */ ++ ++/** Entries per sealed segment. Power of two: see proposals/chain-checkpoints.md for the measurement. */ ++const SEGMENT_SIZE = 1024; ++ ++const SIDECAR_NAME = 'checkpoints.jsonl'; ++const RECORD_VERSION = 1; ++const HEX64 = /^[0-9a-f]{64}$/; ++ ++function sidecarPath(baseDir) { ++ return path.join(baseDir, 'receipts', SIDECAR_NAME); ++} ++ ++function chainPath(baseDir) { ++ return path.join(baseDir, 'receipts', 'chain.jsonl'); ++} ++ ++function toPublicKeyObject(publicKey) { ++ return crypto.createPublicKey(publicKey); ++} ++ ++function hexes(list) { ++ return list.map((b) => b.toString('hex')); ++} ++ ++function buffers(list, what) { ++ if (!Array.isArray(list)) throw new Error(`${what} is not a list`); ++ return list.map((h) => { ++ if (typeof h !== 'string' || !HEX64.test(h)) throw new Error(`${what} holds a value that is not a 64-hex hash`); ++ return Buffer.from(h, 'hex'); ++ }); ++} ++ ++/** ++ * Read the sidecar. A missing file is no checkpoints. A line that does not ++ * parse is an error, not a skip: silently dropping a checkpoint would let a ++ * reader start from an older one without knowing. ++ */ ++function readCheckpointRecords(baseDir) { ++ const file = sidecarPath(baseDir); ++ if (!fs.existsSync(file)) return []; ++ const lines = fs.readFileSync(file, 'utf8').split('\n'); ++ const records = []; ++ for (let i = 0; i < lines.length; i++) { ++ if (lines[i].trim() === '') continue; ++ let rec; ++ try { ++ rec = JSON.parse(lines[i]); ++ } catch { ++ throw new Error(`lotor: ${SIDECAR_NAME} line ${i + 1} is not valid JSON`); ++ } ++ if (rec === null || typeof rec !== 'object' || rec.v !== RECORD_VERSION) { ++ throw new Error(`lotor: ${SIDECAR_NAME} line ${i + 1} is not a v${RECORD_VERSION} checkpoint record`); ++ } ++ records.push(rec); ++ } ++ for (let i = 1; i < records.length; i++) { ++ if (!(records[i].size > records[i - 1].size)) { ++ throw new Error(`lotor: ${SIDECAR_NAME} sizes do not increase at record ${i + 1}`); ++ } ++ } ++ return records; ++} ++ ++/** ++ * Read complete entry lines from chain.jsonl starting at a byte offset. ++ * Stops after maxEntries entries, or at the last complete line (a torn tail ++ * with no newline is left for later). Blank lines are skipped, as loadChain ++ * skips them. A line that does not parse is an error here, where loadChain ++ * would drop it: a checkpoint must never cover a line nobody could read. ++ * ++ * @returns {{ entries: Array<{ entry: Object, end: number }>, fileSize: number }} ++ */ ++function readEntriesFrom(baseDir, offset, maxEntries = Infinity) { ++ const file = chainPath(baseDir); ++ let fd; ++ try { ++ fd = fs.openSync(file, 'r'); ++ } catch (err) { ++ if (err.code === 'ENOENT') { ++ if (offset !== 0) throw new Error(`lotor: chain.jsonl is missing but a checkpoint covers it up to byte ${offset}`); ++ return { entries: [], fileSize: 0 }; ++ } ++ throw err; ++ } ++ try { ++ const fileSize = fs.fstatSync(fd).size; ++ if (offset > fileSize) { ++ throw new Error(`lotor: chain.jsonl is ${fileSize} bytes but a checkpoint covers it up to byte ${offset} (truncated)`); ++ } ++ const entries = []; ++ const CHUNK = 1024 * 1024; ++ let pos = offset; ++ let carry = Buffer.alloc(0); ++ let carryStart = offset; ++ while (pos < fileSize && entries.length < maxEntries) { ++ const len = Math.min(CHUNK, fileSize - pos); ++ const buf = Buffer.allocUnsafe(len); ++ let got = 0; ++ while (got < len) { ++ const n = fs.readSync(fd, buf, got, len - got, pos + got); ++ if (n === 0) throw new Error('lotor: chain file shrank while it was being read'); ++ got += n; ++ } ++ pos += len; ++ const data = carry.length ? Buffer.concat([carry, buf]) : buf; ++ let lineStart = 0; ++ for (;;) { ++ if (entries.length >= maxEntries) break; ++ const nl = data.indexOf(0x0a, lineStart); ++ if (nl < 0) break; ++ const line = data.toString('utf8', lineStart, nl); ++ const end = carryStart + nl + 1; ++ if (line.trim() !== '') { ++ let entry; ++ try { ++ entry = JSON.parse(line); ++ } catch { ++ throw new Error(`lotor: chain.jsonl has an unparseable line ending at byte ${end}`); ++ } ++ entries.push({ entry, end }); ++ } ++ lineStart = nl + 1; ++ } ++ carry = data.subarray(lineStart); ++ carryStart += lineStart; ++ } ++ return { entries, fileSize }; ++ } finally { ++ fs.closeSync(fd); ++ } ++} ++ ++/** ++ * Check a record's caches against its own signed note. Returns the state a ++ * reader may extend from, or throws with the reason. ++ */ ++function trustedStateFrom(record, publicKey) { ++ const cp = verifyCheckpoint(record.note, publicKey); ++ if (!cp.ok) throw new Error(`checkpoint at size ${record.size}: ${cp.reason}`); ++ if (cp.size !== record.size) throw new Error(`checkpoint record says size ${record.size} but its note signs size ${cp.size}`); ++ if (!Number.isSafeInteger(record.offset) || record.offset < 0) throw new Error(`checkpoint at size ${cp.size} has no usable byte offset`); ++ const frontier = buffers(record.frontier, `checkpoint ${cp.size} frontier`); ++ if (frontier.length !== popcount(cp.size) || !frontierRoot(frontier).equals(cp.root)) { ++ throw new Error(`checkpoint at size ${cp.size}: cached frontier does not fold to the signed root`); ++ } ++ let lastHash = GENESIS_PREV_HASH; ++ if (cp.size > 0) { ++ if (typeof record.lastHash !== 'string' || !HEX64.test(record.lastHash)) { ++ throw new Error(`checkpoint at size ${cp.size} has no usable lastHash`); ++ } ++ const lastPath = buffers(record.lastPath, `checkpoint ${cp.size} lastPath`); ++ if (!verifyInclusion(cp.size - 1, cp.size, leafHash(Buffer.from(record.lastHash, 'hex')), lastPath, cp.root)) { ++ throw new Error(`checkpoint at size ${cp.size}: lastHash is not leaf ${cp.size - 1} of the signed tree`); ++ } ++ lastHash = record.lastHash; ++ } ++ return { size: cp.size, root: cp.root, frontier, offset: record.offset, lastHash, note: record.note }; ++} ++ ++const GENESIS_STATE = Object.freeze({ size: 0, root: null, frontier: [], offset: 0, lastHash: GENESIS_PREV_HASH, note: null }); ++ ++/** ++ * Walk entries forward from a trusted state: verify them as a chain range, ++ * extend the tree, and check every later checkpoint record against the tree ++ * this walk computes. Read-only. ++ */ ++function walkFrom(state, items, laterRecords, publicKeyObj, publicKey) { ++ const entries = items.map((x) => x.entry); ++ const range = verifyRange(entries, publicKeyObj, { startSeq: state.size, prevHash: state.lastHash }); ++ if (!range.ok) return range; ++ ++ let frontier = state.frontier; ++ let size = state.size; ++ let ri = 0; ++ let segmentLeaves = []; ++ for (let i = 0; i < items.length; i++) { ++ const leaf = leafHash(Buffer.from(entries[i].hash, 'hex')); ++ frontier = frontierAppend(frontier, size, leaf); ++ size++; ++ segmentLeaves.push(leaf); ++ while (ri < laterRecords.length && laterRecords[ri].size === size) { ++ const rec = laterRecords[ri]; ++ const cp = verifyCheckpoint(rec.note, publicKey); ++ if (!cp.ok) return { ok: false, brokenAt: size - 1, reason: `checkpoint at size ${size}: ${cp.reason}` }; ++ if (cp.size !== size) return { ok: false, brokenAt: size - 1, reason: `checkpoint record at size ${size} signs size ${cp.size}` }; ++ if (!frontierRoot(frontier).equals(cp.root)) { ++ return { ok: false, brokenAt: size - 1, reason: `entries 0..${size - 1} do not hash to the root the size-${size} checkpoint signs` }; ++ } ++ if (rec.offset !== items[i].end || rec.lastHash !== entries[i].hash) { ++ return { ok: false, brokenAt: size - 1, reason: `checkpoint record at size ${size} has a stale offset or lastHash` }; ++ } ++ if (rec.segmentRoot !== rootFromLeafHashes(segmentLeaves).toString('hex')) { ++ return { ok: false, brokenAt: size - 1, reason: `checkpoint record at size ${size} has a segmentRoot its entries do not produce` }; ++ } ++ segmentLeaves = []; ++ ri++; ++ } ++ } ++ if (ri < laterRecords.length) { ++ return { ok: false, brokenAt: size, reason: `a checkpoint covers ${laterRecords[ri].size} entries but the chain has ${size} (truncated or rewritten)` }; ++ } ++ return { ok: true, size }; ++} ++ ++/** ++ * Seal every complete segment not yet sealed. Never rewrites chain.jsonl; ++ * appends records to the sidecar. On a chain with no sidecar this is the ++ * migration: the first checkpoint covers seq 0 .. segmentSize-1 of the file ++ * as it already is. ++ * ++ * Refuses (throws) when the entries to be sealed do not verify, or when the ++ * last record does not check out against its own signature: it never signs ++ * over a history it could not verify. ++ * ++ * options.atHead: after the complete segments, also seal whatever is left ++ * at the head (a partial segment), so the newest receipt can be proven now ++ * rather than after the segment fills. A prediction signed before its ++ * outcome needs this. ++ * ++ * @returns {{ sealed: number, size: number }} checkpoints written, and the ++ * size the newest checkpoint covers ++ */ ++function sealCheckpoints(baseDir, keyPair, options = {}) { ++ const segmentSize = options.segmentSize === undefined ? SEGMENT_SIZE : options.segmentSize; ++ if (!Number.isSafeInteger(segmentSize) || segmentSize < 1) throw new Error(`segmentSize must be a positive integer, got ${segmentSize}`); ++ const maxSegments = options.maxSegments === undefined ? Infinity : options.maxSegments; ++ const publicKeyObj = toPublicKeyObject(keyPair.publicKey); ++ let sealed = 0; ++ let size = null; ++ const step = (atHead) => { ++ const rec = withLock(baseDir, () => sealOneSegment(baseDir, keyPair, publicKeyObj, segmentSize, atHead)); ++ if (rec === null) return false; ++ sealed++; ++ size = rec.size; ++ return true; ++ }; ++ while (sealed < maxSegments && step(false)) { /* one segment per lock */ } ++ if (options.atHead === true && sealed < maxSegments) step(true); ++ if (size === null) { ++ const records = readCheckpointRecords(baseDir); ++ size = records.length ? records[records.length - 1].size : 0; ++ } ++ return { sealed, size }; ++} ++ ++function sealOneSegment(baseDir, keyPair, publicKeyObj, segmentSize, atHead) { ++ const records = readCheckpointRecords(baseDir); ++ const state = records.length ? trustedStateFrom(records[records.length - 1], keyPair.publicKey) : GENESIS_STATE; ++ // Segments end on multiples of segmentSize, so a head checkpoint taken on ++ // demand does not shift the boundaries of the ones after it. ++ const want = (Math.floor(state.size / segmentSize) + 1) * segmentSize - state.size; ++ const { entries: items } = readEntriesFrom(baseDir, state.offset, want); ++ if (items.length === 0 || (items.length < want && !atHead)) return null; ++ ++ const entries = items.map((x) => x.entry); ++ const range = verifyRange(entries, publicKeyObj, { startSeq: state.size, prevHash: state.lastHash }); ++ if (!range.ok) throw new Error(`lotor: refusing to seal: ${range.reason}`); ++ ++ let frontier = state.frontier; ++ let size = state.size; ++ let lastPath = []; ++ const segmentLeaves = []; ++ for (const e of entries) { ++ const leaf = leafHash(Buffer.from(e.hash, 'hex')); ++ lastPath = frontier.slice().reverse(); ++ frontier = frontierAppend(frontier, size, leaf); ++ size++; ++ segmentLeaves.push(leaf); ++ } ++ const root = frontierRoot(frontier); ++ const note = signCheckpoint({ size, root }, keyPair.privateKey, keyPair.publicKey); ++ const record = { ++ v: RECORD_VERSION, ++ size, ++ offset: items[items.length - 1].end, ++ lastHash: entries[entries.length - 1].hash, ++ lastPath: hexes(lastPath), ++ frontier: hexes(frontier), ++ segmentRoot: rootFromLeafHashes(segmentLeaves).toString('hex'), ++ note ++ }; ++ const file = sidecarPath(baseDir); ++ if (!fs.existsSync(file)) fs.writeFileSync(file, '', { mode: 0o600 }); ++ fs.appendFileSync(file, JSON.stringify(record) + '\n'); ++ return record; ++} ++ ++/** ++ * Verify starting from a trusted checkpoint instead of genesis. Read-only. ++ * ++ * options.trusted: a checkpoint note the caller already trusts (one they kept, ++ * or one a cartridge carried). The sidecar must hold a record for the same ++ * size with the same signed root, or the result is a fork. Omitted: the ++ * newest sidecar record is the starting point. ++ * ++ * Reads only the entries after the starting checkpoint. With no checkpoint at ++ * all it reads from genesis, which is the full verify. ++ * ++ * @returns {{ ok: true, mode, from, size } | { ok: false, reason, brokenAt? }} ++ */ ++function verifyIncremental(baseDir, publicKey, options = {}) { ++ const publicKeyObj = toPublicKeyObject(publicKey); ++ let records; ++ try { ++ records = readCheckpointRecords(baseDir); ++ } catch (e) { ++ return { ok: false, reason: e.message }; ++ } ++ ++ let startIndex = records.length - 1; ++ if (options.trusted !== undefined) { ++ const t = verifyCheckpoint(options.trusted, publicKey); ++ if (!t.ok) return { ok: false, reason: `trusted checkpoint: ${t.reason}` }; ++ startIndex = records.findIndex((r) => r.size === t.size); ++ if (startIndex < 0) return { ok: false, reason: `no sealed checkpoint at size ${t.size}: this chain does not contain the checkpoint you trust` }; ++ const mine = verifyCheckpoint(records[startIndex].note, publicKey); ++ if (!mine.ok || !mine.root.equals(t.root)) { ++ return { ok: false, reason: `fork: the size-${t.size} checkpoint on disk does not sign the root you trust` }; ++ } ++ } ++ ++ let state = GENESIS_STATE; ++ if (startIndex >= 0) { ++ try { ++ state = trustedStateFrom(records[startIndex], publicKey); ++ } catch (e) { ++ return { ok: false, reason: e.message }; ++ } ++ } ++ let items; ++ try { ++ ({ entries: items } = readEntriesFrom(baseDir, state.offset)); ++ } catch (e) { ++ return { ok: false, reason: e.message }; ++ } ++ const result = walkFrom(state, items, records.slice(startIndex + 1), publicKeyObj, publicKey); ++ if (!result.ok) return result; ++ return { ok: true, mode: startIndex >= 0 ? 'incremental' : 'full', from: state.size, size: result.size }; ++} ++ ++/** ++ * Full verify of the chain AND every checkpoint, from genesis. Read-only. ++ * The plain chain verify (store.verify) is unchanged and still available. ++ */ ++function verifySealed(baseDir, publicKey) { ++ const publicKeyObj = toPublicKeyObject(publicKey); ++ let records; ++ let items; ++ try { ++ records = readCheckpointRecords(baseDir); ++ ({ entries: items } = readEntriesFrom(baseDir, 0)); ++ } catch (e) { ++ return { ok: false, reason: e.message }; ++ } ++ const result = walkFrom(GENESIS_STATE, items, records, publicKeyObj, publicKey); ++ if (!result.ok) return result; ++ return { ok: true, mode: 'full', from: 0, size: result.size, checkpoints: records.length }; ++} ++ ++function recordForSize(records, size) { ++ if (records.length === 0) throw new Error('lotor: no sealed checkpoint yet; run seal first'); ++ if (size === undefined) return records[records.length - 1]; ++ const rec = records.find((r) => r.size === size); ++ if (!rec) throw new Error(`lotor: no sealed checkpoint at size ${size}`); ++ return rec; ++} ++ ++function leafHashesUpTo(baseDir, size) { ++ const { entries: items } = readEntriesFrom(baseDir, 0, size); ++ if (items.length < size) throw new Error(`lotor: chain has ${items.length} entries, checkpoint needs ${size}`); ++ return items.map((x) => x.entry); ++} ++ ++/** ++ * Inclusion proof for the entry at `seq` against a sealed checkpoint (the ++ * newest by default, or options.size). Reads the entries the checkpoint ++ * covers. The proof is checked before it is returned, so a chain that no ++ * longer matches its checkpoint throws here instead of handing out a proof ++ * that will fail somewhere else. ++ * ++ * @returns {{ leaf, index, size, path: string[], checkpoint: string }} ++ */ ++function proveInclusion(baseDir, publicKey, seq, options = {}) { ++ const rec = recordForSize(readCheckpointRecords(baseDir), options.size); ++ if (!Number.isSafeInteger(seq) || seq < 0 || seq >= rec.size) { ++ throw new Error(`lotor: seq ${seq} is not covered by the size-${rec.size} checkpoint (it covers 0..${rec.size - 1}); seal first`); ++ } ++ const entries = leafHashesUpTo(baseDir, rec.size); ++ const leaves = entries.map((e) => leafHash(Buffer.from(e.hash, 'hex'))); ++ const proof = { ++ leaf: entries[seq].hash, ++ index: seq, ++ size: rec.size, ++ path: hexes(inclusionPath(seq, leaves)), ++ checkpoint: rec.note ++ }; ++ const check = verifyInclusionProof(proof, publicKey); ++ if (!check.ok) throw new Error(`lotor: chain does not match its own checkpoint: ${check.reason}`); ++ return proof; ++} ++ ++/** ++ * Consistency proof that the checkpoint at options.to (newest by default) ++ * extends the one at `fromSize`. Checked before it is returned. ++ * ++ * @returns {{ from: string, to: string, path: string[] }} ++ */ ++function proveConsistency(baseDir, publicKey, fromSize, options = {}) { ++ const records = readCheckpointRecords(baseDir); ++ const a = recordForSize(records, fromSize); ++ const b = recordForSize(records, options.to); ++ if (a.size > b.size) throw new Error(`lotor: checkpoint ${a.size} is newer than ${b.size}`); ++ const entries = leafHashesUpTo(baseDir, b.size); ++ const leaves = entries.map((e) => leafHash(Buffer.from(e.hash, 'hex'))); ++ const proof = { from: a.note, to: b.note, path: hexes(consistencyProof(a.size, leaves)) }; ++ const check = verifyConsistencyProof(proof, publicKey); ++ if (!check.ok) throw new Error(`lotor: checkpoints are not consistent: ${check.reason}`); ++ return proof; ++} ++ ++export { ++ SEGMENT_SIZE, ++ SIDECAR_NAME, ++ readCheckpointRecords, ++ readEntriesFrom, ++ sealCheckpoints, ++ verifyIncremental, ++ verifySealed, ++ proveInclusion, ++ proveConsistency ++}; +diff --git a/src/store/index.js b/src/store/index.js +--- a/src/store/index.js ++++ b/src/store/index.js +@@ -3,6 +3,7 @@ import path from 'node:path'; + import crypto from 'node:crypto'; + import { createChain, verifyChain, generateKeyPair, GENESIS_PREV_HASH } from '../chain/index.js'; + import { withLock } from './lock.js'; ++import * as checkpoints from './checkpoints.js'; + + /** + * src/store/index.js +@@ -467,6 +468,52 @@ function createStore(baseDir = DEFAULT_BASE_DIR) { + // Import public key from PEM for verification + const publicKey = crypto.createPublicKey(keyPair.publicKey); + return verifyChain(materialize(), publicKey); ++ }, ++ ++ /** ++ * Seal every complete segment of the chain into a signed C2SP ++ * checkpoint. Writes receipts/checkpoints.jsonl only; chain.jsonl is ++ * never rewritten. On a chain with no checkpoints this is the ++ * migration. See src/store/checkpoints.js. ++ * @param {Object} [options] - { segmentSize, maxSegments } ++ * @returns {Object} { sealed, size } ++ */ ++ seal(options) { ++ return checkpoints.sealCheckpoints(baseDir, keyPair, options); ++ }, ++ ++ /** ++ * Verify forward from the newest checkpoint, or from one the caller ++ * trusts ({ trusted: }), reading only what came after it. ++ * @returns {Object} { ok, mode, from, size } or { ok: false, reason } ++ */ ++ verifyIncremental(options) { ++ return checkpoints.verifyIncremental(baseDir, keyPair.publicKey, options); ++ }, ++ ++ /** ++ * Verify the whole chain and every checkpoint from genesis. ++ * @returns {Object} { ok, mode, from, size, checkpoints } or { ok: false, reason } ++ */ ++ verifySealed() { ++ return checkpoints.verifySealed(baseDir, keyPair.publicKey); ++ }, ++ ++ /** ++ * RFC 9162 inclusion proof for the entry at seq, against the newest ++ * checkpoint (or { size }). Verify it with verifyInclusionProof() from ++ * src/chain/checkpoint.js, which needs only the proof and the public key. ++ */ ++ proveInclusion(seq, options) { ++ return checkpoints.proveInclusion(baseDir, keyPair.publicKey, seq, options); ++ }, ++ ++ /** ++ * RFC 9162 consistency proof that the newest checkpoint (or { to }) ++ * extends the one at fromSize. ++ */ ++ proveConsistency(fromSize, options) { ++ return checkpoints.proveConsistency(baseDir, keyPair.publicKey, fromSize, options); + } + }; + } diff --git a/proposals/chain-checkpoints/checkpoint.js b/proposals/chain-checkpoints/checkpoint.js new file mode 100644 index 0000000..35a7e02 --- /dev/null +++ b/proposals/chain-checkpoints/checkpoint.js @@ -0,0 +1,285 @@ +import crypto from 'node:crypto'; +import { leafHash, verifyInclusion, verifyConsistency } from './merkle.js'; + +/** + * src/chain/checkpoint.js + * + * Signed checkpoints for the receipt chain, in the published formats, not a + * new one: + * + * - C2SP signed-note (c2sp.org/signed-note): note text, a blank line, then + * signature lines " base64(keyID4 || signature)". + * Ed25519, signature type 0x01, key ID = SHA-256(name || 0x0A || 0x01 || + * pubkey32)[:4]. + * - C2SP tlog-checkpoint (c2sp.org/tlog-checkpoint): note text is exactly + * three lines, origin, tree size, base64 RFC 6962 root. No extension + * lines are written (the spec does not recommend them). + * + * The signature-line marker is U+2014, which this repository otherwise bans + * from its text. It is built with String.fromCharCode below so that the ban + * stays checkable by a byte grep and the spec stays exact. + * + * Leaves are the chain's own entry hashes (32 raw bytes each, the SHA-256 of + * the canonical entry). The entry hash already commits to seq, timestamp, + * nonce, prevHash and payload, so a verifier needs no JSON canonicalisation + * to check a proof, and a cartridge can cite a receipt by its hash alone. + * + * Everything here is pure: no file I/O, no clock. A stranger needs only a + * proof object and the chain's raw 32-byte Ed25519 public key. + */ + +const SIGNATURE_MARK = String.fromCharCode(0x2014); +const NL = String.fromCharCode(10); +const SIG_TYPE_ED25519 = 0x01; +const ORIGIN_PREFIX = 'lotor.local/chain/'; + +/** SPKI DER prefix for a raw Ed25519 public key (RFC 8410). */ +const ED25519_SPKI_PREFIX = Buffer.from('302a300506032b6570032100', 'hex'); + +function rawPublicKey(key) { + if (Buffer.isBuffer(key) && key.length === 32) return key; + const obj = crypto.createPublicKey(key); + return obj.export({ format: 'der', type: 'spki' }).subarray(-32); +} + +function publicKeyObject(raw32) { + return crypto.createPublicKey({ key: Buffer.concat([ED25519_SPKI_PREFIX, raw32]), format: 'der', type: 'spki' }); +} + +/** The log origin (and key name) for a chain key: fixed, derived, offline. */ +function originFor(publicKey) { + const raw = rawPublicKey(publicKey); + return ORIGIN_PREFIX + crypto.createHash('sha256').update(raw).digest('hex').slice(0, 16); +} + +function keyId(name, raw32) { + return crypto.createHash('sha256') + .update(Buffer.from(name, 'utf8')) + .update(Buffer.from([0x0a, SIG_TYPE_ED25519])) + .update(raw32) + .digest() + .subarray(0, 4); +} + +/** C2SP verifier key string: name+hex(keyID)+base64(0x01 || pubkey). */ +function vkey(name, publicKey) { + const raw = rawPublicKey(publicKey); + return `${name}+${keyId(name, raw).toString('hex')}+${Buffer.concat([Buffer.from([SIG_TYPE_ED25519]), raw]).toString('base64')}`; +} + +/** Parse a C2SP vkey into { name, keyId, publicKeyRaw }. Throws on anything else. */ +function parseVkey(s) { + const parts = String(s).split('+'); + if (parts.length !== 3) throw new Error('vkey must have three +-separated parts'); + const [name, idHex, b64] = parts; + const material = decodeCanonicalBase64(b64); + if (material === null || material.length !== 33 || material[0] !== SIG_TYPE_ED25519) { + throw new Error('vkey is not an Ed25519 (type 0x01) key'); + } + const raw = material.subarray(1); + if (keyId(name, raw).toString('hex') !== idHex) throw new Error('vkey key ID does not match its name and key'); + return { name, keyId: Buffer.from(idHex, 'hex'), publicKeyRaw: Buffer.from(raw) }; +} + +function decodeCanonicalBase64(s) { + if (typeof s !== 'string' || s.length === 0 || s.length % 4 !== 0) return null; + if (!/^[A-Za-z0-9+/]*={0,2}$/.test(s)) return null; + const buf = Buffer.from(s, 'base64'); + return buf.toString('base64') === s ? buf : null; +} + +function isValidKeyName(name) { + return typeof name === 'string' && name.length > 0 && !/[\s+]/u.test(name); +} + +/** + * Sign note text (which must end in a newline) as a C2SP signed note. + * @param {string} text + * @param {string} name - key name + * @param {crypto.KeyObject|string} privateKey - Ed25519 private key + * @param {crypto.KeyObject|string|Buffer} publicKey - matching public key + */ +function signNote(text, name, privateKey, publicKey) { + if (typeof text !== 'string' || !text.endsWith(NL)) throw new Error('note text must end in a newline'); + if (!isValidKeyName(name)) throw new Error(`invalid key name: ${name}`); + const raw = rawPublicKey(publicKey); + const sig = crypto.sign(null, Buffer.from(text, 'utf8'), privateKey); + const blob = Buffer.concat([keyId(name, raw), sig]).toString('base64'); + return `${text}${NL}${SIGNATURE_MARK} ${name} ${blob}${NL}`; +} + +/** + * Split a signed note into its text and signature lines. Returns null for + * anything that is not a well-formed note. + */ +function parseNote(note) { + if (typeof note !== 'string' || !note.endsWith(NL)) return null; + for (let i = 0; i < note.length; i++) { + const c = note.charCodeAt(i); + if (c < 0x20 && c !== 0x0a) return null; + } + const sep = note.lastIndexOf(NL + NL); + if (sep < 0) return null; + const text = note.slice(0, sep + 1); + const sigLines = note.slice(sep + 2).split(NL); + sigLines.pop(); // the note ends in a newline, so the last split part is '' + if (sigLines.length === 0) return null; + const signatures = []; + for (const line of sigLines) { + const prefix = SIGNATURE_MARK + ' '; + if (!line.startsWith(prefix)) return null; + const rest = line.slice(prefix.length); + const space = rest.lastIndexOf(' '); + if (space <= 0) return null; + const name = rest.slice(0, space); + const blob = decodeCanonicalBase64(rest.slice(space + 1)); + if (!isValidKeyName(name) || blob === null || blob.length < 5) return null; + signatures.push({ name, keyId: blob.subarray(0, 4), signature: blob.subarray(4) }); + } + return { text, signatures }; +} + +/** + * Verify a signed note against one known Ed25519 key. Per the spec, signatures + * from unknown keys are ignored, a known key whose signature fails rejects the + * note, and a note with no verified known signature is rejected. + * @returns {{ ok: true, text: string } | { ok: false, reason: string }} + */ +function verifyNote(note, name, publicKey) { + const parsed = parseNote(note); + if (parsed === null) return { ok: false, reason: 'not a well-formed signed note' }; + let raw; + try { + raw = rawPublicKey(publicKey); + } catch (e) { + return { ok: false, reason: 'unusable public key' }; + } + const id = keyId(name, raw); + const keyObj = publicKeyObject(raw); + let verified = false; + for (const s of parsed.signatures) { + if (s.name !== name || !s.keyId.equals(id)) continue; + const ok = s.signature.length === 64 && + crypto.verify(null, Buffer.from(parsed.text, 'utf8'), keyObj, s.signature); + if (!ok) return { ok: false, reason: `signature by ${name} does not verify` }; + verified = true; + } + if (!verified) return { ok: false, reason: `no signature by ${name} (key ${id.toString('hex')})` }; + return { ok: true, text: parsed.text }; +} + +/** Checkpoint note text (unsigned) for a tree. */ +function checkpointText(origin, size, root) { + if (!Number.isSafeInteger(size) || size < 0) throw new Error(`tree size must be a non-negative safe integer, got ${size}`); + if (!Buffer.isBuffer(root) || root.length !== 32) throw new Error('root must be 32 bytes'); + return `${origin}${NL}${size}${NL}${root.toString('base64')}${NL}`; +} + +/** Parse verified checkpoint text into { origin, size, root }. Null if malformed. */ +function parseCheckpointText(text) { + const lines = text.split(NL); + lines.pop(); + if (lines.length < 3) return null; + const [origin, sizeStr, rootB64, ...extensions] = lines; + if (!origin || !/^(0|[1-9][0-9]*)$/.test(sizeStr)) return null; + const size = Number(sizeStr); + if (!Number.isSafeInteger(size)) return null; + const root = decodeCanonicalBase64(rootB64); + if (root === null || root.length !== 32) return null; + if (extensions.some((l) => l.length === 0)) return null; + return { origin, size, root, extensions }; +} + +/** Sign a checkpoint for a chain key. Key name = origin, per the spec. */ +function signCheckpoint({ size, root }, privateKey, publicKey) { + const origin = originFor(publicKey); + return signNote(checkpointText(origin, size, root), origin, privateKey, publicKey); +} + +/** + * Verify a checkpoint against a chain public key. + * @returns {{ ok: true, origin, size, root } | { ok: false, reason }} + */ +function verifyCheckpoint(note, publicKey) { + let origin; + try { + origin = originFor(publicKey); + } catch (e) { + return { ok: false, reason: 'unusable public key' }; + } + const v = verifyNote(note, origin, publicKey); + if (!v.ok) return v; + const cp = parseCheckpointText(v.text); + if (cp === null) return { ok: false, reason: 'checkpoint text is malformed' }; + if (cp.origin !== origin) return { ok: false, reason: `checkpoint origin ${cp.origin} is not this chain (${origin})` }; + if (cp.extensions.length !== 0) return { ok: false, reason: 'checkpoint carries extension lines; Lotor writes none' }; + return { ok: true, origin: cp.origin, size: cp.size, root: cp.root }; +} + +const HEX64 = /^[0-9a-f]{64}$/; + +function hexList(list) { + if (!Array.isArray(list)) return null; + const out = []; + for (const h of list) { + if (typeof h !== 'string' || !HEX64.test(h)) return null; + out.push(Buffer.from(h, 'hex')); + } + return out; +} + +/** + * Verify an inclusion proof as produced by the store's proveInclusion(): + * { leaf: , index: , size, path: [hex...], checkpoint: } + * Needs only the proof and the chain's public key. + * @returns {{ ok: true, index, size, root } | { ok: false, reason }} + */ +function verifyInclusionProof(proof, publicKey) { + if (proof === null || typeof proof !== 'object') return { ok: false, reason: 'proof is not an object' }; + const cp = verifyCheckpoint(proof.checkpoint, publicKey); + if (!cp.ok) return { ok: false, reason: `checkpoint: ${cp.reason}` }; + if (proof.size !== cp.size) return { ok: false, reason: `proof is for tree size ${proof.size}, checkpoint is size ${cp.size}` }; + if (typeof proof.leaf !== 'string' || !HEX64.test(proof.leaf)) return { ok: false, reason: 'leaf must be a 64-hex entry hash' }; + const path = hexList(proof.path); + if (path === null) return { ok: false, reason: 'path must be a list of 64-hex hashes' }; + const ok = verifyInclusion(proof.index, cp.size, leafHash(Buffer.from(proof.leaf, 'hex')), path, cp.root); + if (!ok) return { ok: false, reason: `entry ${proof.leaf} is not at index ${proof.index} of the tree this checkpoint signs` }; + return { ok: true, index: proof.index, size: cp.size, root: cp.root }; +} + +/** + * Verify that a newer checkpoint extends an older one: + * { from: , to: , path: [hex...] } + * @returns {{ ok: true, from, to } | { ok: false, reason }} + */ +function verifyConsistencyProof(proof, publicKey) { + if (proof === null || typeof proof !== 'object') return { ok: false, reason: 'proof is not an object' }; + const a = verifyCheckpoint(proof.from, publicKey); + if (!a.ok) return { ok: false, reason: `older checkpoint: ${a.reason}` }; + const b = verifyCheckpoint(proof.to, publicKey); + if (!b.ok) return { ok: false, reason: `newer checkpoint: ${b.reason}` }; + const path = hexList(proof.path); + if (path === null) return { ok: false, reason: 'path must be a list of 64-hex hashes' }; + if (!verifyConsistency(a.size, b.size, a.root, b.root, path)) { + return { ok: false, reason: `the size-${b.size} tree does not extend the size-${a.size} tree` }; + } + return { ok: true, from: a.size, to: b.size }; +} + +export { + SIGNATURE_MARK, + originFor, + keyId, + vkey, + parseVkey, + signNote, + parseNote, + verifyNote, + checkpointText, + parseCheckpointText, + signCheckpoint, + verifyCheckpoint, + verifyInclusionProof, + verifyConsistencyProof, + rawPublicKey +}; diff --git a/proposals/chain-checkpoints/measure-n.mjs b/proposals/chain-checkpoints/measure-n.mjs new file mode 100644 index 0000000..15c2563 --- /dev/null +++ b/proposals/chain-checkpoints/measure-n.mjs @@ -0,0 +1,45 @@ +// Measure per-entry costs to pick the segment size N (WO-CARTRIDGE-SEAL part B). +// Read-only. Run from the repo root: node proposals/chain-checkpoints/measure-n.mjs +import crypto from 'node:crypto'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const repo = process.cwd(); +const chainMod = await import(pathToFileURL(path.join(repo, 'src', 'chain', 'index.js')).href); +const { createChain, verifyChain, generateKeyPair } = chainMod; + +const kp = generateKeyPair(); +const pub = crypto.createPublicKey(kp.publicKey); +const priv = crypto.createPrivateKey(kp.privateKey); +const chain = createChain({ publicKey: pub, privateKey: priv }); +const COUNT = 20000; +let t0 = process.hrtime.bigint(); +for (let i = 0; i < COUNT; i++) chain.append({ type: 'gated_action', tool: 'Bash', i, note: 'x'.repeat(200) }); +let t1 = process.hrtime.bigint(); +console.log('append (sign) per entry us', Number(t1 - t0) / 1000 / COUNT); + +t0 = process.hrtime.bigint(); +const r = verifyChain(chain.entries, pub); +t1 = process.hrtime.bigint(); +console.log('verifyChain ok', r.ok, 'per entry us', Number(t1 - t0) / 1000 / COUNT, 'total ms', Number(t1 - t0) / 1e6); + +// Merkle leaf + node hashing cost per entry +t0 = process.hrtime.bigint(); +const leaves = chain.entries.map((e) => crypto.createHash('sha256').update(Buffer.concat([Buffer.from([0]), Buffer.from(e.hash, 'hex')])).digest()); +let level = leaves; +while (level.length > 1) { + const next = []; + for (let i = 0; i + 1 < level.length; i += 2) next.push(crypto.createHash('sha256').update(Buffer.concat([Buffer.from([1]), level[i], level[i + 1]])).digest()); + if (level.length % 2) next.push(level[level.length - 1]); + level = next; +} +t1 = process.hrtime.bigint(); +console.log('merkle leaf+tree per entry us', Number(t1 - t0) / 1000 / COUNT); + +// JSON parse cost per line +const lines = chain.entries.map((e) => JSON.stringify(e)); +console.log('avg line bytes', Math.round(lines.reduce((a, l) => a + l.length + 1, 0) / COUNT)); +t0 = process.hrtime.bigint(); +for (const l of lines) JSON.parse(l); +t1 = process.hrtime.bigint(); +console.log('parse per entry us', Number(t1 - t0) / 1000 / COUNT); diff --git a/proposals/chain-checkpoints/merkle.js b/proposals/chain-checkpoints/merkle.js new file mode 100644 index 0000000..c5c97b0 --- /dev/null +++ b/proposals/chain-checkpoints/merkle.js @@ -0,0 +1,263 @@ +import crypto from 'node:crypto'; + +/** + * src/chain/merkle.js + * + * RFC 9162 (Certificate Transparency v2) section 2.1 Merkle tree hashing, the + * same tree as RFC 6962 and the one C2SP tlog-checkpoint names for its root + * hash. Node builtins only. + * + * leaf hash = SHA-256(0x00 || leaf data) + * node hash = SHA-256(0x01 || left || right) + * MTH({}) = SHA-256() of the empty string + * MTH(D[n]) = node(MTH(D[0:k]), MTH(D[k:n])), k the largest power of two + * strictly less than n + * + * Every function here takes and returns 32-byte Buffers. Tree sizes and + * indexes are plain non-negative safe integers; halving uses Math.floor, not + * the 32-bit bit operators, so sizes above 2^31 stay correct. + * + * Why no dependency: the algorithms are ~150 lines of RFC text, the npm + * candidates found on 2026-10-26 (ctjs, last published 2018, plus several + * release-candidate packages from 2026) are unmaintained or unproven, and this + * file lands in protected core, where a dependency is a supply-chain surface + * the gate cannot see. The RFC 6962 reference roots are pinned in the tests. + */ + +const LEAF_PREFIX = Buffer.from([0x00]); +const NODE_PREFIX = Buffer.from([0x01]); + +function sha256(...parts) { + const h = crypto.createHash('sha256'); + for (const p of parts) h.update(p); + return h.digest(); +} + +/** MTH of the empty tree. */ +const EMPTY_ROOT = sha256(Buffer.alloc(0)); + +function leafHash(data) { + return sha256(LEAF_PREFIX, data); +} + +function nodeHash(left, right) { + return sha256(NODE_PREFIX, left, right); +} + +function isOdd(n) { + return n % 2 === 1; +} + +function half(n) { + return Math.floor(n / 2); +} + +/** Largest power of two strictly less than n (n >= 2). */ +function splitPoint(n) { + let k = 1; + while (k * 2 < n) k *= 2; + return k; +} + +function requireSize(n, name) { + if (!Number.isSafeInteger(n) || n < 0) { + throw new Error(`${name} must be a non-negative safe integer, got ${n}`); + } +} + +/** + * MTH over leaf hashes[start:end]. + * @param {Buffer[]} hashes - leaf hashes (already leafHash()ed) + */ +function rootFromLeafHashes(hashes, start = 0, end = hashes.length) { + const n = end - start; + if (n === 0) return EMPTY_ROOT; + if (n === 1) return hashes[start]; + const k = splitPoint(n); + return nodeHash(rootFromLeafHashes(hashes, start, start + k), rootFromLeafHashes(hashes, start + k, end)); +} + +/** + * RFC 9162 2.1.3.1 PATH(m, D[n]), leaf-to-root order. + * @param {number} m - leaf index, 0 <= m < n + * @param {Buffer[]} hashes - all n leaf hashes of the tree + */ +function inclusionPath(m, hashes, start = 0, end = hashes.length) { + const n = end - start; + if (!(m >= 0 && m < n)) { + throw new Error(`leaf index ${m} out of range for tree size ${n}`); + } + if (n === 1) return []; + const k = splitPoint(n); + if (m < k) { + return [...inclusionPath(m, hashes, start, start + k), rootFromLeafHashes(hashes, start + k, end)]; + } + return [...inclusionPath(m - k, hashes, start + k, end), rootFromLeafHashes(hashes, start, start + k)]; +} + +/** + * RFC 9162 2.1.3.2: verify an inclusion proof. Pure; returns a boolean and + * never throws on hostile input. + */ +function verifyInclusion(leafIndex, treeSize, leaf, path, root) { + if (!Number.isSafeInteger(leafIndex) || !Number.isSafeInteger(treeSize)) return false; + if (leafIndex < 0 || leafIndex >= treeSize) return false; + if (!Array.isArray(path) || !Buffer.isBuffer(leaf) || !Buffer.isBuffer(root)) return false; + let fn = leafIndex; + let sn = treeSize - 1; + let r = leaf; + for (const p of path) { + if (!Buffer.isBuffer(p) || p.length !== 32) return false; + if (sn === 0) return false; + if (isOdd(fn) || fn === sn) { + r = nodeHash(p, r); + if (!isOdd(fn)) { + while (!isOdd(fn) && fn !== 0) { + fn = half(fn); + sn = half(sn); + } + } + } else { + r = nodeHash(r, p); + } + fn = half(fn); + sn = half(sn); + } + return sn === 0 && r.equals(root); +} + +/** + * RFC 9162 2.1.4.1 SUBPROOF(m, D[n], b). + */ +function subproof(m, hashes, start, end, b) { + const n = end - start; + if (m === n) { + return b ? [] : [rootFromLeafHashes(hashes, start, end)]; + } + const k = splitPoint(n); + if (m <= k) { + return [...subproof(m, hashes, start, start + k, b), rootFromLeafHashes(hashes, start + k, end)]; + } + return [...subproof(m - k, hashes, start + k, end, false), rootFromLeafHashes(hashes, start, start + k)]; +} + +/** + * RFC 9162 2.1.4.1 PROOF(m, D[n]): consistency proof that the tree of the + * first m leaves is a prefix of the tree of all n leaves. Empty when m is 0 + * or m equals n. + */ +function consistencyProof(m, hashes) { + const n = hashes.length; + requireSize(m, 'old tree size'); + if (m > n) throw new Error(`old tree size ${m} exceeds new tree size ${n}`); + if (m === 0 || m === n) return []; + return subproof(m, hashes, 0, n, true); +} + +/** + * RFC 9162 2.1.4.2: verify a consistency proof. Pure; returns a boolean. + */ +function verifyConsistency(first, second, firstHash, secondHash, proof) { + if (!Number.isSafeInteger(first) || !Number.isSafeInteger(second)) return false; + if (first < 0 || first > second) return false; + if (!Array.isArray(proof) || !Buffer.isBuffer(firstHash) || !Buffer.isBuffer(secondHash)) return false; + for (const p of proof) { + if (!Buffer.isBuffer(p) || p.length !== 32) return false; + } + if (first === second) return proof.length === 0 && firstHash.equals(secondHash); + if (first === 0) return proof.length === 0; + if (proof.length === 0) return false; + + const path = isPowerOfTwo(first) ? [firstHash, ...proof] : proof.slice(); + let fn = first - 1; + let sn = second - 1; + while (isOdd(fn)) { + fn = half(fn); + sn = half(sn); + } + let fr = path[0]; + let sr = path[0]; + for (let i = 1; i < path.length; i++) { + const c = path[i]; + if (sn === 0) return false; + if (isOdd(fn) || fn === sn) { + fr = nodeHash(c, fr); + sr = nodeHash(c, sr); + if (!isOdd(fn)) { + while (!isOdd(fn) && fn !== 0) { + fn = half(fn); + sn = half(sn); + } + } + } else { + sr = nodeHash(sr, c); + } + fn = half(fn); + sn = half(sn); + } + return sn === 0 && fr.equals(firstHash) && sr.equals(secondHash); +} + +function isPowerOfTwo(n) { + if (n < 1) return false; + let x = n; + while (x > 1) { + if (isOdd(x)) return false; + x = half(x); + } + return true; +} + +/** + * Compact range ("frontier") of a tree: the roots of the perfect subtrees + * that make up its first `size` leaves, one per set bit of size, left to + * right. It is enough to extend the tree leaf by leaf and to compute the + * root, without the leaves themselves. Folding it right to left gives + * exactly MTH for every size, because MTH's split point is always the + * largest perfect subtree on the left. + */ +function frontierAppend(frontier, size, leaf) { + const f = frontier.slice(); + let carry = leaf; + let s = size; + while (isOdd(s)) { + carry = nodeHash(f.pop(), carry); + s = half(s); + } + f.push(carry); + return f; +} + +function frontierRoot(frontier) { + if (frontier.length === 0) return EMPTY_ROOT; + let r = frontier[frontier.length - 1]; + for (let i = frontier.length - 2; i >= 0; i--) { + r = nodeHash(frontier[i], r); + } + return r; +} + +/** Number of set bits: the frontier length a tree of `size` must have. */ +function popcount(size) { + let c = 0; + let s = size; + while (s > 0) { + if (isOdd(s)) c++; + s = half(s); + } + return c; +} + +export { + EMPTY_ROOT, + leafHash, + nodeHash, + rootFromLeafHashes, + inclusionPath, + verifyInclusion, + consistencyProof, + verifyConsistency, + frontierAppend, + frontierRoot, + popcount +}; diff --git a/proposals/chain-checkpoints/store-checkpoints.js b/proposals/chain-checkpoints/store-checkpoints.js new file mode 100644 index 0000000..4844a32 --- /dev/null +++ b/proposals/chain-checkpoints/store-checkpoints.js @@ -0,0 +1,487 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { verifyRange, GENESIS_PREV_HASH } from '../chain/index.js'; +import { + leafHash, + rootFromLeafHashes, + inclusionPath, + verifyInclusion, + consistencyProof, + frontierAppend, + frontierRoot, + popcount +} from '../chain/merkle.js'; +import { signCheckpoint, verifyCheckpoint, verifyInclusionProof, verifyConsistencyProof } from '../chain/checkpoint.js'; +import { withLock } from './lock.js'; + +/** + * src/store/checkpoints.js + * + * Chain segments, signed checkpoints and inclusion proofs (WO-CARTRIDGE-SEAL + * part B). chain.jsonl is never rewritten: every checkpoint lives in a + * sidecar, receipts/checkpoints.jsonl, one JSON record per line: + * + * { v: 1, size, offset, lastHash, lastPath: [hex], frontier: [hex], + * segmentRoot, note } + * + * size entries covered (seq 0 .. size-1); a multiple of the segment + * size, except a checkpoint sealed at the head on demand + * ({ atHead: true }), after which the next segment seal fills + * up to the next multiple again + * offset byte offset in chain.jsonl just past entry size-1's newline + * lastHash entry size-1's hash; prevHash of entry `size` must equal it + * lastPath RFC 9162 inclusion path of lastHash at index size-1. For the + * last leaf this is the frontier of size-1, reversed. + * frontier the compact range of the tree at `size`: enough to extend it + * without reading the entries before `size` + * segmentRoot MTH of the leaves this record's segment added (informational: + * it is signed only through the cumulative root) + * note the C2SP tlog-checkpoint signed note for (origin, size, root) + * + * Only `note` is signed. Everything else is a cache that the readers below + * check against the signed root before using it (frontier folds to the root; + * lastHash is proven into it by lastPath), so a tampered sidecar fails closed + * instead of being trusted. + * + * Locking: each segment is sealed under its own withLock() acquisition, never + * a whole migration under one, because the lock is considered stale after 30 + * s (src/store/lock.js) and appends give up after 5 s (KNOWN-LIMITS 7). One + * segment of 1024 entries verifies in about 80 ms (measured 2026-10-26). + * + * Signing uses the chain key (KNOWN-LIMITS 8 and 48 apply unchanged): a + * checkpoint shows that the key holder committed to a history. It catches a + * later rewrite only for someone who kept an earlier checkpoint, because the + * same key can sign a fresh set for a rewritten chain. + */ + +/** Entries per sealed segment. Power of two: see proposals/chain-checkpoints.md for the measurement. */ +const SEGMENT_SIZE = 1024; + +const SIDECAR_NAME = 'checkpoints.jsonl'; +const RECORD_VERSION = 1; +const HEX64 = /^[0-9a-f]{64}$/; + +function sidecarPath(baseDir) { + return path.join(baseDir, 'receipts', SIDECAR_NAME); +} + +function chainPath(baseDir) { + return path.join(baseDir, 'receipts', 'chain.jsonl'); +} + +function toPublicKeyObject(publicKey) { + return crypto.createPublicKey(publicKey); +} + +function hexes(list) { + return list.map((b) => b.toString('hex')); +} + +function buffers(list, what) { + if (!Array.isArray(list)) throw new Error(`${what} is not a list`); + return list.map((h) => { + if (typeof h !== 'string' || !HEX64.test(h)) throw new Error(`${what} holds a value that is not a 64-hex hash`); + return Buffer.from(h, 'hex'); + }); +} + +/** + * Read the sidecar. A missing file is no checkpoints. A line that does not + * parse is an error, not a skip: silently dropping a checkpoint would let a + * reader start from an older one without knowing. + */ +function readCheckpointRecords(baseDir) { + const file = sidecarPath(baseDir); + if (!fs.existsSync(file)) return []; + const lines = fs.readFileSync(file, 'utf8').split('\n'); + const records = []; + for (let i = 0; i < lines.length; i++) { + if (lines[i].trim() === '') continue; + let rec; + try { + rec = JSON.parse(lines[i]); + } catch { + throw new Error(`lotor: ${SIDECAR_NAME} line ${i + 1} is not valid JSON`); + } + if (rec === null || typeof rec !== 'object' || rec.v !== RECORD_VERSION) { + throw new Error(`lotor: ${SIDECAR_NAME} line ${i + 1} is not a v${RECORD_VERSION} checkpoint record`); + } + records.push(rec); + } + for (let i = 1; i < records.length; i++) { + if (!(records[i].size > records[i - 1].size)) { + throw new Error(`lotor: ${SIDECAR_NAME} sizes do not increase at record ${i + 1}`); + } + } + return records; +} + +/** + * Read complete entry lines from chain.jsonl starting at a byte offset. + * Stops after maxEntries entries, or at the last complete line (a torn tail + * with no newline is left for later). Blank lines are skipped, as loadChain + * skips them. A line that does not parse is an error here, where loadChain + * would drop it: a checkpoint must never cover a line nobody could read. + * + * @returns {{ entries: Array<{ entry: Object, end: number }>, fileSize: number }} + */ +function readEntriesFrom(baseDir, offset, maxEntries = Infinity) { + const file = chainPath(baseDir); + let fd; + try { + fd = fs.openSync(file, 'r'); + } catch (err) { + if (err.code === 'ENOENT') { + if (offset !== 0) throw new Error(`lotor: chain.jsonl is missing but a checkpoint covers it up to byte ${offset}`); + return { entries: [], fileSize: 0 }; + } + throw err; + } + try { + const fileSize = fs.fstatSync(fd).size; + if (offset > fileSize) { + throw new Error(`lotor: chain.jsonl is ${fileSize} bytes but a checkpoint covers it up to byte ${offset} (truncated)`); + } + const entries = []; + const CHUNK = 1024 * 1024; + let pos = offset; + let carry = Buffer.alloc(0); + let carryStart = offset; + while (pos < fileSize && entries.length < maxEntries) { + const len = Math.min(CHUNK, fileSize - pos); + const buf = Buffer.allocUnsafe(len); + let got = 0; + while (got < len) { + const n = fs.readSync(fd, buf, got, len - got, pos + got); + if (n === 0) throw new Error('lotor: chain file shrank while it was being read'); + got += n; + } + pos += len; + const data = carry.length ? Buffer.concat([carry, buf]) : buf; + let lineStart = 0; + for (;;) { + if (entries.length >= maxEntries) break; + const nl = data.indexOf(0x0a, lineStart); + if (nl < 0) break; + const line = data.toString('utf8', lineStart, nl); + const end = carryStart + nl + 1; + if (line.trim() !== '') { + let entry; + try { + entry = JSON.parse(line); + } catch { + throw new Error(`lotor: chain.jsonl has an unparseable line ending at byte ${end}`); + } + entries.push({ entry, end }); + } + lineStart = nl + 1; + } + carry = data.subarray(lineStart); + carryStart += lineStart; + } + return { entries, fileSize }; + } finally { + fs.closeSync(fd); + } +} + +/** + * Check a record's caches against its own signed note. Returns the state a + * reader may extend from, or throws with the reason. + */ +function trustedStateFrom(record, publicKey) { + const cp = verifyCheckpoint(record.note, publicKey); + if (!cp.ok) throw new Error(`checkpoint at size ${record.size}: ${cp.reason}`); + if (cp.size !== record.size) throw new Error(`checkpoint record says size ${record.size} but its note signs size ${cp.size}`); + if (!Number.isSafeInteger(record.offset) || record.offset < 0) throw new Error(`checkpoint at size ${cp.size} has no usable byte offset`); + const frontier = buffers(record.frontier, `checkpoint ${cp.size} frontier`); + if (frontier.length !== popcount(cp.size) || !frontierRoot(frontier).equals(cp.root)) { + throw new Error(`checkpoint at size ${cp.size}: cached frontier does not fold to the signed root`); + } + let lastHash = GENESIS_PREV_HASH; + if (cp.size > 0) { + if (typeof record.lastHash !== 'string' || !HEX64.test(record.lastHash)) { + throw new Error(`checkpoint at size ${cp.size} has no usable lastHash`); + } + const lastPath = buffers(record.lastPath, `checkpoint ${cp.size} lastPath`); + if (!verifyInclusion(cp.size - 1, cp.size, leafHash(Buffer.from(record.lastHash, 'hex')), lastPath, cp.root)) { + throw new Error(`checkpoint at size ${cp.size}: lastHash is not leaf ${cp.size - 1} of the signed tree`); + } + lastHash = record.lastHash; + } + return { size: cp.size, root: cp.root, frontier, offset: record.offset, lastHash, note: record.note }; +} + +const GENESIS_STATE = Object.freeze({ size: 0, root: null, frontier: [], offset: 0, lastHash: GENESIS_PREV_HASH, note: null }); + +/** + * Walk entries forward from a trusted state: verify them as a chain range, + * extend the tree, and check every later checkpoint record against the tree + * this walk computes. Read-only. + */ +function walkFrom(state, items, laterRecords, publicKeyObj, publicKey) { + const entries = items.map((x) => x.entry); + const range = verifyRange(entries, publicKeyObj, { startSeq: state.size, prevHash: state.lastHash }); + if (!range.ok) return range; + + let frontier = state.frontier; + let size = state.size; + let ri = 0; + let segmentLeaves = []; + for (let i = 0; i < items.length; i++) { + const leaf = leafHash(Buffer.from(entries[i].hash, 'hex')); + frontier = frontierAppend(frontier, size, leaf); + size++; + segmentLeaves.push(leaf); + while (ri < laterRecords.length && laterRecords[ri].size === size) { + const rec = laterRecords[ri]; + const cp = verifyCheckpoint(rec.note, publicKey); + if (!cp.ok) return { ok: false, brokenAt: size - 1, reason: `checkpoint at size ${size}: ${cp.reason}` }; + if (cp.size !== size) return { ok: false, brokenAt: size - 1, reason: `checkpoint record at size ${size} signs size ${cp.size}` }; + if (!frontierRoot(frontier).equals(cp.root)) { + return { ok: false, brokenAt: size - 1, reason: `entries 0..${size - 1} do not hash to the root the size-${size} checkpoint signs` }; + } + if (rec.offset !== items[i].end || rec.lastHash !== entries[i].hash) { + return { ok: false, brokenAt: size - 1, reason: `checkpoint record at size ${size} has a stale offset or lastHash` }; + } + if (rec.segmentRoot !== rootFromLeafHashes(segmentLeaves).toString('hex')) { + return { ok: false, brokenAt: size - 1, reason: `checkpoint record at size ${size} has a segmentRoot its entries do not produce` }; + } + segmentLeaves = []; + ri++; + } + } + if (ri < laterRecords.length) { + return { ok: false, brokenAt: size, reason: `a checkpoint covers ${laterRecords[ri].size} entries but the chain has ${size} (truncated or rewritten)` }; + } + return { ok: true, size }; +} + +/** + * Seal every complete segment not yet sealed. Never rewrites chain.jsonl; + * appends records to the sidecar. On a chain with no sidecar this is the + * migration: the first checkpoint covers seq 0 .. segmentSize-1 of the file + * as it already is. + * + * Refuses (throws) when the entries to be sealed do not verify, or when the + * last record does not check out against its own signature: it never signs + * over a history it could not verify. + * + * options.atHead: after the complete segments, also seal whatever is left + * at the head (a partial segment), so the newest receipt can be proven now + * rather than after the segment fills. A prediction signed before its + * outcome needs this. + * + * @returns {{ sealed: number, size: number }} checkpoints written, and the + * size the newest checkpoint covers + */ +function sealCheckpoints(baseDir, keyPair, options = {}) { + const segmentSize = options.segmentSize === undefined ? SEGMENT_SIZE : options.segmentSize; + if (!Number.isSafeInteger(segmentSize) || segmentSize < 1) throw new Error(`segmentSize must be a positive integer, got ${segmentSize}`); + const maxSegments = options.maxSegments === undefined ? Infinity : options.maxSegments; + const publicKeyObj = toPublicKeyObject(keyPair.publicKey); + let sealed = 0; + let size = null; + const step = (atHead) => { + const rec = withLock(baseDir, () => sealOneSegment(baseDir, keyPair, publicKeyObj, segmentSize, atHead)); + if (rec === null) return false; + sealed++; + size = rec.size; + return true; + }; + while (sealed < maxSegments && step(false)) { /* one segment per lock */ } + if (options.atHead === true && sealed < maxSegments) step(true); + if (size === null) { + const records = readCheckpointRecords(baseDir); + size = records.length ? records[records.length - 1].size : 0; + } + return { sealed, size }; +} + +function sealOneSegment(baseDir, keyPair, publicKeyObj, segmentSize, atHead) { + const records = readCheckpointRecords(baseDir); + const state = records.length ? trustedStateFrom(records[records.length - 1], keyPair.publicKey) : GENESIS_STATE; + // Segments end on multiples of segmentSize, so a head checkpoint taken on + // demand does not shift the boundaries of the ones after it. + const want = (Math.floor(state.size / segmentSize) + 1) * segmentSize - state.size; + const { entries: items } = readEntriesFrom(baseDir, state.offset, want); + if (items.length === 0 || (items.length < want && !atHead)) return null; + + const entries = items.map((x) => x.entry); + const range = verifyRange(entries, publicKeyObj, { startSeq: state.size, prevHash: state.lastHash }); + if (!range.ok) throw new Error(`lotor: refusing to seal: ${range.reason}`); + + let frontier = state.frontier; + let size = state.size; + let lastPath = []; + const segmentLeaves = []; + for (const e of entries) { + const leaf = leafHash(Buffer.from(e.hash, 'hex')); + lastPath = frontier.slice().reverse(); + frontier = frontierAppend(frontier, size, leaf); + size++; + segmentLeaves.push(leaf); + } + const root = frontierRoot(frontier); + const note = signCheckpoint({ size, root }, keyPair.privateKey, keyPair.publicKey); + const record = { + v: RECORD_VERSION, + size, + offset: items[items.length - 1].end, + lastHash: entries[entries.length - 1].hash, + lastPath: hexes(lastPath), + frontier: hexes(frontier), + segmentRoot: rootFromLeafHashes(segmentLeaves).toString('hex'), + note + }; + const file = sidecarPath(baseDir); + if (!fs.existsSync(file)) fs.writeFileSync(file, '', { mode: 0o600 }); + fs.appendFileSync(file, JSON.stringify(record) + '\n'); + return record; +} + +/** + * Verify starting from a trusted checkpoint instead of genesis. Read-only. + * + * options.trusted: a checkpoint note the caller already trusts (one they kept, + * or one a cartridge carried). The sidecar must hold a record for the same + * size with the same signed root, or the result is a fork. Omitted: the + * newest sidecar record is the starting point. + * + * Reads only the entries after the starting checkpoint. With no checkpoint at + * all it reads from genesis, which is the full verify. + * + * @returns {{ ok: true, mode, from, size } | { ok: false, reason, brokenAt? }} + */ +function verifyIncremental(baseDir, publicKey, options = {}) { + const publicKeyObj = toPublicKeyObject(publicKey); + let records; + try { + records = readCheckpointRecords(baseDir); + } catch (e) { + return { ok: false, reason: e.message }; + } + + let startIndex = records.length - 1; + if (options.trusted !== undefined) { + const t = verifyCheckpoint(options.trusted, publicKey); + if (!t.ok) return { ok: false, reason: `trusted checkpoint: ${t.reason}` }; + startIndex = records.findIndex((r) => r.size === t.size); + if (startIndex < 0) return { ok: false, reason: `no sealed checkpoint at size ${t.size}: this chain does not contain the checkpoint you trust` }; + const mine = verifyCheckpoint(records[startIndex].note, publicKey); + if (!mine.ok || !mine.root.equals(t.root)) { + return { ok: false, reason: `fork: the size-${t.size} checkpoint on disk does not sign the root you trust` }; + } + } + + let state = GENESIS_STATE; + if (startIndex >= 0) { + try { + state = trustedStateFrom(records[startIndex], publicKey); + } catch (e) { + return { ok: false, reason: e.message }; + } + } + let items; + try { + ({ entries: items } = readEntriesFrom(baseDir, state.offset)); + } catch (e) { + return { ok: false, reason: e.message }; + } + const result = walkFrom(state, items, records.slice(startIndex + 1), publicKeyObj, publicKey); + if (!result.ok) return result; + return { ok: true, mode: startIndex >= 0 ? 'incremental' : 'full', from: state.size, size: result.size }; +} + +/** + * Full verify of the chain AND every checkpoint, from genesis. Read-only. + * The plain chain verify (store.verify) is unchanged and still available. + */ +function verifySealed(baseDir, publicKey) { + const publicKeyObj = toPublicKeyObject(publicKey); + let records; + let items; + try { + records = readCheckpointRecords(baseDir); + ({ entries: items } = readEntriesFrom(baseDir, 0)); + } catch (e) { + return { ok: false, reason: e.message }; + } + const result = walkFrom(GENESIS_STATE, items, records, publicKeyObj, publicKey); + if (!result.ok) return result; + return { ok: true, mode: 'full', from: 0, size: result.size, checkpoints: records.length }; +} + +function recordForSize(records, size) { + if (records.length === 0) throw new Error('lotor: no sealed checkpoint yet; run seal first'); + if (size === undefined) return records[records.length - 1]; + const rec = records.find((r) => r.size === size); + if (!rec) throw new Error(`lotor: no sealed checkpoint at size ${size}`); + return rec; +} + +function leafHashesUpTo(baseDir, size) { + const { entries: items } = readEntriesFrom(baseDir, 0, size); + if (items.length < size) throw new Error(`lotor: chain has ${items.length} entries, checkpoint needs ${size}`); + return items.map((x) => x.entry); +} + +/** + * Inclusion proof for the entry at `seq` against a sealed checkpoint (the + * newest by default, or options.size). Reads the entries the checkpoint + * covers. The proof is checked before it is returned, so a chain that no + * longer matches its checkpoint throws here instead of handing out a proof + * that will fail somewhere else. + * + * @returns {{ leaf, index, size, path: string[], checkpoint: string }} + */ +function proveInclusion(baseDir, publicKey, seq, options = {}) { + const rec = recordForSize(readCheckpointRecords(baseDir), options.size); + if (!Number.isSafeInteger(seq) || seq < 0 || seq >= rec.size) { + throw new Error(`lotor: seq ${seq} is not covered by the size-${rec.size} checkpoint (it covers 0..${rec.size - 1}); seal first`); + } + const entries = leafHashesUpTo(baseDir, rec.size); + const leaves = entries.map((e) => leafHash(Buffer.from(e.hash, 'hex'))); + const proof = { + leaf: entries[seq].hash, + index: seq, + size: rec.size, + path: hexes(inclusionPath(seq, leaves)), + checkpoint: rec.note + }; + const check = verifyInclusionProof(proof, publicKey); + if (!check.ok) throw new Error(`lotor: chain does not match its own checkpoint: ${check.reason}`); + return proof; +} + +/** + * Consistency proof that the checkpoint at options.to (newest by default) + * extends the one at `fromSize`. Checked before it is returned. + * + * @returns {{ from: string, to: string, path: string[] }} + */ +function proveConsistency(baseDir, publicKey, fromSize, options = {}) { + const records = readCheckpointRecords(baseDir); + const a = recordForSize(records, fromSize); + const b = recordForSize(records, options.to); + if (a.size > b.size) throw new Error(`lotor: checkpoint ${a.size} is newer than ${b.size}`); + const entries = leafHashesUpTo(baseDir, b.size); + const leaves = entries.map((e) => leafHash(Buffer.from(e.hash, 'hex'))); + const proof = { from: a.note, to: b.note, path: hexes(consistencyProof(a.size, leaves)) }; + const check = verifyConsistencyProof(proof, publicKey); + if (!check.ok) throw new Error(`lotor: checkpoints are not consistent: ${check.reason}`); + return proof; +} + +export { + SEGMENT_SIZE, + SIDECAR_NAME, + readCheckpointRecords, + readEntriesFrom, + sealCheckpoints, + verifyIncremental, + verifySealed, + proveInclusion, + proveConsistency +}; diff --git a/proposals/make-chain-checkpoints-patch.mjs b/proposals/make-chain-checkpoints-patch.mjs new file mode 100644 index 0000000..29ee318 --- /dev/null +++ b/proposals/make-chain-checkpoints-patch.mjs @@ -0,0 +1,303 @@ +// WO-CARTRIDGE-SEAL part B: generate proposals/chain-checkpoints.patch from a +// script, never by hand (KNOWN-LIMITS 80). Reads the two core modules READ +// ONLY, applies exact string replacements to copies in a temp dir, places the +// three new core files from proposals/chain-checkpoints/, and emits the diff +// with git diff --no-index. Run from the repo root: +// node proposals/make-chain-checkpoints-patch.mjs +// +// Nothing here writes under src/. The patch lands only at a signing sitting, +// one target per apply (KNOWN-LIMITS 82), never from a shell in a live install. +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; + +const LF = String.fromCharCode(10); +const CRLF = String.fromCharCode(13, 10); +const ROOT = process.cwd(); +const PROPOSAL_DIR = path.join(ROOT, 'proposals', 'chain-checkpoints'); +const PATCH_OUT = path.join(ROOT, 'proposals', 'chain-checkpoints.patch'); + +const CHAIN_REL = ['src', 'chain', 'index.js']; +const STORE_REL = ['src', 'store', 'index.js']; +const NEW_FILES = [ + { from: 'merkle.js', to: ['src', 'chain', 'merkle.js'] }, + { from: 'checkpoint.js', to: ['src', 'chain', 'checkpoint.js'] }, + { from: 'store-checkpoints.js', to: ['src', 'store', 'checkpoints.js'] } +]; + +function readLF(p) { + return fs.readFileSync(p, 'utf8').split(CRLF).join(LF); +} + +function fail(msg) { + console.log('make-chain-checkpoints-patch: ' + msg); + process.exit(2); +} + +function replaceOnce(text, label, oldStr, newStr) { + const count = text.split(oldStr).length - 1; + if (count !== 1) fail('replacement ' + label + ': old text occurs ' + count + ' times, expected exactly 1'); + return text.split(oldStr).join(newStr); +} + +// ---- src/chain/index.js: verifyChain becomes verifyRange from genesis ---- + +const VERIFY_START = '/**' + LF + ' * Verify a chain of entries.'; +const VERIFY_END = LF + '/**' + LF + ' * Generate an Ed25519 keypair for chain signing.'; + +// Lines the replaced block must contain, so a drifted source fails loudly +// instead of being silently rewritten. +const VERIFY_MUST_CONTAIN = [ + 'function verifyChain(entries, publicKey) {', + ' const expectedHash = computeHash(entry);', + ' reason: `hash mismatch at entry ${i}: expected ${expectedHash}, got ${entry.hash}`', + ' reason: `signature verification failed at entry ${i}`', + ' if (!Number.isSafeInteger(entry.seq) || entry.seq !== i) {', + ' reason: `seq discontinuity at entry ${i}: expected ${i}, got ${JSON.stringify(entry.seq)}`', + ' reason: `genesis entry prevHash mismatch at entry ${i}: expected ${GENESIS_PREV_HASH}, got ${entry.prevHash}`', + ' reason: `prevHash link broken at entry ${i}: expected ${prevEntry.hash}, got ${entry.prevHash}`' +]; + +const VERIFY_NEW = [ + '/**', + ' * Verify a chain of entries.', + ' * @param {Array} entries - Array of entry objects', + ' * @param {crypto.KeyObject} publicKey - Ed25519 public key', + ' * @returns {Object} { ok: boolean, brokenAt?: number, reason?: string }', + ' */', + 'function verifyChain(entries, publicKey) {', + ' if (!entries || entries.length === 0) {', + ' return { ok: true };', + ' }', + '', + ' return verifyRange(entries, publicKey, { startSeq: 0, prevHash: GENESIS_PREV_HASH });', + '}', + '', + '/**', + ' * Verify a contiguous range of entries: the first carries seq startSeq and', + ' * links to prevHash (the hash of entry startSeq - 1, or the genesis constant', + ' * when startSeq is 0). Same checks, same order and same reasons as a full', + ' * verifyChain; entry numbers in brokenAt and in the reasons are absolute seqs,', + ' * so from startSeq 0 the result is exactly what verifyChain returned before', + ' * this function existed. A caller that already trusts the chain up to', + ' * startSeq (a verified checkpoint, src/store/checkpoints.js) verifies only', + ' * what came after it.', + ' * @param {Array} entries - Array of entry objects, in seq order', + ' * @param {crypto.KeyObject} publicKey - Ed25519 public key', + ' * @param {Object} [from] - { startSeq = 0, prevHash = GENESIS_PREV_HASH }', + ' * @returns {Object} { ok: boolean, brokenAt?: number, reason?: string }', + ' */', + 'function verifyRange(entries, publicKey, from = {}) {', + ' const startSeq = from.startSeq === undefined ? 0 : from.startSeq;', + ' const startPrevHash = from.prevHash === undefined ? GENESIS_PREV_HASH : from.prevHash;', + ' if (!Number.isSafeInteger(startSeq) || startSeq < 0) {', + ' throw new Error(`verifyRange: startSeq must be a non-negative safe integer, got ${startSeq}`);', + ' }', + ' if (!entries || entries.length === 0) {', + ' return { ok: true };', + ' }', + '', + ' for (let i = 0; i < entries.length; i++) {', + ' const entry = entries[i];', + ' const at = startSeq + i;', + '', + ' // Verify hash recomputation', + ' const expectedHash = computeHash(entry);', + ' if (expectedHash !== entry.hash) {', + ' return {', + ' ok: false,', + ' brokenAt: at,', + ' reason: `hash mismatch at entry ${at}: expected ${expectedHash}, got ${entry.hash}`', + ' };', + ' }', + '', + ' // Verify signature', + " const sigBuf = Buffer.from(entry.sig, 'hex');", + " const hashBuf = Buffer.from(entry.hash, 'hex');", + ' const sigOk = crypto.verify(null, hashBuf, publicKey, sigBuf);', + '', + ' if (!sigOk) {', + ' return {', + ' ok: false,', + ' brokenAt: at,', + ' reason: `signature verification failed at entry ${at}`', + ' };', + ' }', + '', + ' // Verify seq continuity. Entry i of a range that starts at startSeq must', + ' // carry seq startSeq + i (seq i for a full chain from genesis). The hash', + ' // covers seq, so a wrong seq can only be signed in at write time; this is', + ' // where it is caught.', + ' if (!Number.isSafeInteger(entry.seq) || entry.seq !== at) {', + ' return {', + ' ok: false,', + ' brokenAt: at,', + ' reason: `seq discontinuity at entry ${at}: expected ${at}, got ${JSON.stringify(entry.seq)}`', + ' };', + ' }', + '', + ' // Verify prevHash linkage', + ' if (at === 0) {', + ' // Genesis entry: prevHash must be genesis constant', + ' if (entry.prevHash !== GENESIS_PREV_HASH) {', + ' return {', + ' ok: false,', + ' brokenAt: at,', + ' reason: `genesis entry prevHash mismatch at entry ${at}: expected ${GENESIS_PREV_HASH}, got ${entry.prevHash}`', + ' };', + ' }', + ' } else {', + ' const expectedPrev = i > 0 ? entries[i - 1].hash : startPrevHash;', + ' if (entry.prevHash !== expectedPrev) {', + ' return {', + ' ok: false,', + ' brokenAt: at,', + ' reason: `prevHash link broken at entry ${at}: expected ${expectedPrev}, got ${entry.prevHash}`', + ' };', + ' }', + ' }', + ' }', + '', + ' return { ok: true };', + '}', + '' +].join(LF); + +function patchChain(src) { + const start = src.indexOf(VERIFY_START); + const end = src.indexOf(VERIFY_END); + if (start < 0 || end < 0 || end < start) fail('chain: verifyChain block markers not found'); + if (src.indexOf(VERIFY_START, start + 1) >= 0) fail('chain: verifyChain start marker occurs twice'); + const block = src.slice(start, end + 1); + for (const line of VERIFY_MUST_CONTAIN) { + if (block.split(line).length - 1 !== 1) fail('chain: expected line not found exactly once in verifyChain: ' + line); + } + let out = src.slice(0, start) + VERIFY_NEW + src.slice(end + 1); + out = replaceOnce(out, 'chain exports', + ' verifyChain,' + LF + ' generateKeyPair,', + ' verifyChain,' + LF + ' verifyRange,' + LF + ' generateKeyPair,'); + return out; +} + +// ---- src/store/index.js: expose the checkpoint operations on the store ---- + +function patchStore(src) { + let out = replaceOnce(src, 'store import', + "import { withLock } from './lock.js';", + "import { withLock } from './lock.js';" + LF + "import * as checkpoints from './checkpoints.js';"); + const OLD_TAIL = [ + ' const publicKey = crypto.createPublicKey(keyPair.publicKey);', + ' return verifyChain(materialize(), publicKey);', + ' }', + ' };', + '}' + ].join(LF); + const NEW_TAIL = [ + ' const publicKey = crypto.createPublicKey(keyPair.publicKey);', + ' return verifyChain(materialize(), publicKey);', + ' },', + '', + ' /**', + ' * Seal every complete segment of the chain into a signed C2SP', + ' * checkpoint. Writes receipts/checkpoints.jsonl only; chain.jsonl is', + ' * never rewritten. On a chain with no checkpoints this is the', + ' * migration. See src/store/checkpoints.js.', + ' * @param {Object} [options] - { segmentSize, maxSegments }', + ' * @returns {Object} { sealed, size }', + ' */', + ' seal(options) {', + ' return checkpoints.sealCheckpoints(baseDir, keyPair, options);', + ' },', + '', + ' /**', + ' * Verify forward from the newest checkpoint, or from one the caller', + ' * trusts ({ trusted: }), reading only what came after it.', + ' * @returns {Object} { ok, mode, from, size } or { ok: false, reason }', + ' */', + ' verifyIncremental(options) {', + ' return checkpoints.verifyIncremental(baseDir, keyPair.publicKey, options);', + ' },', + '', + ' /**', + ' * Verify the whole chain and every checkpoint from genesis.', + ' * @returns {Object} { ok, mode, from, size, checkpoints } or { ok: false, reason }', + ' */', + ' verifySealed() {', + ' return checkpoints.verifySealed(baseDir, keyPair.publicKey);', + ' },', + '', + ' /**', + ' * RFC 9162 inclusion proof for the entry at seq, against the newest', + ' * checkpoint (or { size }). Verify it with verifyInclusionProof() from', + ' * src/chain/checkpoint.js, which needs only the proof and the public key.', + ' */', + ' proveInclusion(seq, options) {', + ' return checkpoints.proveInclusion(baseDir, keyPair.publicKey, seq, options);', + ' },', + '', + ' /**', + ' * RFC 9162 consistency proof that the newest checkpoint (or { to })', + ' * extends the one at fromSize.', + ' */', + ' proveConsistency(fromSize, options) {', + ' return checkpoints.proveConsistency(baseDir, keyPair.publicKey, fromSize, options);', + ' }', + ' };', + '}' + ].join(LF); + out = replaceOnce(out, 'store methods', OLD_TAIL, NEW_TAIL); + return out; +} + +let tmp = null; +try { + const chainOld = readLF(path.join(ROOT, ...CHAIN_REL)); + const storeOld = readLF(path.join(ROOT, ...STORE_REL)); + const chainNew = patchChain(chainOld); + const storeNew = patchStore(storeOld); + + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'chain-checkpoints-patch-')); + const put = (side, rel, text) => { + const p = path.join(tmp, side, ...rel); + fs.mkdirSync(path.dirname(p), { recursive: true }); + fs.writeFileSync(p, text); + }; + put('old', CHAIN_REL, chainOld); + put('old', STORE_REL, storeOld); + put('new', CHAIN_REL, chainNew); + put('new', STORE_REL, storeNew); + for (const f of NEW_FILES) { + const text = readLF(path.join(PROPOSAL_DIR, f.from)); + if (text.indexOf(String.fromCharCode(0x2014)) >= 0) fail(f.from + ' contains U+2014; build it with String.fromCharCode'); + put('new', f.to, text); + } + + const r = spawnSync('git', ['-c', 'core.autocrlf=false', 'diff', '--no-index', '--no-color', '--no-renames', 'old', 'new'], + { cwd: tmp, encoding: 'utf8' }); + if (r.status !== 1) { + fail('git diff exited ' + r.status + ' (expected 1): ' + (r.stderr || r.error)); + } + // Only header lines carry the temp-dir prefixes; body lines stay + // byte-exact. Headers are recognised only between a 'diff --git' line and + // its first '@@', so a removed body line that starts '-- ' (or an added one + // that starts '++ ') is never mistaken for a header and rewritten. + const lines = r.stdout.split(LF).filter((l) => !l.startsWith('index ')); + let inHeader = false; + const rewritten = lines.map((l) => { + if (l.startsWith('diff --git ')) { + inHeader = true; + // A new file's header names the new side twice (a/new/... b/new/...). + return l.split(' a/old/').join(' a/').split(' a/new/').join(' a/').split(' b/new/').join(' b/'); + } + if (l.startsWith('@@')) inHeader = false; + if (inHeader && (l.startsWith('--- ') || l.startsWith('+++ '))) { + return l.split('--- a/old/').join('--- a/').split('+++ b/new/').join('+++ b/'); + } + return l; + }).join(LF); + fs.writeFileSync(PATCH_OUT, rewritten.endsWith(LF) ? rewritten : rewritten + LF); + console.log('wrote proposals/chain-checkpoints.patch, ' + fs.statSync(PATCH_OUT).size + ' bytes'); +} finally { + if (tmp !== null) fs.rmSync(tmp, { recursive: true, force: true }); +} diff --git a/test/chain-checkpoints.test.js b/test/chain-checkpoints.test.js new file mode 100644 index 0000000..b8a6ce8 --- /dev/null +++ b/test/chain-checkpoints.test.js @@ -0,0 +1,564 @@ +/** + * test/chain-checkpoints.test.js + * + * WO-CARTRIDGE-SEAL part B: chain segments, C2SP signed checkpoints, RFC 9162 + * inclusion and consistency proofs. + * + * STAGED, NOT YET LANDED. Every module this needs is core (src/chain, + * src/store), so it lands only at the owner's signing sitting, as + * proposals/chain-checkpoints.patch (generated by + * proposals/make-chain-checkpoints-patch.mjs, never typed). Same shape as + * test/chain-seq-continuity.test.js: + * + * - FIRST describe: the cases against the LIVE modules. Until the patch + * lands they run as TODO and fail (the functions do not exist), which is + * the fail-first half. The landing tripwire is src/chain/merkle.js + * existing in the live tree. + * - SECOND describe: the same cases against the QUEUED PATCH. In a temp + * dir it copies today's src/chain and src/store, applies the patch with + * git apply, imports the patched modules and runs every case for real. + * + * The two controls that pass on today's main are part of the evidence: a + * key-holder rewrite of a sealed segment verifies { ok: true } under the + * live verifyChain, and a forged checkpoint parses cleanly when nobody checks + * its signature. The checkpoint layer is what turns both into failures. + */ + +import { describe, it, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +import * as liveChain from '../src/chain/index.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO = path.join(__dirname, '..'); +const PATCH_PATH = path.join(REPO, 'proposals', 'chain-checkpoints.patch'); +const LANDED = fs.existsSync(path.join(REPO, 'src', 'chain', 'merkle.js')); +const TODO_MSG = 'lands with proposals/chain-checkpoints.patch at the signing sitting'; +const NL = String.fromCharCode(10); + +const lf = (s) => s.replace(/\r\n/g, '\n'); + +async function importSet(root) { + const load = async (...rel) => { + try { + return await import(pathToFileURL(path.join(root, ...rel)).href); + } catch { + return {}; + } + }; + return { + chain: await load('src', 'chain', 'index.js'), + merkle: await load('src', 'chain', 'merkle.js'), + cp: await load('src', 'chain', 'checkpoint.js'), + store: await load('src', 'store', 'index.js'), + storeCk: await load('src', 'store', 'checkpoints.js') + }; +} + +// ---------- helpers (test-side only, no core writes) ---------- + +function tmpHome(prefix) { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +function chainFile(home) { + return path.join(home, 'receipts', 'chain.jsonl'); +} + +function sidecarFile(home) { + return path.join(home, 'receipts', 'checkpoints.jsonl'); +} + +function sha256File(p) { + return crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex'); +} + +/** Append n receipts through the real store. */ +function appendMany(store, n, tag = 'r') { + for (let i = 0; i < n; i++) store.appendReceipt({ type: 'test', tag, i }); +} + +function readEntries(home) { + return fs.readFileSync(chainFile(home), 'utf8').split('\n').filter((l) => l.trim()).map((l) => JSON.parse(l)); +} + +function writeEntries(home, entries) { + fs.writeFileSync(chainFile(home), entries.map((e) => JSON.stringify(e) + '\n').join('')); +} + +/** + * A key holder rewrites history: change the payload of entry `at` and + * re-sign every entry from there with the SAME chain key. The plain chain + * verify cannot tell, by construction. + */ +function keyHolderRewrite(M, home, keyPair, at) { + const entries = readEntries(home); + const kept = entries.slice(0, at); + const c = M.chain.createChain(keyPair, at, kept); + c.append({ ...entries[at].payload, rewritten: true }); + for (let i = at + 1; i < entries.length; i++) c.append(entries[i].payload); + writeEntries(home, c.entries); +} + +const SEG = 8; // small segments keep the store cases fast; the 40k case uses the default + +// RFC 6962 reference vectors (the eight leaves from the certificate-transparency +// test data, as carried by transparency-dev/merkle), roots for sizes 1..8. +const RFC6962_LEAVES = ['', '00', '10', '2021', '3031', '40414243', '5051525354555657', '606162636465666768696a6b6c6d6e6f']; +const RFC6962_ROOTS = [ + '6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d', + 'fac54203e7cc696cf0dfcb42c92a1d9dbaf70ad9e621f4bd8d98662f00e3c125', + 'aeb6bcfe274b70a14fb067a5e5578264db0fa9b51af5e0ba159158f329e06e77', + 'd37ee418976dd95753c1c73862b9398fa2a2cf9b4ff0fdfe8b30cd95209614b7', + '4e3bbb1f7b478dcfe71fb631631519a3bca12c9aefca1612bfce4c13a86264d4', + '76e67dadbcdf1e10e1b74ddc608abd2f98dfb16fbce75277b5232a127f2087ef', + 'ddb89be403809e325750d3d263cd78929c2942b7942a34b77e122c9594a74c8c', + '5dc9da79a70659a9ad559cb701ded9a2ab9d823aad2f4960cfe370eff4604328' +]; + +// C2SP signed-note example (c2sp.org/signed-note, "Verifier keys", Example). +// The signature line's leading U+2014 is built, not typed. +const SPEC_VKEY = 'example.com/foo+530d903a+AekyeRrm56hApGFkyQR4ZCbV54Id2LKaANYcrnKv3U2k'; +const SPEC_NOTE = 'This is an example message.' + NL + NL + String.fromCharCode(0x2014) + + ' example.com/foo Uw2QOkn8srV1yJGh2VYRlL1Tnagv1YEq6TfXppzi2ONncAlTgK7Ztg1ERYNZXsYjOBH3mFXmRKuwHjG1Yu72IneyaQM=' + NL; + +/** + * The cases, as { name, fn(M) }. M is a module set from importSet(). + */ +function cases() { + return [ + { + name: 'merkle: RFC 6962 reference roots for sizes 1..8', + fn(M) { + const leaves = RFC6962_LEAVES.map((h) => M.merkle.leafHash(Buffer.from(h, 'hex'))); + for (let n = 1; n <= 8; n++) { + assert.equal(M.merkle.rootFromLeafHashes(leaves.slice(0, n)).toString('hex'), RFC6962_ROOTS[n - 1], `root of size ${n}`); + } + assert.equal(M.merkle.EMPTY_ROOT.toString('hex'), crypto.createHash('sha256').digest('hex')); + } + }, + { + name: 'merkle: every inclusion and consistency proof up to size 40 verifies, and a wrong one does not', + fn(M) { + const all = Array.from({ length: 40 }, (_, i) => M.merkle.leafHash(Buffer.from([i, i * 7 % 256]))); + for (let n = 1; n <= 40; n++) { + const leaves = all.slice(0, n); + const root = M.merkle.rootFromLeafHashes(leaves); + for (let m = 0; m < n; m++) { + const p = M.merkle.inclusionPath(m, leaves); + assert.ok(M.merkle.verifyInclusion(m, n, leaves[m], p, root), `inclusion ${m}/${n}`); + if (n > 1) assert.ok(!M.merkle.verifyInclusion((m + 1) % n, n, leaves[m], p, root), `wrong index ${m}/${n}`); + if (p.length) { + const bad = p.map((x) => Buffer.from(x)); + bad[0][0] ^= 1; + assert.ok(!M.merkle.verifyInclusion(m, n, leaves[m], bad, root), `tampered path ${m}/${n}`); + } + } + for (let m = 1; m <= n; m++) { + const old = M.merkle.rootFromLeafHashes(leaves.slice(0, m)); + const p = M.merkle.consistencyProof(m, leaves); + assert.ok(M.merkle.verifyConsistency(m, n, old, root, p), `consistency ${m}->${n}`); + if (m < n) { + const forkedOld = M.merkle.rootFromLeafHashes([...leaves.slice(0, m - 1), M.merkle.leafHash(Buffer.from('fork'))]); + assert.ok(!M.merkle.verifyConsistency(m, n, forkedOld, root, p), `forked consistency ${m}->${n}`); + } + } + } + } + }, + { + name: 'merkle: the frontier folds to MTH at every size 0..70', + fn(M) { + let f = []; + const leaves = []; + assert.ok(M.merkle.frontierRoot(f).equals(M.merkle.EMPTY_ROOT)); + for (let n = 1; n <= 70; n++) { + const leaf = M.merkle.leafHash(Buffer.from(String(n))); + f = M.merkle.frontierAppend(f, n - 1, leaf); + leaves.push(leaf); + assert.equal(f.length, M.merkle.popcount(n)); + assert.ok(M.merkle.frontierRoot(f).equals(M.merkle.rootFromLeafHashes(leaves)), `size ${n}`); + } + } + }, + { + name: 'signed note: the C2SP spec example verifies, and one changed byte does not', + fn(M) { + const k = M.cp.parseVkey(SPEC_VKEY); + assert.equal(k.keyId.toString('hex'), '530d903a'); + const v = M.cp.verifyNote(SPEC_NOTE, k.name, k.publicKeyRaw); + assert.deepEqual(v, { ok: true, text: 'This is an example message.' + NL }); + const tampered = SPEC_NOTE.replace('example message', 'example massage'); + assert.equal(M.cp.verifyNote(tampered, k.name, k.publicKeyRaw).ok, false); + assert.equal(M.cp.vkey(k.name, k.publicKeyRaw), SPEC_VKEY, 'vkey round-trips'); + } + }, + { + name: 'verifyRange from genesis returns exactly what verifyChain returns (live module vs patched module)', + fn(M) { + const kp = M.chain.generateKeyPair(); + const pub = crypto.createPublicKey(kp.publicKey); + const make = () => { + const c = M.chain.createChain(kp); + for (let i = 0; i < 6; i++) c.append({ i }); + return c.entries.map((e) => ({ ...e })); + }; + const variants = []; + variants.push(make()); + let v = make(); v[3].payload = { i: 99 }; variants.push(v); + v = make(); v[2].sig = v[2].sig.replace(/^./, (c) => (c === 'a' ? 'b' : 'a')); variants.push(v); + v = make(); v[4].prevHash = v[1].hash; variants.push(v); + const skip = M.chain.createChain(kp, 3, make().slice(0, 2)); skip.append({ x: 1 }); variants.push(skip.entries); + const g = M.chain.createChain(kp, 5); g.append({ x: 1 }); variants.push(g.entries); + variants.push([]); + for (const entries of variants) { + const expected = liveChain.verifyChain(entries, pub); + assert.deepEqual(M.chain.verifyChain(entries, pub), expected); + assert.deepEqual(M.chain.verifyRange(entries, pub), expected); + } + } + }, + { + name: 'verifyRange: a tail verified from a trusted (seq, prevHash) reports absolute seqs', + fn(M) { + const kp = M.chain.generateKeyPair(); + const pub = crypto.createPublicKey(kp.publicKey); + const c = M.chain.createChain(kp); + for (let i = 0; i < 10; i++) c.append({ i }); + const tail = c.entries.slice(6); + assert.deepEqual(M.chain.verifyRange(tail, pub, { startSeq: 6, prevHash: c.entries[5].hash }), { ok: true }); + const wrong = M.chain.verifyRange(tail, pub, { startSeq: 6, prevHash: c.entries[4].hash }); + assert.equal(wrong.ok, false); + assert.equal(wrong.brokenAt, 6); + assert.match(wrong.reason, /prevHash link broken at entry 6/); + const off = M.chain.verifyRange(tail, pub, { startSeq: 5, prevHash: c.entries[4].hash }); + assert.equal(off.ok, false); + assert.match(off.reason, /seq discontinuity at entry 5: expected 5, got 6/); + } + }, + { + name: 'FAIL-FIRST (tamper inside a sealed segment): a key-holder rewrite verifies under verifyChain, and a kept checkpoint catches it', + fn(M) { + const home = tmpHome('lotor-ckpt-rewrite-'); + try { + const store = M.store.createStore(home); + appendMany(store, 3 * SEG + 3); + assert.deepEqual(store.seal({ segmentSize: SEG }), { sealed: 3, size: 3 * SEG }); + // A stranger keeps the size-16 checkpoint and a proof for entry 5. + const keptProof = store.proveInclusion(5, { size: 2 * SEG }); + const kept = keptProof.checkpoint; + assert.equal(M.cp.verifyInclusionProof(keptProof, store.keyPair.publicKey).ok, true); + + keyHolderRewrite(M, home, store.keyPair, 5); + fs.rmSync(sidecarFile(home)); + const store2 = M.store.createStore(home); + store2.seal({ segmentSize: SEG }); + + // Control, true on today's main: the rewritten chain verifies, and + // so does its freshly re-sealed checkpoint set. + const pub = crypto.createPublicKey(store2.keyPair.publicKey); + assert.deepEqual(liveChain.verifyChain(readEntries(home), pub), { ok: true }); + assert.equal(store2.verifySealed().ok, true); + + // The kept checkpoint is what catches it. + const inc = store2.verifyIncremental({ trusted: kept }); + assert.equal(inc.ok, false); + assert.match(inc.reason, /fork/); + const newProof = store2.proveInclusion(5, { size: 2 * SEG }); + assert.notEqual(newProof.leaf, keptProof.leaf); + assert.equal(M.cp.verifyInclusionProof({ ...newProof, checkpoint: kept }, store2.keyPair.publicKey).ok, false); + const cons = store2.proveConsistency(2 * SEG); + assert.equal(M.cp.verifyConsistencyProof({ ...cons, from: kept }, store2.keyPair.publicKey).ok, false); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + } + }, + { + name: 'FAIL-FIRST (tamper inside a sealed segment, no key): full sealed verify and a walk from an earlier checkpoint catch it', + fn(M) { + const home = tmpHome('lotor-ckpt-tamper-'); + try { + const store = M.store.createStore(home); + appendMany(store, 3 * SEG); + store.seal({ segmentSize: SEG }); + const first = store.proveInclusion(0, { size: SEG }).checkpoint; + const lines = fs.readFileSync(chainFile(home), 'utf8').split('\n'); + lines[SEG + 2] = lines[SEG + 2].replace('"tag":"r"', '"tag":"R"'); + fs.writeFileSync(chainFile(home), lines.join('\n')); + + const sealed = store.verifySealed(); + assert.equal(sealed.ok, false); + assert.match(sealed.reason, /hash mismatch at entry 10/); + const fromFirst = store.verifyIncremental({ trusted: first }); + assert.equal(fromFirst.ok, false); + assert.match(fromFirst.reason, /hash mismatch at entry 10/); + // Documented, not a defect: from the NEWEST checkpoint nothing before + // it is read, so an edit inside an older sealed segment is out of + // that walk's sight. That is what "incremental" means; verifySealed + // and store.verify() are the full reads. + const fromNewest = store.verifyIncremental(); + assert.equal(fromNewest.ok, true); + assert.equal(fromNewest.from, 3 * SEG); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + } + }, + { + name: 'FAIL-FIRST (forged checkpoint): wrong key, edited root, or a forged sidecar note is rejected', + fn(M) { + const home = tmpHome('lotor-ckpt-forge-'); + try { + const store = M.store.createStore(home); + appendMany(store, 2 * SEG); + store.seal({ segmentSize: SEG }); + const pub = store.keyPair.publicKey; + const real = store.proveInclusion(1).checkpoint; + const origin = M.cp.originFor(pub); + const cp = M.cp.parseCheckpointText(real.slice(0, real.lastIndexOf(NL + NL) + 1)); + + // (1) Same origin and tree, signed by a different key. + const other = M.chain.generateKeyPair(); + const text = M.cp.checkpointText(origin, cp.size, cp.root); + const forgedSig = text + NL + String.fromCharCode(0x2014) + ' ' + origin + ' ' + + Buffer.concat([M.cp.keyId(origin, M.cp.rawPublicKey(pub)), crypto.sign(null, Buffer.from(text), other.privateKey)]).toString('base64') + NL; + // (2) The real signature over an edited root. + const editedRoot = Buffer.from(cp.root); editedRoot[0] ^= 1; + const forgedRoot = real.replace(cp.root.toString('base64'), editedRoot.toString('base64')); + + for (const forged of [forgedSig, forgedRoot]) { + // Control: without the signature check, the forgery is a fine checkpoint. + const body = M.cp.parseCheckpointText(forged.slice(0, forged.lastIndexOf(NL + NL) + 1)); + assert.ok(body && body.size === cp.size, 'a signature-skipping reader would accept this'); + assert.equal(M.cp.verifyCheckpoint(forged, pub).ok, false); + } + + // (3) A forged note planted in the sidecar: readers and the sealer refuse it. + const recs = fs.readFileSync(sidecarFile(home), 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)); + recs[1].note = forgedSig.replace(NL + cp.size + NL, NL + recs[1].size + NL); + fs.writeFileSync(sidecarFile(home), recs.map((r) => JSON.stringify(r) + '\n').join('')); + assert.equal(store.verifyIncremental().ok, false); + assert.equal(store.verifySealed().ok, false); + appendMany(store, SEG); + assert.throws(() => store.seal({ segmentSize: SEG }), /checkpoint at size 16/); + assert.throws(() => store.proveInclusion(1), /does not match its own checkpoint|checkpoint/); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + } + }, + { + name: 'FAIL-FIRST (proof against the wrong checkpoint): another size, a forged size, or another chain fails', + fn(M) { + const home = tmpHome('lotor-ckpt-wrong-'); + const home2 = tmpHome('lotor-ckpt-wrong2-'); + try { + const store = M.store.createStore(home); + appendMany(store, 2 * SEG); + store.seal({ segmentSize: SEG }); + const pub = store.keyPair.publicKey; + const p8 = store.proveInclusion(3, { size: SEG }); + const p16 = store.proveInclusion(3, { size: 2 * SEG }); + assert.equal(M.cp.verifyInclusionProof(p8, pub).ok, true); + assert.equal(M.cp.verifyInclusionProof(p16, pub).ok, true); + + const mixed = M.cp.verifyInclusionProof({ ...p8, checkpoint: p16.checkpoint }, pub); + assert.equal(mixed.ok, false); + assert.match(mixed.reason, /tree size 8, checkpoint is size 16/); + const resized = M.cp.verifyInclusionProof({ ...p8, size: 2 * SEG, checkpoint: p16.checkpoint }, pub); + assert.equal(resized.ok, false); + assert.match(resized.reason, /is not at index 3/); + const otherLeaf = M.cp.verifyInclusionProof({ ...p16, index: 4 }, pub); + assert.equal(otherLeaf.ok, false); + + const store2 = M.store.createStore(home2); + appendMany(store2, 2 * SEG); + store2.seal({ segmentSize: SEG }); + const foreign = store2.proveInclusion(3, { size: SEG }); + assert.equal(M.cp.verifyInclusionProof(foreign, pub).ok, false, 'a proof from another chain fails under this key'); + assert.equal(M.cp.verifyInclusionProof({ ...p8, checkpoint: foreign.checkpoint }, store2.keyPair.publicKey).ok, false); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + fs.rmSync(home2, { recursive: true, force: true }); + } + } + }, + { + name: 'a head checkpoint on demand proves the newest receipt now, and segment boundaries stay on multiples', + fn(M) { + const home = tmpHome('lotor-ckpt-head-'); + try { + const store = M.store.createStore(home); + appendMany(store, 2 * SEG + 3); + assert.deepEqual(store.seal({ segmentSize: SEG }), { sealed: 2, size: 2 * SEG }); + assert.throws(() => store.proveInclusion(2 * SEG + 2), /not covered/); + assert.deepEqual(store.seal({ segmentSize: SEG, atHead: true }), { sealed: 1, size: 2 * SEG + 3 }); + const p = store.proveInclusion(2 * SEG + 2); + assert.equal(M.cp.verifyInclusionProof(p, store.keyPair.publicKey).ok, true); + assert.deepEqual(store.seal({ segmentSize: SEG, atHead: true }), { sealed: 0, size: 2 * SEG + 3 }, 'nothing new, nothing sealed'); + appendMany(store, SEG); + assert.deepEqual(store.seal({ segmentSize: SEG }), { sealed: 1, size: 3 * SEG }, 'the next segment ends on the multiple'); + const recs = fs.readFileSync(sidecarFile(home), 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)); + assert.deepEqual(recs.map((r) => r.size), [SEG, 2 * SEG, 2 * SEG + 3, 3 * SEG]); + assert.equal(store.verifySealed().ok, true); + // The older head checkpoint is still consistent with the newer tree. + const cons = store.proveConsistency(2 * SEG + 3); + assert.equal(M.cp.verifyConsistencyProof(cons, store.keyPair.publicKey).ok, true); + // Re-sealing from scratch reproduces byte-identical notes (Ed25519 is + // deterministic and the note text is fixed by size and root), so a + // stranger's kept checkpoint survives a lost sidecar. + const notes = recs.map((r) => r.note); + fs.rmSync(sidecarFile(home)); + store.seal({ segmentSize: SEG }); + const again = fs.readFileSync(sidecarFile(home), 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l).note); + assert.deepEqual(again, [notes[0], notes[1], notes[3]]); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + } + }, + { + name: 'sidecar caches fail closed: an edited frontier, an edited lastHash, or a truncated chain', + fn(M) { + const home = tmpHome('lotor-ckpt-cache-'); + try { + const store = M.store.createStore(home); + appendMany(store, 2 * SEG + 2); + store.seal({ segmentSize: SEG }); + const original = fs.readFileSync(sidecarFile(home), 'utf8'); + const edit = (fn) => { + const recs = original.split('\n').filter(Boolean).map((l) => JSON.parse(l)); + fn(recs[recs.length - 1]); + fs.writeFileSync(sidecarFile(home), recs.map((r) => JSON.stringify(r) + '\n').join('')); + }; + edit((r) => { r.frontier[0] = '0'.repeat(64); }); + assert.match(store.verifyIncremental().reason, /frontier does not fold/); + edit((r) => { r.lastHash = '1'.repeat(64); }); + assert.match(store.verifyIncremental().reason, /lastHash is not leaf 15/); + fs.writeFileSync(sidecarFile(home), original); + assert.equal(store.verifyIncremental().ok, true); + + const lines = fs.readFileSync(chainFile(home), 'utf8').split('\n').filter(Boolean); + fs.writeFileSync(chainFile(home), lines.slice(0, SEG + 3).map((l) => l + '\n').join('')); + const t = store.verifyIncremental(); + assert.equal(t.ok, false); + assert.match(t.reason, /truncated/); + assert.equal(store.verifySealed().ok, false); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + } + }, + { + name: 'FAIL-FIRST (migration): a 40,000-entry chain is sealed without changing a byte, and every entry hash is reproduced', + fn(M) { + // Fail fast where the feature is absent, before building 40k entries. + const N = M.storeCk.SEGMENT_SIZE; + assert.equal(N, 1024); + const home = tmpHome('lotor-ckpt-40k-'); + try { + const store = M.store.createStore(home); + // Build the fixture with the chain module's own writer and the + // store's on-disk line format (JSON + LF), as a real chain was written. + const c = M.chain.createChain(store.keyPair); + for (let i = 0; i < 40000; i++) c.append({ type: 'fixture', i, tool: i % 3 ? 'Bash' : 'Edit' }); + writeEntries(home, c.entries); + const before = sha256File(chainFile(home)); + const hashes = c.entries.map((e) => e.hash); + + const res = store.seal(); + assert.deepEqual(res, { sealed: Math.floor(40000 / N), size: Math.floor(40000 / N) * N }); + + assert.equal(sha256File(chainFile(home)), before, 'chain.jsonl is byte-identical after migration'); + assert.deepEqual(readEntries(home).map((e) => e.hash), hashes, 'every entry hash is reproduced'); + const recs = fs.readFileSync(sidecarFile(home), 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)); + assert.equal(recs[0].size, N, 'the first checkpoint covers seq 0..N-1 of the existing file'); + assert.deepEqual(store.verifySealed(), { ok: true, mode: 'full', from: 0, size: 40000, checkpoints: recs.length }); + assert.deepEqual(store.verifyIncremental(), { ok: true, mode: 'incremental', from: res.size, size: 40000 }); + + const pub = store.keyPair.publicKey; + for (const seq of [0, 1, 1023, 1024, 20000, res.size - 1]) { + const p = store.proveInclusion(seq); + assert.equal(p.leaf, hashes[seq]); + assert.equal(M.cp.verifyInclusionProof(p, pub).ok, true, `proof for ${seq}`); + } + assert.throws(() => store.proveInclusion(res.size), /not covered/); + + // Measurement for the cartridge follow-on (not asserted). + const p = store.proveInclusion(20000); + const noteBytes = Buffer.byteLength(p.checkpoint); + const compactFirst = 4 + 4 + 32 * p.path.length + 32 + 64; // index, size, path, root, sig + const compactNext = 4 + 32 * p.path.length; + console.log(` 40k chain: path ${p.path.length} hashes; note ${noteBytes} B; compact first proof ${compactFirst} B, each further proof ${compactNext} B`); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + } + } + ]; +} + +describe('chain checkpoints (live modules)', () => { + let live; + before(async () => { live = await importSet(REPO); }); + for (const c of cases()) { + it(c.name, LANDED ? {} : { todo: TODO_MSG }, () => c.fn(live)); + } +}); + +describe('queued patch: proposals/chain-checkpoints.patch', () => { + if (LANDED) { + it('the staged patch is already landed', { skip: 'src/chain/merkle.js is live; nothing left to rehearse' }, () => {}); + return; + } + + let tmp; + let patched; + + before(async () => { + // Rehearse the landing: today's core source (CRLF folded to LF, the + // house patch format), plus exactly the staged diff, in a throwaway repo. + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'chain-checkpoints-')); + for (const dir of [['src', 'chain'], ['src', 'store']]) { + fs.mkdirSync(path.join(tmp, ...dir), { recursive: true }); + for (const name of fs.readdirSync(path.join(REPO, ...dir))) { + const text = fs.readFileSync(path.join(REPO, ...dir, name), 'utf8'); + fs.writeFileSync(path.join(tmp, ...dir, name), lf(text)); + } + } + fs.writeFileSync(path.join(tmp, 'b.patch'), lf(fs.readFileSync(PATCH_PATH, 'utf8'))); + execFileSync('git', ['init', '-q'], { cwd: tmp }); + execFileSync('git', ['apply', 'b.patch'], { cwd: tmp }); + patched = await importSet(tmp); + assert.equal(typeof patched.storeCk.sealCheckpoints, 'function', 'the patched store exposes sealCheckpoints'); + }); + after(() => { if (tmp) fs.rmSync(tmp, { recursive: true, force: true }); }); + + it('the staged patch applies cleanly to today\'s source', () => { + assert.ok(patched); + }); + + it('the patch adds the three new core files and touches only src/chain and src/store', () => { + const text = lf(fs.readFileSync(PATCH_PATH, 'utf8')); + const targets = text.split('\n').filter((l) => l.startsWith('+++ ')).map((l) => l.slice(4)); + assert.deepEqual(targets.sort(), [ + 'b/src/chain/checkpoint.js', + 'b/src/chain/index.js', + 'b/src/chain/merkle.js', + 'b/src/store/checkpoints.js', + 'b/src/store/index.js' + ]); + assert.equal(text.includes(String.fromCharCode(0x2014)), false, 'the patch carries no U+2014 byte'); + }); + + for (const c of cases()) { + it(c.name, {}, () => c.fn(patched)); + } +});