diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b5ebead --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,57 @@ +name: ci + +on: + pull_request: + workflow_call: + workflow_dispatch: + +permissions: + contents: read + +jobs: + tests: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + path: marekvs + - name: resolve tested ondaDB revision + id: ondadb + working-directory: marekvs + run: | + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import tomllib + with open("Cargo.lock", "rb") as f: + lock = tomllib.load(f) + package = next(p for p in lock["package"] if p["name"] == "ondadb") + revision = package["source"].rsplit("#", 1)[1] + assert len(revision) == 40 and all(c in "0123456789abcdef" for c in revision) + print(f"ref={revision}") + PY + - uses: actions/checkout@v4 + with: + repository: ${{ github.repository_owner }}/ondadb + ref: ${{ steps.ondadb.outputs.ref }} + path: ondadb + - uses: dtolnay/rust-toolchain@1.97.1 + with: + components: rustfmt, clippy + - uses: Swatinem/rust-cache@v2 + with: + workspaces: | + marekvs + ondadb + - name: marekvs formatting, lint and regression suites + working-directory: marekvs + run: | + cargo fmt --all --check + cargo clippy --locked --workspace --all-targets -- -D warnings + cargo test --locked --workspace + python3 tests/chaos/grudge.py --test + - name: ondaDB regression suites (including range mask reuse) + working-directory: ondadb + run: | + cargo clippy --locked --all-targets -- -D warnings + cargo test --locked + cargo clippy --locked --all-targets --features unsafe-fastpath -- -D warnings + cargo test --locked --features unsafe-fastpath diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 897ccd9..acca078 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,16 +21,18 @@ on: branches: [main] tags: ["v*"] workflow_dispatch: - inputs: - ondadb-ref: - description: ondadb ref to build against - default: main env: IMAGE: ghcr.io/${{ github.repository }} jobs: + quality: + uses: ./.github/workflows/ci.yml + permissions: + contents: read + build: + needs: quality strategy: fail-fast: true matrix: @@ -49,11 +51,25 @@ jobs: with: path: marekvs + - name: resolve pinned ondaDB revision + id: ondadb + working-directory: marekvs + run: | + python3 - <<'PYTHON' >> "$GITHUB_OUTPUT" + import tomllib + with open("Cargo.lock", "rb") as f: + lock = tomllib.load(f) + package = next(p for p in lock["package"] if p["name"] == "ondadb") + revision = package["source"].rsplit("#", 1)[1] + assert len(revision) == 40 and all(c in "0123456789abcdef" for c in revision) + print(f"ref={revision}") + PYTHON + - name: checkout ondadb (path dependency) uses: actions/checkout@v4 with: repository: ${{ github.repository_owner }}/ondadb - ref: ${{ inputs.ondadb-ref || 'main' }} + ref: ${{ steps.ondadb.outputs.ref }} path: ondadb - name: point cargo at the checked-out ondadb diff --git a/Cargo.lock b/Cargo.lock index 624e30a..414e7bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1387,7 +1387,7 @@ dependencies = [ [[package]] name = "marekvs-cluster" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "chitchat", @@ -1401,7 +1401,7 @@ dependencies = [ [[package]] name = "marekvs-core" -version = "0.3.2" +version = "0.3.3" dependencies = [ "proptest", "serde_json", @@ -1411,7 +1411,7 @@ dependencies = [ [[package]] name = "marekvs-diff" -version = "0.3.2" +version = "0.3.3" dependencies = [ "marekvs-core", "proptest", @@ -1423,7 +1423,7 @@ dependencies = [ [[package]] name = "marekvs-engine" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "crossbeam-channel", @@ -1452,7 +1452,7 @@ dependencies = [ [[package]] name = "marekvs-operator" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "futures", @@ -1470,7 +1470,7 @@ dependencies = [ [[package]] name = "marekvs-proto" -version = "0.3.2" +version = "0.3.3" dependencies = [ "postcard", "serde", @@ -1479,7 +1479,7 @@ dependencies = [ [[package]] name = "marekvs-repl" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "marekvs-cluster", @@ -1496,11 +1496,11 @@ dependencies = [ [[package]] name = "marekvs-resp" -version = "0.3.2" +version = "0.3.3" [[package]] name = "marekvs-server" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "marekvs-cluster", @@ -1658,6 +1658,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "ondadb" version = "0.9.0" +source = "git+https://github.com/yannick/ondadb.git?rev=0f4ebc67434551c9d3d4cba899829234760f4215#0f4ebc67434551c9d3d4cba899829234760f4215" dependencies = [ "crc32fast", "crossbeam-channel", diff --git a/Cargo.toml b/Cargo.toml index 9d05029..a8f3a49 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ ] [workspace.package] -version = "0.3.2" +version = "0.3.3" edition = "2021" # 1.89: ondaDB ≥0.7 uses std::fs::File advisory locking for its DB LOCK file. rust-version = "1.89" @@ -22,7 +22,7 @@ rust-version = "1.89" [workspace.dependencies] # Canonical source; a sibling checkout at ../ondadb overrides it via the # [patch] in .cargo/config.toml (auto-generated by `just` — see _cargo-config). -ondadb = { git = "https://github.com/yannick/ondadb.git" } +ondadb = { git = "https://github.com/yannick/ondadb.git", rev = "0f4ebc67434551c9d3d4cba899829234760f4215" } tokio = { version = "1", features = ["full"] } bytes = "1" postcard = { version = "1", features = ["alloc"] } diff --git a/crates/marekvs-cluster/src/lib.rs b/crates/marekvs-cluster/src/lib.rs index 4a38d75..6000dde 100644 --- a/crates/marekvs-cluster/src/lib.rs +++ b/crates/marekvs-cluster/src/lib.rs @@ -372,6 +372,13 @@ impl Cluster { self.view.read().clone() } + /// Execute a short synchronous operation while placement cannot change. + /// Used for the final cold-purge eligibility check and local range commit. + pub fn with_view(&self, f: impl FnOnce(&View) -> T) -> T { + let view = self.view.read(); + f(&view) + } + /// Subscribe to view changes (value = epoch). pub fn watch(&self) -> watch::Receiver { self.view_tx.subscribe() diff --git a/crates/marekvs-engine/examples/idle_maintenance.rs b/crates/marekvs-engine/examples/idle_maintenance.rs new file mode 100644 index 0000000..f123b66 --- /dev/null +++ b/crates/marekvs-engine/examples/idle_maintenance.rs @@ -0,0 +1,105 @@ +//! Disposable, release-mode idle CPU fixture; see tests/idle_maintenance/README.md. +use marekvs_core::{ + envelope::{Envelope, RecordType}, + ikey, +}; +use marekvs_engine::{ + store::{Store, StoreConfig}, + Engine, +}; +use std::time::{Duration, Instant}; + +fn cpu_seconds() -> f64 { + let mut usage = std::mem::MaybeUninit::::uninit(); + // SAFETY: getrusage initializes the correctly sized output on success. + assert_eq!( + unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) }, + 0 + ); + let usage = unsafe { usage.assume_init() }; + usage.ru_utime.tv_sec as f64 + + usage.ru_stime.tv_sec as f64 + + (usage.ru_utime.tv_usec + usage.ru_stime.tv_usec) as f64 / 1_000_000.0 +} + +fn main() { + let args: Vec = std::env::args() + .skip(1) + .map(|s| s.parse().unwrap()) + .collect(); + assert_eq!( + args.len(), + 6, + "usage: idle_maintenance SHARDS RANGES DISTINCT_RANGES KEYS WARM_SECONDS SAMPLE_SECONDS" + ); + let (shards, ranges, distinct, keys, warm, seconds) = + (args[0], args[1], args[2], args[3], args[4], args[5]); + assert!( + shards > 0 + && shards <= 4096 + && distinct <= 4096 + && (ranges == 0 || distinct > 0) + && seconds > 0 + ); + let dir = tempfile::tempdir().unwrap(); + let store = Store::open(&StoreConfig { + data_dir: dir.path().to_string_lossy().into_owned(), + shard_threads: shards, + ..StoreConfig::default() + }) + .unwrap(); + // Seed the live memtables without closing/flushing them. All writes use + // their owning shard, exactly like production commands and repair. + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + for shard in 0..shards { + store + .run(shard as u16, move |ctx| { + for i in 0..ranges { + let pid = (i % distinct) as u16; + if pid as usize % shards != shard { + continue; + } + marekvs_engine::store::delete_partition_range(ctx, pid).unwrap(); + } + for i in 0..keys { + let key = format!("idle-key-{i}"); + if marekvs_core::pid_of(key.as_bytes()) as usize % shards != shard { + continue; + } + let env = Envelope { + flags: RecordType::String as u8, + hlc: 1 << 16, + origin: 1, + ttl_deadline_ms: 0, + }; + marekvs_engine::store::put_raw( + ctx, + &ikey::string_key(key.as_bytes()), + &env.encode_with(b"value"), + ); + } + }) + .await; + } + }); + let engine = Engine::new(store); + std::thread::sleep(Duration::from_secs(warm as u64)); + println!( + "BEGIN_METRICS\n{}", + engine.metrics.render(engine.started_at_ms, 0) + ); + let wall = Instant::now(); + let cpu = cpu_seconds(); + std::thread::sleep(Duration::from_secs(seconds as u64)); + let cpu = cpu_seconds() - cpu; + let wall = wall.elapsed().as_secs_f64(); + println!( + "END_METRICS\n{}", + engine.metrics.render(engine.started_at_ms, 0) + ); + println!("RESULT shards={shards} ranges={ranges} distinct={distinct} keys={keys} warm_seconds={warm} sample_seconds={wall:.3} cpu_seconds={cpu:.6} cpu_percent={:.3}", cpu / wall * 100.0); +} diff --git a/crates/marekvs-engine/src/cmd/generic.rs b/crates/marekvs-engine/src/cmd/generic.rs index f4d28c6..a424b75 100644 --- a/crates/marekvs-engine/src/cmd/generic.rs +++ b/crates/marekvs-engine/src/cmd/generic.rs @@ -124,10 +124,11 @@ pub async fn type_cmd(engine: &Arc, args: &[Vec]) -> Reply { } /// The envelope currently carrying this key's TTL (string, list, or head). -fn ttl_envelope(ctx: &ShardCtx, key: &[u8]) -> Option { +pub(crate) fn ttl_envelope(ctx: &ShardCtx, key: &[u8]) -> Option { // Lists carry their TTL on the collection head now (ctype 5 → `_` arm). match key_type(ctx, key)? { b's' => read_lww(ctx, &ikey::string_key(key), 0).map(|(e, _)| e), + b'l' => read_lww(ctx, &ikey::list_key(key), 0).map(|(e, _)| e), _ => get_head(ctx, key).map(|(e, _, _)| e), } } diff --git a/crates/marekvs-engine/src/cmd/server.rs b/crates/marekvs-engine/src/cmd/server.rs index b332465..e0b5d80 100644 --- a/crates/marekvs-engine/src/cmd/server.rs +++ b/crates/marekvs-engine/src/cmd/server.rs @@ -427,24 +427,90 @@ pub async fn info(engine: &Arc, args: &[Vec]) -> Reply { out.push_str(&format!("# Cluster\r\n{cluster}\r\n")); } if want("keyspace") { - let keys = keyspace_count(engine).await; + let stats = keyspace_stats(engine).await; out.push_str("# Keyspace\r\n"); - if keys > 0 { - out.push_str(&format!("db0:keys={keys},expires=0,avg_ttl=0\r\n")); + if stats.keys > 0 { + out.push_str(&format!( + "db0:keys={},expires={},avg_ttl={}\r\n", + stats.keys, + stats.expires, + stats.avg_ttl_ms() + )); } out.push_str("\r\n"); } Reply::Bulk(out.into_bytes()) } -/// Distinct visible user keys (same walk as DBSIZE; INFO is not a hot path). -async fn keyspace_count(engine: &Arc) -> i64 { - match dbsize(engine).await { - Reply::Int(n) => n, - _ => 0, +/// An observed aggregate across shards, not a global read snapshot. +#[derive(Default)] +struct KeyspaceStats { + keys: u64, + expires: u64, + remaining_ttl_ms: u128, +} + +impl KeyspaceStats { + fn avg_ttl_ms(&self) -> u64 { + self.remaining_ttl_ms + .checked_div(self.expires as u128) + .unwrap_or(0) as u64 } } +/// Enumerate candidates once, then inspect visibility and key-level deadlines +/// on each key's owning shard. Member deadlines and shadowed physical records +/// are not logical key expirations. This scan is explicit command work only. +async fn keyspace_stats(engine: &Arc) -> KeyspaceStats { + let shard_count = engine.store.shard_count(); + let groups = engine + .store + .run(0, move |ctx| { + let mut seen = std::collections::HashSet::new(); + let mut groups = vec![Vec::new(); shard_count]; + crate::store::scan_prefix_cmd(ctx, &[], |k, _| { + if let Some(p) = marekvs_core::ikey::parse(k) { + if p.tag != b'Z' + && p.userkey.first() != Some(&0) + && seen.insert(p.userkey.to_vec()) + { + groups[p.pid as usize % shard_count].push(p.userkey.to_vec()); + } + } + true + }); + groups + }) + .await; + let mut total = KeyspaceStats::default(); + for (shard, keys) in groups.into_iter().enumerate() { + if keys.is_empty() { + continue; + } + let stats = engine + .store + .run(shard as u16, move |ctx| { + let mut stats = KeyspaceStats::default(); + let now = crate::store::now_ms(); + for key in keys { + if let Some(env) = crate::cmd::generic::ttl_envelope(ctx, &key) { + stats.keys += 1; + if env.ttl_deadline_ms > now { + stats.expires += 1; + stats.remaining_ttl_ms += (env.ttl_deadline_ms - now) as u128; + } + } + } + stats + }) + .await; + total.keys += stats.keys; + total.expires += stats.expires; + total.remaining_ttl_ms += stats.remaining_ttl_ms; + } + total +} + /// REPLICAOF host port | REPLICAOF NO ONE (SLAVEOF alias). Reply is immediate; /// the actual sync/stream work happens in the background task installed via /// [`Engine::set_replicaof`]. @@ -468,40 +534,7 @@ pub fn replicaof(engine: &Arc, args: &[Vec]) -> Reply { } pub async fn dbsize(engine: &Arc) -> Reply { - // Approximate: count distinct visible user keys (bounded walk). - engine - .store - .run(0, |ctx| { - let mut n = 0i64; - let mut last: Option> = None; - crate::store::scan_prefix_cmd(ctx, &[], |k, v| { - if let Some(p) = marekvs_core::ikey::parse(k) { - if p.tag == b'Z' || p.userkey.first() == Some(&0) { - return true; - } - if last.as_deref() == Some(p.userkey) { - return true; - } - if let Some((env, pay)) = marekvs_core::envelope::Envelope::decode(v) { - let now = crate::store::now_ms(); - let vis = if p.tag == b'M' { - !env.is_tombstone() && !env.is_expired(now) - } else { - crate::store::visible(&env, pay, 0, now).is_some() - && (!env.rtype().is_or_element() - || marekvs_core::merge::element_value(pay).is_some()) - }; - if vis { - last = Some(p.userkey.to_vec()); - n += 1; - } - } - } - true - }); - Reply::Int(n) - }) - .await + Reply::Int(keyspace_stats(engine).await.keys.min(i64::MAX as u64) as i64) } pub async fn flushall(engine: &Arc) -> Reply { diff --git a/crates/marekvs-engine/src/lib.rs b/crates/marekvs-engine/src/lib.rs index d66b642..d012caa 100644 --- a/crates/marekvs-engine/src/lib.rs +++ b/crates/marekvs-engine/src/lib.rs @@ -281,6 +281,7 @@ impl Engine { // 40-hex run id from boot time + node id (unique enough per boot; // Redis semantics only need it stable for the process lifetime). let metrics = metrics::Metrics::new(store.node_id); + store.maintenance.metrics.register(&metrics.registry); let now = store::now_ms(); let run_id = format!( "{:016x}{:016x}{:08x}", diff --git a/crates/marekvs-engine/src/metrics.rs b/crates/marekvs-engine/src/metrics.rs index 33116ea..d797297 100644 --- a/crates/marekvs-engine/src/metrics.rs +++ b/crates/marekvs-engine/src/metrics.rs @@ -109,11 +109,17 @@ pub struct Metrics { /// Range-delete records committed to the `data` CF, and the durable /// fragments they still cost after flush and compaction have clipped them. /// - /// One cold-partition purge is one range delete, so `db_range_deletes` - /// tracks `cold_purged_partitions_total`. `db_range_fragments` is the one - /// to alert on: fragments growing without bound mean purges are outrunning - /// compaction's ability to retire them, and every read pays for the - /// unretired spans. + /// These CF counters cover committed range deletes and catalogued SST + /// fragments. They exclude live memtable spans; the DB range_memtable + /// and range_fragment_cache/retained metrics expose that memory separately. + pub cold_purge_empty_skips_total: IntCounter, + pub cold_purge_stale_proofs_total: IntCounter, + pub db_range_memtable_spans: IntGauge, + pub db_range_memtable_bytes: IntGauge, + pub db_range_fragment_cache_bytes: IntGauge, + pub db_range_fragment_retained_bytes: IntGauge, + pub db_range_fragment_cache_builds: IntGauge, + pub db_range_fragment_cache_hits: IntGauge, pub db_range_deletes: IntGauge, pub db_range_fragments: IntGauge, /// Tables delete-only excise has retired, and the bytes they held — space @@ -402,6 +408,14 @@ impl Metrics { "marekvs_db_compaction_write_stopped", "1 while client write commands are refused (compaction backlog above high-water)" ), + cold_purge_empty_skips_total: counter!(registry, "marekvs_cold_purge_empty_skips_total", "Complete empty cold-partition probes skipped without issuing range deletes"), + cold_purge_stale_proofs_total: counter!(registry, "marekvs_cold_purge_stale_proofs_total", "Cold cleanup attempts rejected due to stale generation or ownership proof"), + db_range_memtable_spans: gauge!(registry, "marekvs_db_range_memtable_spans", "Live range-delete spans retained by memtables"), + db_range_memtable_bytes: gauge!(registry, "marekvs_db_range_memtable_bytes", "Bytes retained by live memtable range-delete spans"), + db_range_fragment_cache_bytes: gauge!(registry, "marekvs_db_range_fragment_cache_bytes", "Live current range-fragment snapshot bytes"), + db_range_fragment_retained_bytes: gauge!(registry, "marekvs_db_range_fragment_retained_bytes", "All live retained range-fragment snapshots including superseded generations"), + db_range_fragment_cache_builds: gauge!(registry, "marekvs_db_range_fragment_cache_builds", "Range-fragment snapshot builds across currently live memtable sets"), + db_range_fragment_cache_hits: gauge!(registry, "marekvs_db_range_fragment_cache_hits", "Range-fragment snapshot hits across currently live memtable sets"), db_range_deletes: gauge!( registry, "marekvs_db_range_deletes", diff --git a/crates/marekvs-engine/src/store.rs b/crates/marekvs-engine/src/store.rs index b866cbf..e0e560e 100644 --- a/crates/marekvs-engine/src/store.rs +++ b/crates/marekvs-engine/src/store.rs @@ -5,6 +5,8 @@ //! atomic read-modify-write without locks. The tokio side submits closures //! and awaits a oneshot. +pub mod expiry; + use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -112,6 +114,9 @@ type Job = Box; /// Everything a storage job can touch. One per shard thread. pub struct ShardCtx { + /// Raw access for reads/maintenance. Every data mutation must execute + /// through run/run_key on its owning shard, including direct transactions. + /// Off-shard raw data writers violate expiry and cold-purge serialization. pub db: DB, pub data: Arc, pub meta: Arc, @@ -122,6 +127,8 @@ pub struct ShardCtx { /// can never collide with the dead incarnation's escrow records. pub epoch: u64, pub shard: usize, + pub shard_count: usize, + pub maintenance: Arc, /// Pop-front cursors: collection scan prefix → internal key of the last /// popped element. Pops (SPOP/ZPOPMIN) leave element tombstones at the /// scan front, so pop #k would otherwise skip k dead records — the LSM @@ -314,6 +321,9 @@ pub fn scan_from( } pub struct Store { + /// Raw access for reads/maintenance. Every data mutation must execute + /// through run/run_key on its owning shard, including direct transactions. + /// Off-shard raw data writers violate expiry and cold-purge serialization. pub db: DB, pub data: Arc, pub meta: Arc, @@ -327,6 +337,8 @@ pub struct Store { pub epoch_fresh: bool, /// Data directory, kept for filesystem usage stats (disk-full guard). pub data_dir: std::path::PathBuf, + pub maintenance: Arc, + replication_observer: Arc>>, shards: Vec>, shard_handles: Vec>, } @@ -518,7 +530,6 @@ impl Store { None => db.create_column_family("meta", cf_config())?, }; let hlc = Arc::new(Hlc::new()); - SHARD_TOTAL.store(cfg.shard_threads, std::sync::atomic::Ordering::Relaxed); // Store epoch: minted only when absent; persisted in the meta CF so // it is stable across restarts of the same data directory. @@ -555,6 +566,29 @@ impl Store { let mut shards = Vec::with_capacity(cfg.shard_threads); let mut shard_handles = Vec::with_capacity(cfg.shard_threads); + let maintenance = Arc::new(expiry::MaintenanceState::new()); + let replication_observer = Arc::new(parking_lot::RwLock::new(None::)); + let observed = maintenance.clone(); + let replication = replication_observer.clone(); + // Permanent local observer. A replacing/suppressed replication hook + // never suppresses maintenance, including budget and index commits. + // Hooks run after visibility but before commit returns. All production + // data writers and proof publication execute on the owning shard, + // so no proof can pass through the visible-before-hook gap. Public DB + // handles must not be used to introduce off-shard data writers. + data.set_commit_hook(Some(Arc::new(move |seq, ops| { + #[cfg(debug_assertions)] + observed.before_observer(); + for op in ops { + if op.key.len() >= 2 { + observed.invalidate(u16::from_be_bytes([op.key[0], op.key[1]])); + } + } + let callback = replication.read().clone(); + if let Some(callback) = callback { + callback(seq, ops); + } + }))); for shard in 0..cfg.shard_threads { let (tx, rx): (Sender, Receiver) = crossbeam_channel::bounded(4096); let ctx = ShardCtx { @@ -565,6 +599,8 @@ impl Store { node_id: cfg.node_id, epoch, shard, + shard_count: cfg.shard_threads, + maintenance: maintenance.clone(), pop_hints: std::cell::RefCell::new(std::collections::HashMap::new()), }; let handle = std::thread::Builder::new() @@ -583,6 +619,8 @@ impl Store { epoch, epoch_fresh, data_dir: std::path::PathBuf::from(&cfg.data_dir), + maintenance, + replication_observer, shards, shard_handles, })) @@ -634,9 +672,13 @@ impl Store { let _ = self.shards[self.shard_of(pid)].send(Box::new(f)); } - /// Install the post-commit hook on the data CF (replication feed). + pub fn partition_generation(&self, pid: Pid) -> u64 { + self.maintenance.generation(pid) + } + + /// Replace the replication observer; local maintenance remains installed. pub fn set_commit_hook(&self, hook: Option) { - self.data.set_commit_hook(hook); + *self.replication_observer.write() = hook; } } @@ -669,94 +711,27 @@ pub fn with_inline_ctx(shard_idx: usize, f: impl FnOnce(&ShardCtx) -> T) -> O fn shard_loop(ctx: ShardCtx, rx: Receiver) { let ctx = std::rc::Rc::new(ctx); CURRENT_SHARD_CTX.with(|c| *c.borrow_mut() = Some(ctx.clone())); - // Expiry sweeping (design/01): incremental cursor walk between jobs. - let mut sweep_cursor: Vec = Vec::new(); + let mut scheduler = + expiry::ExpiryScheduler::new(ctx.maintenance.clone(), ctx.shard, ctx.shard_count); + let mut next_poll = std::time::Instant::now() + scheduler.next_wait(now_ms()); loop { - match rx.recv_timeout(Duration::from_millis(100)) { + match rx.recv_timeout(next_poll.saturating_duration_since(std::time::Instant::now())) { Ok(job) => job(&ctx), - Err(crossbeam_channel::RecvTimeoutError::Timeout) => { - if let Err(e) = sweep_expired(&ctx, &mut sweep_cursor, 128) { - // Passive ondaDB TTL is still the backstop, so a failed - // sweep delays active expiry rather than losing it. - tracing::warn!(shard = ctx.shard, error = %e, "expiry sweep incomplete"); - } - } + Err(crossbeam_channel::RecvTimeoutError::Timeout) => {} Err(crossbeam_channel::RecvTimeoutError::Disconnected) => return, } - } -} - -/// Walk up to `budget` records from the cursor; write expiry tombstones for -/// records whose TTL deadline passed. Expiry tombstone HLC = deadline<<16 so -/// every node converges on the identical tombstone (design/05). -fn sweep_expired( - ctx: &ShardCtx, - cursor: &mut Vec, - budget: usize, -) -> Result<(), ScanIncomplete> { - let now = now_ms(); - let mut expired: Vec<(Vec, Vec)> = Vec::new(); - let outcome; - { - let txn = ctx.db.begin(); - let mut it = txn.new_iterator(&ctx.data); - if cursor.is_empty() { - it.seek_to_first(); - } else { - it.seek(cursor); - } - let mut n = 0; - while it.valid() && n < budget { - // Shard ownership check: this thread only touches its own pids. - if let Some(parsed) = ikey::parse(it.key()) { - // Budget records (tag 'b') NEVER expire generically: token - // deadlines live in the payload and only the issuing node - // may fold them — a replica-sweeper tombstone here would - // destroy pre-fold state the issuer's escrow credit needs - // (design/13). They carry no envelope TTL today; the skip is - // explicit so a future TTL use can't reintroduce the trace. - if parsed.tag != ikey::Tag::Budget as u8 - && parsed.pid as usize % shard_total(ctx) == ctx.shard - { - if let Some((env, pay)) = Envelope::decode(it.value()) { - if !env.is_tombstone() && env.is_expired(now) { - expired.push((it.key().to_vec(), expiry_tombstone(&env, pay))); - } - } - } + if std::time::Instant::now() >= next_poll { + if let Err(e) = scheduler.poll(&ctx, now_ms(), 128, 8, Duration::from_millis(2)) { + tracing::warn!(shard = ctx.shard, error = %e, "expiry discovery incomplete"); } - n += 1; - it.next(); - } - outcome = scan_outcome(&it); - // A failed walk leaves the cursor where it was so the next tick retries - // the same ground. Treating the invalid iterator as "reached the end" - // would rewind the sweep to the start of the keyspace on every error. - if outcome.is_ok() { - *cursor = if it.valid() { - it.key().to_vec() - } else { - Vec::new() - }; + // Never catch up unbounded work after a pause, and never let an + // always-ready foreground queue starve due maintenance. + next_poll = std::time::Instant::now() + + scheduler.next_wait(now_ms()).max(Duration::from_millis(1)); } } - for (k, v) in expired { - // Normal merged write → commit hook fires → expiry replicates. - // These were genuinely observed, so they are written even if the walk - // was cut short. - write_merged(ctx, &k, &v); - } - outcome } -fn shard_total(_ctx: &ShardCtx) -> usize { - // Each ShardCtx knows only its index; total is implied by construction. - // Stored once at startup in a global to keep ShardCtx Copy-free. - SHARD_TOTAL.load(std::sync::atomic::Ordering::Relaxed) -} -pub(crate) static SHARD_TOTAL: std::sync::atomic::AtomicUsize = - std::sync::atomic::AtomicUsize::new(1); - fn expiry_tombstone(env: &Envelope, payload: &[u8]) -> Vec { let rtype = env.rtype(); if rtype.is_or_element() { @@ -874,7 +849,24 @@ pub fn del_raw(ctx: &ShardCtx, ikey: &[u8]) { /// The scan-and-tombstone predecessor depended on a `suppress_commit_hook()` /// guard for that; here it is a property of the operation instead of something /// a future refactor has to remember. +pub fn partition_has_data(ctx: &ShardCtx, pid: Pid) -> Result { + let txn = ctx.db.begin(); + let lower = pid.to_be_bytes(); + let upper = pid + .checked_add(1) + .ok_or_else(|| ScanIncomplete(format!("partition {pid} has no upper bound")))? + .to_be_bytes(); + let mut it = bounded_iter(&txn, ctx, &lower, Some(&upper)); + it.seek_to_first(); + scan_outcome(&it)?; + Ok(it.valid()) +} + pub fn delete_partition_range(ctx: &ShardCtx, pid: Pid) -> anyhow::Result<()> { + if !partition_has_data(ctx, pid)? { + return Ok(()); + } + // `Pid` is u16 and `ikey::PARTITIONS` is 4096, so `pid + 1` cannot overflow // a u16 in practice — but compute in u32 and encode two bytes anyway, // because a 4-byte end bound would sort BELOW every 2-byte key and silently @@ -883,7 +875,9 @@ pub fn delete_partition_range(ctx: &ShardCtx, pid: Pid) -> anyhow::Result<()> { .map_err(|_| anyhow::anyhow!("partition {pid} has no representable upper bound"))?; ctx.db .delete_range(&ctx.data, &pid.to_be_bytes(), &end.to_be_bytes()) - .map_err(|e| anyhow::anyhow!("range delete for partition {pid} failed: {e:?}")) + .map_err(|e| anyhow::anyhow!("range delete for partition {pid} failed: {e:?}"))?; + ctx.maintenance.invalidate(pid); + Ok(()) } pub(crate) fn onda_ttl_for(value: &[u8]) -> Duration { @@ -956,9 +950,29 @@ pub fn put_many_lww(ctx: &ShardCtx, items: &[(Vec, Vec)]) { /// Merge `incoming` into whatever is stored under `ikey`. /// Returns true when the stored bytes changed. pub fn write_merged(ctx: &ShardCtx, ikey: &[u8], incoming: &[u8]) -> bool { - let changed = match get_raw(ctx, ikey) { + match write_merged_checked(ctx, ikey, incoming) { + Ok(changed) => changed, + Err(e) => { + tracing::error!(?e, "merged write failed"); + false + } + } +} + +fn write_merged_checked(ctx: &ShardCtx, ikey: &[u8], incoming: &[u8]) -> anyhow::Result { + let local = match ctx.db.get(&ctx.data, ikey) { + Ok(value) => Some(value), + Err(ondadb::OndaError::NotFound) => None, + Err(e) => return Err(e.into()), + }; + let changed = match local { None => { - put_raw(ctx, ikey, incoming); + ctx.db.put( + &ctx.data, + ikey, + incoming, + onda_ttl_for_keyed(ikey, incoming), + )?; true } Some(local) => { @@ -976,7 +990,8 @@ pub fn write_merged(ctx: &ShardCtx, ikey: &[u8], incoming: &[u8]) -> bool { MergeOutcome::KeepLocal => false, _ => { let winner = resolve(&local, incoming, &outcome); - put_raw(ctx, ikey, winner); + ctx.db + .put(&ctx.data, ikey, winner, onda_ttl_for_keyed(ikey, winner))?; true } } @@ -997,7 +1012,7 @@ pub fn write_merged(ctx: &ShardCtx, ikey: &[u8], incoming: &[u8]) -> bool { } } } - changed + Ok(changed) } /// Collection head lookup: (envelope, ctype, del_hlc). diff --git a/crates/marekvs-engine/src/store/expiry.rs b/crates/marekvs-engine/src/store/expiry.rs new file mode 100644 index 0000000..6ed70b4 --- /dev/null +++ b/crates/marekvs-engine/src/store/expiry.rs @@ -0,0 +1,464 @@ +//! Advisory, generation-checked TTL discovery. All polling and data writes run +//! on the owning shard. Iterator construction and one seek/next cannot be +//! preempted; the elapsed budget is checked between those storage operations. +use super::{bounded_iter, expiry_tombstone, scan_outcome, write_merged_checked, ShardCtx}; +use marekvs_core::{ + envelope::Envelope, + ikey::{self, Pid}, +}; +use prometheus::{Histogram, HistogramOpts, IntCounter, Registry}; +use std::sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, +}; +use std::time::{Duration, Instant}; + +#[derive(Clone)] +pub struct ExpiryMetrics { + pub polls: IntCounter, + pub due_partitions: IntCounter, + pub discovery_partitions: IntCounter, + pub iterator_opens: IntCounter, + pub records_visited: IntCounter, + pub partitions_completed: IntCounter, + pub tombstones_written: IntCounter, + pub incomplete_scans: IntCounter, + pub poll_seconds: Histogram, + pub iterator_seconds: Histogram, +} +impl ExpiryMetrics { + fn new() -> Self { + let counter = |name, help| IntCounter::new(name, help).expect("expiry metric"); + Self { + due_partitions: counter( + "marekvs_expiry_due_partitions_total", + "Partition quanta scanned for a known due deadline", + ), + discovery_partitions: counter( + "marekvs_expiry_discovery_partitions_total", + "Partition quanta scanned for initial or invalidated discovery", + ), + polls: counter( + "marekvs_expiry_passes_total", + "Bounded expiry maintenance polls", + ), + iterator_opens: counter( + "marekvs_expiry_iterators_total", + "Partition iterators opened for expiry discovery", + ), + records_visited: counter( + "marekvs_expiry_records_total", + "Surfaced records visited by expiry discovery", + ), + partitions_completed: counter( + "marekvs_expiry_partitions_completed_total", + "Complete generation-stable partition discoveries", + ), + tombstones_written: counter( + "marekvs_expiry_tombstones_total", + "Successfully committed active-expiry tombstones", + ), + incomplete_scans: counter( + "marekvs_expiry_scan_errors_total", + "Failed expiry scans retried without publishing absence", + ), + poll_seconds: Histogram::with_opts(HistogramOpts::new( + "marekvs_expiry_tick_seconds", + "Actual bounded expiry poll duration", + )) + .unwrap(), + iterator_seconds: Histogram::with_opts(HistogramOpts::new( + "marekvs_expiry_iterator_seconds", + "Actual iterator construction and scan quantum duration", + )) + .unwrap(), + } + } + pub fn register(&self, registry: &Registry) { + for counter in [ + &self.polls, + &self.due_partitions, + &self.discovery_partitions, + &self.iterator_opens, + &self.records_visited, + &self.partitions_completed, + &self.tombstones_written, + &self.incomplete_scans, + ] { + registry + .register(Box::new(counter.clone())) + .expect("register expiry counter"); + } + registry + .register(Box::new(self.poll_seconds.clone())) + .unwrap(); + registry + .register(Box::new(self.iterator_seconds.clone())) + .unwrap(); + } +} + +pub struct MaintenanceState { + generations: Vec, + dirty: Vec, + pub metrics: ExpiryMetrics, + #[cfg(debug_assertions)] + observer_barrier: parking_lot::Mutex>>, +} +impl Default for MaintenanceState { + fn default() -> Self { + Self::new() + } +} +impl MaintenanceState { + pub fn new() -> Self { + Self { + generations: (0..ikey::PARTITIONS).map(|_| AtomicU64::new(0)).collect(), + dirty: (0..ikey::PARTITIONS) + .map(|_| AtomicBool::new(true)) + .collect(), + metrics: ExpiryMetrics::new(), + #[cfg(debug_assertions)] + observer_barrier: parking_lot::Mutex::new(None), + } + } + /// Deterministic test rendezvous after visibility but before invalidation. + #[cfg(debug_assertions)] + #[doc(hidden)] + pub fn set_observer_barrier_for_tests(&self, barrier: Option>) { + *self.observer_barrier.lock() = barrier; + } + #[cfg(debug_assertions)] + pub(super) fn before_observer(&self) { + let barrier = self.observer_barrier.lock().clone(); + if let Some(barrier) = barrier { + barrier(); + } + } + pub fn generation(&self, pid: Pid) -> u64 { + self.generations[pid as usize].load(Ordering::Acquire) + } + /// Point observer and explicit successful range commits share this path. + /// fetch_add is essential: hook invocation order is not commit order. + pub fn invalidate(&self, pid: Pid) { + if let Some(generation) = self.generations.get(pid as usize) { + generation.fetch_add(1, Ordering::AcqRel); + self.dirty[pid as usize].store(true, Ordering::Release); + } + } +} + +#[derive(Debug, Default)] +pub struct ExpiryProgress { + pub records_visited: usize, + pub partitions_completed: usize, + pub iterator_opens: usize, + pub tombstones_written: usize, + pub has_more_work: bool, +} +#[derive(Debug)] +pub struct ScanIncomplete { + pub pid: Pid, + pub source: super::ScanIncomplete, +} +impl std::fmt::Display for ScanIncomplete { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "partition {}: {}", self.pid, self.source) + } +} +impl std::error::Error for ScanIncomplete {} + +#[derive(Debug)] +enum Discovery { + Unknown, + Scanning { + generation: u64, + cursor: Vec, + deadline_ms: u64, + }, + NoTtl { + generation: u64, + }, + Due { + generation: u64, + deadline_ms: u64, + }, +} +struct Partition { + pid: Pid, + state: Discovery, + retry_at: Option, +} +pub struct ExpiryScheduler { + shared: Arc, + partitions: Vec, + next: usize, + urgent_next: usize, + prefer_urgent: bool, +} +impl ExpiryScheduler { + pub fn new(shared: Arc, shard_index: usize, shard_count: usize) -> Self { + assert!(shard_count > 0 && shard_index < shard_count); + Self { + shared, + partitions: (shard_index..ikey::PARTITIONS as usize) + .step_by(shard_count) + .map(|pid| Partition { + pid: pid as Pid, + state: Discovery::Unknown, + retry_at: None, + }) + .collect(), + next: 0, + urgent_next: 0, + prefer_urgent: true, + } + } + pub fn invalidate(&mut self, pid: Pid) { + self.shared.invalidate(pid); + } + pub fn next_wait(&self, now_ms: u64) -> Duration { + let mut wait = Duration::from_secs(1); + for p in &self.partitions { + let generation = self.shared.generation(p.pid); + let armed = match p.state { + Discovery::NoTtl { generation: g } if g == generation => continue, + Discovery::Due { + generation: g, + deadline_ms, + } if g == generation => Duration::from_millis(deadline_ms.saturating_sub(now_ms)), + _ => Duration::from_millis(100), + }; + let armed = p.retry_at.map_or(armed, |at| { + armed.max(at.saturating_duration_since(Instant::now())) + }); + wait = wait.min(armed); + } + wait + } + pub fn poll( + &mut self, + ctx: &ShardCtx, + now_ms: u64, + record_budget: usize, + partition_budget: usize, + elapsed_budget: Duration, + ) -> Result { + let metrics = self.shared.metrics.clone(); + metrics.polls.inc(); + let _timer = metrics.poll_seconds.start_timer(); + let started = Instant::now(); + let mut progress = ExpiryProgress::default(); + let mut transitions = 0; + let mut failure = None; + // Due deadlines and mutations of parked partitions must not wait for + // the initial 4096-partition discovery rotation. Rotate fairly within + // each class, visiting each partition at most once in this poll. + let mut urgent = Vec::new(); + let mut ordinary = Vec::new(); + for offset in 0..self.partitions.len() { + let i = (self.next + offset) % self.partitions.len(); + let p = &self.partitions[i]; + if p.retry_at.is_some_and(|at| at > Instant::now()) { + continue; + } + let generation = self.shared.generation(p.pid); + let priority = match p.state { + Discovery::Due { + generation: g, + deadline_ms, + } => g != generation || deadline_ms <= now_ms, + Discovery::NoTtl { generation: g } => g != generation, + _ => false, + }; + if priority { + urgent.push(i); + } else if matches!(p.state, Discovery::Unknown | Discovery::Scanning { .. }) { + ordinary.push(i); + } + } + urgent.sort_by_key(|i| { + (i + self.partitions.len() - self.urgent_next) % self.partitions.len() + }); + let mut urgent = urgent.into_iter().peekable(); + let mut ordinary = ordinary.into_iter().peekable(); + while transitions < partition_budget + && progress.records_visited < record_budget + && (transitions == 0 || started.elapsed() < elapsed_budget) + { + let use_urgent = + (self.prefer_urgent && urgent.peek().is_some()) || ordinary.peek().is_none(); + let Some(i) = (if use_urgent { + urgent.next() + } else { + ordinary.next() + }) else { + break; + }; + self.prefer_urgent = !use_urgent; + if use_urgent { + self.urgent_next = (i + 1) % self.partitions.len(); + } else { + self.next = (i + 1) % self.partitions.len(); + } + let p = &mut self.partitions[i]; + if p.retry_at.is_some_and(|at| at > Instant::now()) { + continue; + } + let generation = self.shared.generation(p.pid); + let due = matches!(p.state, Discovery::Due { generation: g, deadline_ms } if g == generation && deadline_ms <= now_ms); + match &p.state { + Discovery::NoTtl { generation: g } if *g == generation => continue, + Discovery::Due { + generation: g, + deadline_ms, + } if *g == generation && *deadline_ms > now_ms => continue, + Discovery::Scanning { generation: g, .. } if *g == generation => {} + _ => { + p.state = Discovery::Scanning { + generation, + cursor: Vec::new(), + deadline_ms: 0, + } + } + } + // Safe because this is the only shard that can write this pid. + // Generation checks remain authoritative, so even a late dirty + // store cannot erase or manufacture a stable proof. + self.shared.dirty[p.pid as usize].swap(false, Ordering::AcqRel); + transitions += 1; + if due { + metrics.due_partitions.inc(); + } else { + metrics.discovery_partitions.inc(); + } + let Discovery::Scanning { + cursor, + deadline_ms, + generation: scan_generation, + } = &mut p.state + else { + unreachable!() + }; + let iterator_timer = metrics.iterator_seconds.start_timer(); + let txn = ctx.db.begin(); + let lower = p.pid.to_be_bytes(); + let upper = (p.pid + 1).to_be_bytes(); + let mut it = bounded_iter(&txn, ctx, &lower, Some(&upper)); + progress.iterator_opens += 1; + metrics.iterator_opens.inc(); + if cursor.is_empty() { + it.seek_to_first(); + } else { + it.seek(cursor); + } + let mut expected_generation = generation; + let mut write_error = None; + let mut quantum = 0; + while it.valid() + && progress.records_visited < record_budget + && quantum < 16 + && (quantum == 0 || started.elapsed() < elapsed_budget) + { + if let Some(parsed) = ikey::parse(it.key()) { + debug_assert_eq!(parsed.pid, p.pid); + if parsed.tag != ikey::Tag::Budget as u8 { + if let Some((env, pay)) = Envelope::decode(it.value()) { + if !env.is_tombstone() && env.ttl_deadline_ms > 0 { + if env.is_expired(now_ms) { + // Commit before advancing the cursor. The + // next budget check occurs after this one + // non-preemptible record, never after an + // unbounded drain of synchronous commits. + let tombstone = expiry_tombstone(&env, pay); + match write_merged_checked(ctx, it.key(), &tombstone) { + Ok(true) => { + progress.tombstones_written += 1; + metrics.tombstones_written.inc(); + expected_generation = + expected_generation.wrapping_add(1); + } + Ok(false) => {} + Err(error) => { + write_error = + Some(super::ScanIncomplete(error.to_string())); + } + } + } else if *deadline_ms == 0 || env.ttl_deadline_ms < *deadline_ms { + *deadline_ms = env.ttl_deadline_ms; + } + } + } + } + } + quantum += 1; + progress.records_visited += 1; + metrics.records_visited.inc(); + if write_error.is_some() { + break; + } + it.next(); + } + let outcome = scan_outcome(&it); + let complete = !it.valid(); + let next_cursor = if complete { + Vec::new() + } else { + it.key().to_vec() + }; + drop(it); + drop(txn); + drop(iterator_timer); + let outcome = if let Some(error) = write_error { + Err(error) + } else { + outcome + }; + // The only intervening writes on this owning shard are the + // successful one-record commits above. Account for each exact + // observer increment; any other generation change discards proof. + *scan_generation = expected_generation; + match outcome { + Err(source) => { + p.state = Discovery::Unknown; + p.retry_at = Some(Instant::now() + Duration::from_millis(100)); + metrics.incomplete_scans.inc(); + failure = Some(ScanIncomplete { pid: p.pid, source }); + } + Ok(()) if expected_generation != self.shared.generation(p.pid) => { + p.state = Discovery::Unknown + } + Ok(()) if complete => { + let deadline = *deadline_ms; + p.state = if deadline == 0 { + Discovery::NoTtl { + generation: expected_generation, + } + } else { + Discovery::Due { + generation: expected_generation, + deadline_ms: deadline, + } + }; + p.retry_at = None; + progress.partitions_completed += 1; + metrics.partitions_completed.inc(); + } + Ok(()) => *cursor = next_cursor, + } + } + progress.has_more_work = self.partitions.iter().any(|p| match p.state { + Discovery::NoTtl { generation } => generation != self.shared.generation(p.pid), + Discovery::Due { + generation, + deadline_ms, + } => generation != self.shared.generation(p.pid) || deadline_ms <= now_ms, + _ => true, + }); + if let Some(error) = failure { + Err(error) + } else { + Ok(progress) + } + } +} diff --git a/crates/marekvs-engine/tests/expiry_maintenance.rs b/crates/marekvs-engine/tests/expiry_maintenance.rs new file mode 100644 index 0000000..9679d4e --- /dev/null +++ b/crates/marekvs-engine/tests/expiry_maintenance.rs @@ -0,0 +1,487 @@ +use marekvs_core::{ + envelope::{Envelope, RecordType}, + ikey, +}; +use marekvs_engine::store::{self, expiry::ExpiryScheduler, ShardCtx, Store, StoreConfig}; +use std::{sync::Arc, time::Duration}; + +fn open(dir: &tempfile::TempDir, shards: usize) -> Arc { + Store::open(&StoreConfig { + data_dir: dir.path().to_string_lossy().into_owned(), + node_id: 1, + shard_threads: shards, + sync_mode: ondadb::SyncMode::Interval, + }) + .unwrap() +} +fn key(pid: u16, suffix: &[u8]) -> Vec { + let mut k = ikey::string_key(suffix); + k[..2].copy_from_slice(&pid.to_be_bytes()); + k +} +fn put(ctx: &ShardCtx, pid: u16, suffix: &[u8], deadline: u64) { + store::put_raw( + ctx, + &key(pid, suffix), + &Envelope::new(RecordType::String, ctx.hlc.now(), 1) + .with_ttl(deadline) + .encode_with(b"value"), + ); +} +fn discover(s: &mut ExpiryScheduler, ctx: &ShardCtx, now: u64) -> usize { + let mut completed = 0; + for _ in 0..20_000 { + let progress = s.poll(ctx, now, 128, 8, Duration::from_secs(10)).unwrap(); + completed += progress.partitions_completed; + if !progress.has_more_work { + return completed; + } + } + panic!("discovery did not converge"); +} + +#[tokio::test] +async fn no_ttl_discovery_parks_for_sixty_seconds_and_shards_are_disjoint() { + for shards in [1, 2, 10] { + let dir = tempfile::tempdir().unwrap(); + let store = open(&dir, shards); + for shard in 0..shards { + store + .run(shard as u16, move |ctx| { + let now = store::now_ms(); + put(ctx, shard as u16, b"no-ttl", 0); + let mut s = + ExpiryScheduler::new(ctx.maintenance.clone(), ctx.shard, ctx.shard_count); + let first = s.poll(ctx, now, 128, 8, Duration::from_secs(10)).unwrap(); + assert_eq!(first.iterator_opens, 8); + assert_eq!(first.partitions_completed, 8); + let completed = 8 + discover(&mut s, ctx, now); + let count = (ikey::PARTITIONS as usize - 1 - shard) / shards + 1; + assert_eq!(completed, count); + for tick in 1..=600 { + let p = s + .poll(ctx, now + tick * 100, 128, 8, Duration::from_secs(10)) + .unwrap(); + assert_eq!(p.iterator_opens, 0); + assert_eq!(p.records_visited, 0); + assert!(!p.has_more_work); + assert_eq!(s.next_wait(now + tick * 100), Duration::from_secs(1)); + } + }) + .await; + } + } +} + +#[tokio::test] +async fn future_ttl_persist_extension_and_clock_jumps() { + let dir = tempfile::tempdir().unwrap(); + let store = open(&dir, 1); + store + .run(0, |ctx| { + let now = store::now_ms(); + put(ctx, 0, b"due", now + 60_000); + put(ctx, 0, b"persist", now + 60_000); + put(ctx, 0, b"extend", now + 60_000); + let mut s = ExpiryScheduler::new(ctx.maintenance.clone(), 0, 1); + discover(&mut s, ctx, now); + assert_eq!( + s.poll(ctx, now - 60_000, 128, 8, Duration::from_secs(10)) + .unwrap() + .iterator_opens, + 0 + ); + put(ctx, 0, b"persist", 0); + put(ctx, 0, b"extend", now + 180_000); + discover(&mut s, ctx, now); + discover(&mut s, ctx, now + 60_001); + assert!( + Envelope::decode(&store::get_raw(ctx, &key(0, b"due")).unwrap()) + .unwrap() + .0 + .is_tombstone() + ); + for k in [b"persist".as_slice(), b"extend"] { + assert!(!Envelope::decode(&store::get_raw(ctx, &key(0, k)).unwrap()) + .unwrap() + .0 + .is_tombstone()); + } + assert_eq!(ctx.maintenance.metrics.tombstones_written.get(), 1); + discover(&mut s, ctx, now + 180_001); + assert_eq!(ctx.maintenance.metrics.tombstones_written.get(), 2); + }) + .await; +} + +#[tokio::test] +async fn write_behind_cursor_invalidates_proof_and_busy_pid_rotates() { + let dir = tempfile::tempdir().unwrap(); + let store = open(&dir, 1); + store + .run(0, |ctx| { + let now = store::now_ms(); + for i in 0..80 { + put(ctx, 0, format!("k{i:04}").as_bytes(), 0); + } + put(ctx, 1, b"future", now + 60_000); + let mut s = ExpiryScheduler::new(ctx.maintenance.clone(), 0, 1); + let first = s.poll(ctx, now, 128, 8, Duration::from_secs(10)).unwrap(); + assert_eq!(first.records_visited, 17); + assert_eq!(first.partitions_completed, 7); + put(ctx, 0, b"a-behind", now + 60_000); + discover(&mut s, ctx, now); + discover(&mut s, ctx, now + 60_001); + assert_eq!(ctx.maintenance.metrics.tombstones_written.get(), 2); + }) + .await; +} + +#[tokio::test] +async fn observer_survives_replacement_and_suppression_and_range_mutations() { + let dir = tempfile::tempdir().unwrap(); + let store = open(&dir, 1); + store.set_commit_hook(Some(Arc::new(|_, _| {}))); + store.set_commit_hook(None); + store + .run(0, |ctx| { + let before = ctx.maintenance.generation(0); + { + let _guard = store::suppress_commit_hook(); + put(ctx, 0, b"suppressed", 0); + } + assert!(ctx.maintenance.generation(0) > before); + let before = ctx.maintenance.generation(0); + store::delete_partition_range(ctx, 0).unwrap(); + assert!(ctx.maintenance.generation(0) > before); + }) + .await; +} + +#[tokio::test] +async fn tiny_elapsed_budget_still_advances_one_storage_record() { + let dir = tempfile::tempdir().unwrap(); + let store = open(&dir, 1); + store + .run(0, |ctx| { + put(ctx, 0, b"a", 0); + put(ctx, 0, b"b", 0); + let mut s = ExpiryScheduler::new(ctx.maintenance.clone(), 0, 1); + let p = s + .poll(ctx, store::now_ms(), 128, 8, Duration::from_nanos(1)) + .unwrap(); + assert_eq!(p.iterator_opens, 1); + assert_eq!(p.records_visited, 1); + }) + .await; +} + +#[tokio::test] +async fn restart_rebuilds_unknown_state_and_replica_ttl_rearms_absence() { + let dir = tempfile::tempdir().unwrap(); + let store = open(&dir, 1); + let deadline = store::now_ms() + 60_000; + store + .run(0, move |ctx| { + let mut s = ExpiryScheduler::new(ctx.maintenance.clone(), 0, 1); + discover(&mut s, ctx, deadline - 60_000); + let _origin = store::set_apply_origin(2); + put(ctx, 0, b"replicated", deadline); + discover(&mut s, ctx, deadline - 1); + assert_eq!(s.next_wait(deadline - 1), Duration::from_millis(1)); + }) + .await; + drop(store); + let reopened = open(&dir, 1); + reopened + .run(0, move |ctx| { + let mut s = ExpiryScheduler::new(ctx.maintenance.clone(), 0, 1); + discover(&mut s, ctx, deadline + 1); + assert!( + Envelope::decode(&store::get_raw(ctx, &key(0, b"replicated")).unwrap()) + .unwrap() + .0 + .is_tombstone() + ); + }) + .await; +} + +#[tokio::test] +async fn busy_command_queue_does_not_starve_due_tombstones() { + let dir = tempfile::tempdir().unwrap(); + let store = open(&dir, 1); + store + .run(0, |ctx| put(ctx, 0, b"busy-due", store::now_ms() + 150)) + .await; + // Jobs remain queued across several deadlines; no receive timeout occurs. + let (tx, rx) = std::sync::mpsc::channel(); + for i in 0..300 { + let tx = tx.clone(); + store.spawn_on(0, move |ctx| { + std::thread::sleep(Duration::from_millis(2)); + if ctx.maintenance.metrics.tombstones_written.get() > 0 { + let _ = tx.send(i); + } + }); + } + let first = rx + .recv_timeout(Duration::from_secs(5)) + .expect("queued jobs starved expiry"); + assert!(first < 299, "expiry ran only after queue drained"); + store.run(0, |_| ()).await; +} + +#[tokio::test] +async fn callback_barrier_serializes_proof_and_preserves_generation() { + let dir = tempfile::tempdir().unwrap(); + let store = open(&dir, 1); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let release = std::sync::Mutex::new(release_rx); + store + .maintenance + .set_observer_barrier_for_tests(Some(Arc::new(move || { + entered_tx.send(()).unwrap(); + release.lock().unwrap().recv().unwrap(); + }))); + store.spawn_on(0, |ctx| put(ctx, 0, b"paused", store::now_ms() + 60_000)); + entered_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + // The commit is already visible to raw readers, but its writing shard is + // still inside the callback. No proof publication can overtake it. + assert!(store.db.get(&store.data, &key(0, b"paused")).is_ok()); + assert_eq!(store.partition_generation(0), 0); + let (done_tx, done_rx) = std::sync::mpsc::channel(); + store.spawn_on(0, move |ctx| { + let mut s = ExpiryScheduler::new(ctx.maintenance.clone(), 0, 1); + discover(&mut s, ctx, store::now_ms()); + done_tx.send(s.next_wait(store::now_ms())).unwrap(); + }); + assert!(done_rx.try_recv().is_err()); + release_tx.send(()).unwrap(); + assert!(done_rx.recv_timeout(Duration::from_secs(3)).is_ok()); + store.maintenance.set_observer_barrier_for_tests(None); +} + +#[tokio::test] +async fn member_ttl_keeps_remove_dots_and_budget_records_are_exempt() { + let dir = tempfile::tempdir().unwrap(); + let store = open(&dir, 1); + store + .run(0, |ctx| { + let now = store::now_ms(); + let mut member = ikey::set_member_key(b"set", b"member"); + member[..2].copy_from_slice(&0u16.to_be_bytes()); + let added = marekvs_core::merge::element_add_ttl( + RecordType::SetMember, + ctx.hlc.now(), + 1, + b"member", + now + 60_000, + ); + store::put_raw(ctx, &member, &added); + let mut budget = ikey::budget_slot_key(b"budget", 1, 1, 1); + budget[..2].copy_from_slice(&0u16.to_be_bytes()); + store::put_raw( + ctx, + &budget, + &Envelope::new(RecordType::String, ctx.hlc.now(), 1) + .with_ttl(now + 60_000) + .encode_with(b"budget-payload"), + ); + let mut s = ExpiryScheduler::new(ctx.maintenance.clone(), 0, 1); + discover(&mut s, ctx, now); + discover(&mut s, ctx, now + 60_001); + let removed = store::get_raw(ctx, &member).unwrap(); + let (env, payload) = Envelope::decode(&removed).unwrap(); + assert_eq!(env.hlc, (now + 60_000) << 16); + assert!(marekvs_core::merge::element_value(payload).is_none()); + // Replaying the original live add cannot resurrect an expired + // member: the remove retains its causal coverage. + store::write_merged(ctx, &member, &added); + let replayed = store::get_raw(ctx, &member).unwrap(); + assert!( + marekvs_core::merge::element_value(Envelope::decode(&replayed).unwrap().1) + .is_none() + ); + assert!(!Envelope::decode(&store::get_raw(ctx, &budget).unwrap()) + .unwrap() + .0 + .is_tombstone()); + assert_eq!(ctx.maintenance.metrics.tombstones_written.get(), 1); + }) + .await; +} + +#[tokio::test] +async fn native_ttl_filtering_is_not_counted_as_active_tombstoning() { + let dir = tempfile::tempdir().unwrap(); + let store = open(&dir, 1); + store + .run(0, |ctx| { + let now = store::now_ms(); + let record = Envelope::new(RecordType::String, ctx.hlc.now(), 1) + .with_ttl(now + 60_000) + .encode_with(b"passively-removed"); + ctx.db + .put( + &ctx.data, + &key(0, b"native"), + &record, + Duration::from_millis(1), + ) + .unwrap(); + std::thread::sleep(Duration::from_millis(5)); + let mut s = ExpiryScheduler::new(ctx.maintenance.clone(), 0, 1); + discover(&mut s, ctx, now + 60_001); + assert_eq!(ctx.maintenance.metrics.tombstones_written.get(), 0); + assert!(store::get_raw(ctx, &key(0, b"native")).is_none()); + }) + .await; +} + +fn klog_files(dir: &std::path::Path, files: &mut Vec) { + for entry in std::fs::read_dir(dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + klog_files(&path, files); + } else if path.extension().is_some_and(|ext| ext == "klog") { + files.push(path); + } + } +} + +#[tokio::test] +async fn unreadable_sst_keeps_discovery_armed_and_cannot_prove_empty_purge() { + let dir = tempfile::tempdir().unwrap(); + let store = open(&dir, 1); + let path = dir.path().to_path_buf(); + store + .run(0, move |ctx| { + put(ctx, 0, b"on-disk", 0); + ctx.db.flush_memtable(&ctx.data).unwrap(); + ctx.db.set_max_open_reader_bytes(1); + let mut files = Vec::new(); + klog_files(&path, &mut files); + assert!(!files.is_empty(), "fixture must have SST files"); + for file in &files { + std::fs::rename(file, file.with_extension("hidden")).unwrap(); + } + let mut s = ExpiryScheduler::new(ctx.maintenance.clone(), 0, 1); + let error = s + .poll(ctx, store::now_ms(), 128, 8, Duration::from_secs(10)) + .unwrap_err(); + assert_eq!(error.pid, 0); + assert!(s.next_wait(store::now_ms()) <= Duration::from_millis(101)); + assert!(store::partition_has_data(ctx, 0).is_err()); + let ranges = ctx.data.stats().range_deletes; + assert!(store::delete_partition_range(ctx, 0).is_err()); + assert_eq!(ctx.data.stats().range_deletes, ranges); + for file in &files { + std::fs::rename(file.with_extension("hidden"), file).unwrap(); + } + std::thread::sleep(Duration::from_millis(110)); + discover(&mut s, ctx, store::now_ms()); + assert_eq!( + s.poll(ctx, store::now_ms(), 128, 8, Duration::from_secs(10)) + .unwrap() + .iterator_opens, + 0 + ); + }) + .await; +} + +#[tokio::test] +async fn batched_multi_partition_and_derived_delete_ingress_invalidates_every_pid() { + let dir = tempfile::tempdir().unwrap(); + let store = open(&dir, 2); + store + .run(0, |ctx| { + let a = ctx.maintenance.generation(0); + let b = ctx.maintenance.generation(2); + let record = Envelope::new(RecordType::String, ctx.hlc.now(), 1).encode_with(b"batch"); + store::put_many_lww( + ctx, + &[(key(0, b"a"), record.clone()), (key(2, b"b"), record)], + ); + assert!(ctx.maintenance.generation(0) > a); + assert!(ctx.maintenance.generation(2) > b); + let before = ctx.maintenance.generation(2); + store::del_raw(ctx, &key(2, b"b")); + assert!(ctx.maintenance.generation(2) > before); + let before = ctx.maintenance.generation(0); + let _guard = store::suppress_commit_hook(); + let mut txn = ctx.db.begin(); + txn.put( + &ctx.data, + &key(0, b"direct-transaction"), + b"derived", + Duration::ZERO, + ) + .unwrap(); + txn.commit().unwrap(); + assert!(ctx.maintenance.generation(0) > before); + }) + .await; +} + +#[tokio::test] +async fn eight_continuously_dirty_parked_partitions_cannot_starve_unknown_discovery() { + let dir = tempfile::tempdir().unwrap(); + let store = open(&dir, 1); + store + .run(0, |ctx| { + let now = store::now_ms(); + let mut s = ExpiryScheduler::new(ctx.maintenance.clone(), 0, 1); + assert_eq!( + s.poll(ctx, now, 128, 8, Duration::from_secs(10)) + .unwrap() + .partitions_completed, + 8 + ); + put(ctx, 8, b"behind-busy", now + 60_000); + let initial_records = ctx.maintenance.metrics.records_visited.get(); + for _ in 0..8 { + for pid in 0..8 { + s.invalidate(pid); + } + s.poll(ctx, now, 128, 8, Duration::from_secs(10)).unwrap(); + } + assert!( + ctx.maintenance.metrics.records_visited.get() > initial_records, + "unknown pid 8 starved behind eight dirty parked pids" + ); + }) + .await; +} + +#[tokio::test] +async fn slow_expiry_commit_stops_at_elapsed_budget_and_retries_remaining_records() { + let dir = tempfile::tempdir().unwrap(); + let store = open(&dir, 1); + store + .run(0, |ctx| { + let now = store::now_ms(); + for i in 0..16 { + put(ctx, 0, format!("due-{i:02}").as_bytes(), now + 60_000); + } + let mut s = ExpiryScheduler::new(ctx.maintenance.clone(), 0, 1); + discover(&mut s, ctx, now); + ctx.maintenance + .set_observer_barrier_for_tests(Some(Arc::new(|| { + std::thread::sleep(Duration::from_millis(20)) + }))); + let progress = s + .poll(ctx, now + 60_001, 128, 8, Duration::from_millis(5)) + .unwrap(); + assert_eq!( + progress.tombstones_written, 1, + "one slow commit must not drain all observed expiries" + ); + ctx.maintenance.set_observer_barrier_for_tests(None); + discover(&mut s, ctx, now + 60_001); + assert_eq!(ctx.maintenance.metrics.tombstones_written.get(), 16); + }) + .await; +} diff --git a/crates/marekvs-engine/tests/info_expiry.rs b/crates/marekvs-engine/tests/info_expiry.rs new file mode 100644 index 0000000..446330a --- /dev/null +++ b/crates/marekvs-engine/tests/info_expiry.rs @@ -0,0 +1,92 @@ +use marekvs_core::{ + envelope::{head, Envelope, RecordType}, + ikey, +}; +use marekvs_engine::{ + cmd::{generic, server}, + reply::Reply, + store::{self, Store, StoreConfig}, + Engine, +}; + +#[tokio::test] +async fn info_counts_only_live_key_deadlines_and_obeys_sections() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::open(&StoreConfig { + data_dir: dir.path().to_string_lossy().into_owned(), + shard_threads: 2, + ..StoreConfig::default() + }) + .unwrap(); + let engine = Engine::new(store.clone()); + let deadline = store::now_ms() + 60_000; + for (key, ttl) in [ + (b"persistent".as_slice(), 0), + (b"expiring", deadline), + (b"expired", 1), + ] { + let key = key.to_vec(); + store + .run_key(&key.clone(), move |ctx| { + let value = store::new_lww(ctx, RecordType::String, b"value", ttl); + store::put_raw(ctx, &ikey::string_key(&key), &value); + }) + .await; + } + for (key, head_ttl, member_ttl) in [ + (b"head-ttl".as_slice(), deadline, 0), + (b"member-ttl", 0, deadline), + (b"deleted", 0, 0), + ] { + let key = key.to_vec(); + store + .run_key(&key.clone(), move |ctx| { + store::ensure_head(ctx, &key, head::CTYPE_HASH); + let raw = store::get_raw(ctx, &ikey::head_key(&key)).unwrap(); + let (mut env, pay) = Envelope::decode(&raw).unwrap(); + env.ttl_deadline_ms = head_ttl; + store::put_raw(ctx, &ikey::head_key(&key), &env.encode_with(pay)); + let value = marekvs_core::merge::element_add( + RecordType::HashField, + ctx.hlc.now(), + ctx.node_id, + b"value", + ); + let (mut env, pay) = Envelope::decode(&value).unwrap(); + env.ttl_deadline_ms = member_ttl; + store::put_raw( + ctx, + &ikey::hash_field_key(&key, b"field"), + &env.encode_with(pay), + ); + if key == b"deleted" { + generic::del_key(ctx, &key); + } + }) + .await; + } + let Reply::Bulk(info) = server::info(&engine, &[b"INFO".to_vec(), b"keyspace".to_vec()]).await + else { + panic!("expected INFO bulk"); + }; + let info = String::from_utf8(info).unwrap(); + assert!(info.contains("db0:keys=4,expires=2,avg_ttl="), "{info}"); + let avg: u64 = info + .split("avg_ttl=") + .nth(1) + .unwrap() + .trim() + .parse() + .unwrap(); + assert!( + (deadline.saturating_sub(store::now_ms())..=60_000).contains(&avg), + "{avg}" + ); + assert!(!info.contains("# Server")); + let Reply::Bulk(info) = server::info(&engine, &[b"INFO".to_vec(), b"server".to_vec()]).await + else { + panic!("expected INFO bulk"); + }; + assert!(!String::from_utf8(info).unwrap().contains("# Keyspace")); + assert!(matches!(server::dbsize(&engine).await, Reply::Int(4))); +} diff --git a/crates/marekvs-engine/tests/range_purge.rs b/crates/marekvs-engine/tests/range_purge.rs index 63a5e5f..0d55f3a 100644 --- a/crates/marekvs-engine/tests/range_purge.rs +++ b/crates/marekvs-engine/tests/range_purge.rs @@ -97,3 +97,34 @@ async fn records_written_after_a_purge_survive() { let got = s.run(0, |ctx| get_raw(ctx, &key_in(7, b"back"))).await; assert_eq!(got.as_deref(), Some(&b"after"[..])); } + +#[tokio::test] +async fn repeated_empty_purge_does_not_add_range_records() { + let dir = tempfile::tempdir().unwrap(); + let store = store(&dir); + store + .run(42, |ctx| put_raw(ctx, &key_in(42, b"one"), b"value")) + .await; + store + .run(42, |ctx| delete_partition_range(ctx, 42)) + .await + .unwrap(); + let first = store.data.stats().range_deletes; + for _ in 0..10 { + store + .run(42, |ctx| delete_partition_range(ctx, 42)) + .await + .unwrap(); + } + assert_eq!(store.data.stats().range_deletes, first); +} + +#[tokio::test] +async fn invalid_partition_upper_bound_returns_an_error() { + let dir = tempfile::tempdir().unwrap(); + let store = store(&dir); + assert!(store + .run(0, |ctx| delete_partition_range(ctx, u16::MAX)) + .await + .is_err()); +} diff --git a/crates/marekvs-proto/src/lib.rs b/crates/marekvs-proto/src/lib.rs index 442fefb..de40854 100644 --- a/crates/marekvs-proto/src/lib.rs +++ b/crates/marekvs-proto/src/lib.rs @@ -65,7 +65,10 @@ pub mod features { pub const COUNTER_FIELD: u32 = 1 << 0; /// Everything this build can do. - pub const ALL: u32 = COUNTER_FIELD; + /// Correlated, generation-bound evidence for node-local cold cleanup. + /// Legacy Merkle acknowledgements never authorize destructive cleanup. + pub const COLD_PROOF: u32 = 1 << 1; + pub const ALL: u32 = COUNTER_FIELD | COLD_PROOF; } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -228,6 +231,17 @@ pub enum PeerMsg { id: u64, result: Result, }, + // APPEND ONLY: retain every existing postcard discriminant. Send only + // after the peer advertises features::COLD_PROOF in Hello. + ColdProofRequest { + pid: Pid, + nonce: [u8; 24], + }, + ColdProofResponse { + pid: Pid, + nonce: [u8; 24], + root: u64, + }, } /// Token identity on the wire (client form is `gen-hlc-node-epoch` hex). @@ -412,9 +426,35 @@ mod proto_tests { fn announced_features_are_implemented() { assert_eq!( features::ALL, - features::COUNTER_FIELD, + features::COUNTER_FIELD | features::COLD_PROOF, "a capability bit was added to ALL — confirm the code that honours \ it landed in the same change" ); } } + +#[cfg(test)] +mod cold_proof_wire_tests { + use super::*; + #[test] + fn correlated_cold_proof_round_trips_without_changing_legacy_messages() { + for message in [ + PeerMsg::ColdProofRequest { + pid: 4095, + nonce: [7; 24], + }, + PeerMsg::ColdProofResponse { + pid: 4095, + nonce: [7; 24], + root: 123, + }, + PeerMsg::MerkleRoot { pid: 17, root: 123 }, + PeerMsg::MerkleRootMatch { pid: 17 }, + ] { + let bytes = encode(&message).unwrap(); + let (decoded, consumed) = decode(&bytes).unwrap().unwrap(); + assert_eq!(decoded, message); + assert_eq!(consumed, bytes.len()); + } + } +} diff --git a/crates/marekvs-repl/src/lib.rs b/crates/marekvs-repl/src/lib.rs index 489a1c7..f3b5f45 100644 --- a/crates/marekvs-repl/src/lib.rs +++ b/crates/marekvs-repl/src/lib.rs @@ -351,6 +351,154 @@ fn cold_ae_key(pid: Pid) -> Vec { format!("cold_ae:{pid}").into_bytes() } +/// Nonces contain a fresh OS-random boot identifier and an incrementing +/// sequence. No timestamp, reset generation, or restarted connection can +/// accidentally turn a delayed response into proof for a newer request. +#[derive(Clone, Copy, Debug)] +struct PendingColdProof { + peer: NodeId, + nonce: [u8; 24], + root: u64, + generation: u64, + view_epoch: u64, +} +struct ColdProofRequests { + boot: Option<[u8; 16]>, + sequence: u64, + pending: HashMap, +} +impl ColdProofRequests { + fn new() -> Self { + use std::io::Read; + let mut boot = [0u8; 16]; + let random = + std::fs::File::open("/dev/urandom").and_then(|mut file| file.read_exact(&mut boot)); + if let Err(error) = &random { + tracing::warn!(%error, "cold proof disabled: cannot obtain unique boot nonce"); + } + Self { + boot: random.ok().map(|_| boot), + sequence: 0, + pending: HashMap::new(), + } + } + fn issue( + &mut self, + pid: Pid, + peer: NodeId, + root: u64, + generation: u64, + view_epoch: u64, + ) -> Option<[u8; 24]> { + let boot = self.boot?; + self.sequence = self.sequence.checked_add(1)?; + let mut nonce = [0u8; 24]; + nonce[..16].copy_from_slice(&boot); + nonce[16..].copy_from_slice(&self.sequence.to_be_bytes()); + self.pending.insert( + pid, + PendingColdProof { + peer, + nonce, + root, + generation, + view_epoch, + }, + ); + Some(nonce) + } + fn take(&mut self, pid: Pid, peer: NodeId, nonce: [u8; 24]) -> Option { + if !self + .pending + .get(&pid) + .is_some_and(|pending| pending.peer == peer && pending.nonce == nonce) + { + return None; + } + self.pending.remove(&pid) + } +} +fn supports_cold_proof(features: Option) -> bool { + features.is_some_and(|bits| bits & marekvs_proto::features::COLD_PROOF != 0) +} +fn valid_message_partition(msg: &PeerMsg) -> bool { + let pid = match msg { + PeerMsg::InterestRenew { pid, .. } + | PeerMsg::MerkleRoot { pid, .. } + | PeerMsg::MerkleRootMatch { pid } + | PeerMsg::MerkleBuckets { pid, .. } + | PeerMsg::BucketKeys { pid, .. } + | PeerMsg::RepairOps { pid, .. } + | PeerMsg::RequestKeys { pid, .. } + | PeerMsg::BootstrapReq { pid } + | PeerMsg::BootstrapChunk { pid, .. } + | PeerMsg::BootstrapDone { pid, .. } + | PeerMsg::ColdProofRequest { pid, .. } + | PeerMsg::ColdProofResponse { pid, .. } => *pid, + _ => return true, + }; + (pid as usize) < marekvs_core::PARTITIONS as usize +} + +/// Runtime-only proof: generations restart at zero, so clean rounds must not +/// survive restart. The persisted ownership-loss age is separate. +#[derive(Debug, Clone, Copy)] +struct ColdEvidence { + generation: u64, + view_epoch: u64, + rounds: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PurgeOutcome { + Purged, + Empty, + StaleEvidence, + Ineligible, +} + +#[allow(clippy::too_many_arguments)] // one purge = one fully-specified eligibility check +fn purge_if_eligible( + ctx: &store::ShardCtx, + pid: Pid, + now_ms: u64, + delay_ms: u64, + need_rounds: u64, + view_epoch: u64, + eligible: bool, + evidence: Option<&mut ColdEvidence>, +) -> anyhow::Result { + if !eligible { + return Ok(PurgeOutcome::Ineligible); + } + let marked = match ctx.db.get(&ctx.meta, &cold_marker_key(pid)) { + Ok(v) if v.len() == 8 => u64::from_be_bytes(v.as_slice().try_into().unwrap()), + Ok(_) | Err(ondadb::OndaError::NotFound) => return Ok(PurgeOutcome::Ineligible), + Err(e) => return Err(e.into()), + }; + if now_ms.saturating_sub(marked) < delay_ms { + return Ok(PurgeOutcome::Ineligible); + } + // Proving emptiness is safe even without AE evidence; no deletion occurs. + // Keeping this ahead of proof validation also makes repeated empty checks + // cheap after a successful purge has correctly cleared its clean rounds. + if !store::partition_has_data(ctx, pid)? { + return Ok(PurgeOutcome::Empty); + } + let Some(proof) = evidence else { + return Ok(PurgeOutcome::StaleEvidence); + }; + if proof.generation != ctx.maintenance.generation(pid) || proof.view_epoch != view_epoch { + return Ok(PurgeOutcome::StaleEvidence); + } + if proof.rounds < need_rounds { + return Ok(PurgeOutcome::Ineligible); + } + store::delete_partition_range(ctx, pid)?; + proof.rounds = 0; + Ok(PurgeOutcome::Purged) +} + /// How long a partition must sit un-owned before its data may be purged. /// /// Generous by default: the data on an ex-owner is exactly what stranded-record @@ -510,7 +658,9 @@ pub struct ReplEngine { /// the commit hook's dirty set; quiescent partitions cost NO scan per /// round (previously the whole keyspace was re-hashed every ~5 s — /// linear I/O in data size, Tier-2 #7). - ae_roots: Mutex>, + ae_roots: Mutex>, + cold_evidence: Arc>>, + cold_requests: Arc>, /// Pids written since their root was last computed (set by the commit /// hook on every committed op, including AE repairs and rejoin drops). ae_dirty: Arc>>, @@ -647,6 +797,8 @@ impl ReplEngine { }), streams_served: Mutex::new(HashMap::new()), ae_roots: Mutex::new(HashMap::new()), + cold_evidence: Arc::new(Mutex::new(HashMap::new())), + cold_requests: Arc::new(Mutex::new(ColdProofRequests::new())), ae_dirty: ae_dirty.clone(), returning_member, started: Instant::now(), @@ -890,6 +1042,19 @@ impl ReplEngine { // what turns the masked interval back into free disk. m.db_range_deletes.set(cf.range_deletes as i64); m.db_range_fragments.set(cf.range_fragments as i64); + let ranges = self.store.db.stats(); + m.db_range_memtable_spans + .set(ranges.range_memtable_spans as i64); + m.db_range_memtable_bytes + .set(ranges.range_memtable_bytes as i64); + m.db_range_fragment_cache_bytes + .set(ranges.range_fragment_cache_bytes as i64); + m.db_range_fragment_retained_bytes + .set(ranges.range_fragment_retained_bytes as i64); + m.db_range_fragment_cache_builds + .set(ranges.range_fragment_cache_builds as i64); + m.db_range_fragment_cache_hits + .set(ranges.range_fragment_cache_hits as i64); m.db_excised_tables.set(cf.excised_tables as i64); m.db_excised_bytes.set(cf.excised_bytes as i64); m.db_periodic_compactions @@ -1459,53 +1624,104 @@ impl ReplEngine { /// minutes would otherwise reset its clock forever and never reclaim the /// disk, which is the leak this exists to close. async fn track_ownership_loss(&self) { - let owned: HashSet = self.cluster.owned_pids().into_iter().collect(); - self.store - .run(0, move |ctx| { - for pid in 0..marekvs_core::PARTITIONS as Pid { - let key = cold_marker_key(pid); - let marked = matches!(ctx.db.get(&ctx.meta, &key), Ok(v) if v.len() == 8); - if owned.contains(&pid) { - // Owned again: drop the marker AND the evidence count, - // so a later loss starts its delay and its clean-round - // tally from scratch. - if marked { - let _ = ctx.db.delete(&ctx.meta, &key); + for pid in 0..marekvs_core::PARTITIONS as Pid { + let cluster = self.cluster.clone(); + let evidence = self.cold_evidence.clone(); + self.store + .run(pid, move |ctx| { + cluster.with_view(|view| { + let key = cold_marker_key(pid); + let marked = match ctx.db.get(&ctx.meta, &key) { + Ok(v) => v.len() == 8, + Err(ondadb::OndaError::NotFound) => false, + Err(e) => { + tracing::warn!(pid, ?e, "cold marker read failed"); + return; + } + }; + if view.is_owner(pid, cluster.replicas_n, cluster.self_id) { + evidence.lock().remove(&pid); + if marked { + let _ = ctx.db.delete(&ctx.meta, &key); + } + let _ = ctx.db.delete(&ctx.meta, &cold_ae_key(pid)); + } else if !marked { + evidence.lock().remove(&pid); + let _ = ctx.db.put( + &ctx.meta, + &key, + &store::now_ms().to_be_bytes(), + Duration::ZERO, + ); let _ = ctx.db.delete(&ctx.meta, &cold_ae_key(pid)); } - } else if !marked { - let _ = ctx.db.put( - &ctx.meta, - &key, - &store::now_ms().to_be_bytes(), - Duration::ZERO, - ); - let _ = ctx.db.delete(&ctx.meta, &cold_ae_key(pid)); + }) + }) + .await; + } + } + + async fn offer_cold_proof(&self, pid: Pid, peer: NodeId) { + if !supports_cold_proof(self.mesh.peer_features(peer)) { + return; + } + let epoch = self.cluster.view().epoch; + let Ok((root, generation)) = self.partition_root_snapshot(pid).await else { + return; + }; + if root == 0 { + return; + } + let cluster = self.cluster.clone(); + let requests = self.cold_requests.clone(); + let nonce = self + .store + .run(pid, move |ctx| { + cluster.with_view(|view| { + if view.epoch != epoch + || ctx.maintenance.generation(pid) != generation + || view.is_owner(pid, cluster.replicas_n, cluster.self_id) + || !view.owners(pid, cluster.replicas_n).contains(&peer) + || !view.members.iter().any(|m| { + m.node == peer && m.phase == marekvs_cluster::NodePhase::Active + }) + { + return None; } - } + requests.lock().issue(pid, peer, root, generation, epoch) + }) }) .await; + if let Some(nonce) = nonce { + self.mesh + .send_ctl(peer, PeerMsg::ColdProofRequest { pid, nonce }); + } } - /// Record one clean stranded-AE exchange for a partition we no longer own: - /// an owner answered `MerkleRootMatch`, i.e. it holds exactly what we hold. - fn note_clean_cold_round(&self, pid: Pid) { - self.store.spawn_on(0, move |ctx| { - let key = cold_marker_key(pid); - // Only count while the partition is actually marked cold; a match - // for an owned pid says nothing about safe-to-purge. - if !matches!(ctx.db.get(&ctx.meta, &key), Ok(v) if v.len() == 8) { - return; - } - let k = cold_ae_key(pid); - let n = match ctx.db.get(&ctx.meta, &k) { - Ok(v) if v.len() == 8 => u64::from_be_bytes(v.as_slice().try_into().unwrap()), - _ => 0, - }; - let _ = ctx - .db - .put(&ctx.meta, &k, &(n + 1).to_be_bytes(), Duration::ZERO); - }); + /// Only a consumed nonce-correlated proof can reach this path. Recheck + /// its send-time generation/epoch on the owning shard before counting. + async fn note_clean_cold_round( + &self, + pid: Pid, + peer: NodeId, + generation: u64, + view_epoch: u64, + ) { + let cluster = self.cluster.clone(); + let evidence = self.cold_evidence.clone(); + self.store.run(pid, move |ctx| cluster.with_view(|view| { + if view.epoch != view_epoch || ctx.maintenance.generation(pid) != generation + || view.is_owner(pid, cluster.replicas_n, cluster.self_id) + || !view.owners(pid, cluster.replicas_n).contains(&peer) + || !view.members.iter().any(|m| m.node == peer && m.phase == marekvs_cluster::NodePhase::Active) + || !matches!(ctx.db.get(&ctx.meta, &cold_marker_key(pid)), Ok(v) if v.len() == 8) { return; } + let mut entries = evidence.lock(); + let proof = entries.entry(pid).or_insert(ColdEvidence { generation, view_epoch, rounds: 0 }); + if proof.generation != generation || proof.view_epoch != view_epoch { + *proof = ColdEvidence { generation, view_epoch, rounds: 0 }; + } + proof.rounds = proof.rounds.saturating_add(1); + })).await; } /// Slow reclaim loop: drop the local copy of partitions this node has not @@ -1528,92 +1744,61 @@ impl ReplEngine { tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { tick.tick().await; - if self.rejoin.lock().active { - continue; - } - let delay_ms = cold_purge_delay().as_millis() as u64; - let need_rounds = cold_purge_clean_rounds(); - let n = self.cluster.replicas_n; - let view = self.cluster.view(); - let owned: HashSet = self.cluster.owned_pids().into_iter().collect(); - for pid in 0..marekvs_core::PARTITIONS as Pid { - if owned.contains(&pid) { - continue; - } - // Never purge into a degraded cluster: if the owners are - // not all Active we may be holding the redundancy. - let owners = view.owners(pid, n); - let healthy = owners.len() >= n - && owners.iter().all(|o| { - view.members.iter().any(|m| { - m.node == *o && m.phase == marekvs_cluster::NodePhase::Active - }) - }); - if !healthy { - continue; - } - - let (aged, rounds) = self - .store - .run(pid, move |ctx| { - let marked = match ctx.db.get(&ctx.meta, &cold_marker_key(pid)) { - Ok(v) if v.len() == 8 => { - Some(u64::from_be_bytes(v.as_slice().try_into().unwrap())) - } - _ => None, - }; - let rounds = match ctx.db.get(&ctx.meta, &cold_ae_key(pid)) { - Ok(v) if v.len() == 8 => { - u64::from_be_bytes(v.as_slice().try_into().unwrap()) - } - _ => 0, - }; - let aged = marked - .is_some_and(|at| store::now_ms().saturating_sub(at) >= delay_ms); - (aged, rounds) - }) - .await; - if !aged || rounds < need_rounds { - continue; - } - match self.purge_partition(pid).await { - Ok(()) => { - self.engine.metrics.cold_purged_partitions_total.inc(); - tracing::info!( - pid, - rounds, - "cold purge: dropped local copy of an un-owned partition" - ); + Ok(PurgeOutcome::Purged) => { + self.engine.metrics.cold_purged_partitions_total.inc() + } + Ok(PurgeOutcome::Empty) => { + self.engine.metrics.cold_purge_empty_skips_total.inc() } - Err(e) => { - tracing::warn!(pid, error = %e, "cold purge range delete failed"); + Ok(PurgeOutcome::StaleEvidence) => { + self.engine.metrics.cold_purge_stale_proofs_total.inc() } + Ok(PurgeOutcome::Ineligible) => {} + Err(e) => tracing::warn!(pid, error = %e, "cold purge incomplete"), } } } }); } - /// Physically drop this node's records for `pid` with a single range delete. - /// - /// Deliberately a *local* drop: this node is discarding its own copy of a - /// partition it no longer owns, not deleting the records cluster-wide. - /// ondaDB never surfaces a range delete to a commit hook, so — unlike the - /// scan-and-tombstone predecessor, which needed an explicit - /// `suppress_commit_hook()` guard for exactly this — nothing here can reach - /// the replication ring even by accident. - /// - /// There is no chunking and no inter-chunk yield any more, because there is - /// no per-record work to spread out. The commit does take ondaDB's - /// database-wide commit lock (a range commit is atomic against every - /// isolation level), which is why this is only ever called for a whole cold - /// partition and never on a client path. - async fn purge_partition(&self, pid: Pid) -> anyhow::Result<()> { - self.store - .run(pid, move |ctx| store::delete_partition_range(ctx, pid)) + async fn purge_partition(self: &Arc, pid: Pid) -> anyhow::Result { + let repl = self.clone(); + let outcome = self + .store + .run(pid, move |ctx| { + // Hold placement stable throughout proof, probe and commit. The + // shard excludes all audited same-partition production writers, + // including the visible-before-postcommit-observer window. + repl.cluster.with_view(|view| { + let rejoin = repl.rejoin.lock(); + let owners = view.owners(pid, repl.cluster.replicas_n); + let eligible = !rejoin.active + && !owners.contains(&repl.store.node_id) + && owners.len() >= repl.cluster.replicas_n + && owners.iter().all(|owner| { + view.members.iter().any(|m| { + m.node == *owner && m.phase == marekvs_cluster::NodePhase::Active + }) + }); + let mut evidence = repl.cold_evidence.lock(); + purge_if_eligible( + ctx, + pid, + store::now_ms(), + cold_purge_delay().as_millis() as u64, + cold_purge_clean_rounds(), + view.epoch, + eligible, + evidence.get_mut(&pid), + ) + }) + }) .await?; + if outcome != PurgeOutcome::Purged { + return Ok(outcome); + } // Reclaim eagerly. A purge is exactly what delete-only excise is for: // a whole interval proven deleted, so a table lying entirely inside it @@ -1635,7 +1820,7 @@ impl ReplEngine { Ok(Err(e)) => tracing::warn!(pid, error = ?e, "excise pass failed after purge"), Err(e) => tracing::warn!(pid, error = %e, "excise task panicked after purge"), } - Ok(()) + Ok(outcome) } fn complete_rejoin_pid(&self, pid: Pid) { @@ -1982,6 +2167,7 @@ impl ReplEngine { } let peer = owners[(self.pseudo_rand() % owners.len() as u64) as usize]; self.mesh.send_ctl(peer, PeerMsg::MerkleRoot { pid, root }); + self.offer_cold_proof(pid, peer).await; } } } @@ -1998,16 +2184,25 @@ impl ReplEngine { /// hook). Quiescent partitions cost no I/O per AE round — previously /// the full keyspace was re-hashed every ~5 s. async fn partition_root_cached(&self, pid: Pid) -> ae::AeResult { + self.partition_root_snapshot(pid) + .await + .map(|(root, _)| root) + } + + async fn partition_root_snapshot(&self, pid: Pid) -> ae::AeResult<(u64, u64)> { if !self.ae_dirty.lock().contains(&pid) { - if let Some((root, at)) = self.ae_roots.lock().get(&pid) { - if at.elapsed() < AE_ROOT_CACHE_TTL { - return Ok(*root); + if let Some((root, generation, at)) = self.ae_roots.lock().get(&pid) { + if *generation == self.store.partition_generation(pid) + && at.elapsed() < AE_ROOT_CACHE_TTL + { + return Ok((*root, *generation)); } } } // Clear BEFORE scanning: writes landing mid-scan re-mark the pid, // so an invalidation can never be lost (worst case: one extra scan). self.ae_dirty.lock().remove(&pid); + let generation = self.store.partition_generation(pid); let root = match ae::partition_root(&self.store, pid).await { Ok(root) => root, Err(e) => { @@ -2019,9 +2214,18 @@ impl ReplEngine { return Err(e); } }; - self.ae_roots.lock().insert(pid, (root, Instant::now())); + if generation == self.store.partition_generation(pid) { + self.ae_roots + .lock() + .insert(pid, (root, generation, Instant::now())); + } else { + self.ae_dirty.lock().insert(pid); + return Err(store::ScanIncomplete(format!( + "partition {pid} changed during root computation" + ))); + } self.engine.metrics.ae_digest_scans_total.inc(); - Ok(root) + Ok((root, generation)) } // ------------------------------------------------------------------ @@ -2037,7 +2241,42 @@ impl ReplEngine { } async fn handle(self: &Arc, peer: NodeId, msg: PeerMsg) { + // Wire pids are u16, but runtime arrays contain exactly 4096 entries. + // Drop malformed frames before any generation or placement lookup. + if !valid_message_partition(&msg) { + tracing::warn!(peer, "ignoring peer message with invalid partition"); + return; + } match msg { + PeerMsg::ColdProofRequest { pid, nonce } => { + if !supports_cold_proof(self.mesh.peer_features(peer)) { + return; + } + let view = self.cluster.view(); + if !view.is_owner(pid, self.cluster.replicas_n, self.store.node_id) + || !view.members.iter().any(|m| { + m.node == self.store.node_id + && m.phase == marekvs_cluster::NodePhase::Active + }) + { + return; + } + let Ok((root, _)) = self.partition_root_snapshot(pid).await else { + return; + }; + self.mesh + .send_ctl(peer, PeerMsg::ColdProofResponse { pid, nonce, root }); + } + PeerMsg::ColdProofResponse { pid, nonce, root } => { + if !supports_cold_proof(self.mesh.peer_features(peer)) { + return; + } + let pending = self.cold_requests.lock().take(pid, peer, nonce); + if let Some(pending) = pending.filter(|pending| pending.root == root) { + self.note_clean_cold_round(pid, peer, pending.generation, pending.view_epoch) + .await; + } + } PeerMsg::Hello { .. } => {} PeerMsg::Repl(batch) => { tracing::debug!(peer, n = batch.ops.len(), "recv ReplBatch"); @@ -2226,7 +2465,7 @@ impl ReplEngine { // Cold purge (T2-9): for a partition we no longer own, a match // is direct evidence that an owner holds exactly our bytes — // the evidence the purge fence requires. - self.note_clean_cold_round(pid); + // Rootless acknowledgements never authorize destructive cleanup. } PeerMsg::MerkleBuckets { pid, digests } => { let Ok(ours) = ae::bucket_digests(&self.store, pid).await else { @@ -2467,6 +2706,9 @@ impl ReplEngine { let Some(p) = ikey::parse(&op.ikey) else { return; }; + if p.pid as usize >= marekvs_core::PARTITIONS as usize { + return; + } // HLC receive rule (Kulkarni): observe every ingested record's // timestamp so our next local write sorts after everything we have // seen. Without this, a peer with a lagging wall clock loses LWW @@ -3138,3 +3380,347 @@ mod compaction_guard_tests { assert_eq!(resolve_debt_water_marks(Some(1), None), (1, 0)); } } + +#[cfg(test)] +mod cold_maintenance_tests { + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT: AtomicU64 = AtomicU64::new(0); + fn test_store() -> (std::path::PathBuf, Arc) { + let path = std::env::temp_dir().join(format!( + "marekvs-cold-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let store = Store::open(&store::StoreConfig { + data_dir: path.to_string_lossy().into_owned(), + node_id: 1, + shard_threads: 2, + sync_mode: ondadb::SyncMode::Interval, + }) + .unwrap(); + (path, store) + } + fn key(pid: Pid) -> Vec { + let mut key = ikey::string_key(b"cold"); + key[..2].copy_from_slice(&pid.to_be_bytes()); + key + } + fn mark(ctx: &store::ShardCtx, pid: Pid) { + ctx.db + .put( + &ctx.meta, + &cold_marker_key(pid), + &1u64.to_be_bytes(), + Duration::ZERO, + ) + .unwrap(); + } + #[tokio::test] + async fn purge_once_then_ten_empty_checks_and_new_data_needs_new_proof() { + let (path, store) = test_store(); + store + .run(7, |ctx| { + mark(ctx, 7); + assert_eq!( + purge_if_eligible(ctx, 7, 100, 1, 2, 9, true, None).unwrap(), + PurgeOutcome::Empty + ); + store::put_raw(ctx, &key(7), b"old"); + let mut proof = ColdEvidence { + generation: ctx.maintenance.generation(7), + view_epoch: 9, + rounds: 2, + }; + assert_eq!( + purge_if_eligible(ctx, 7, 100, 1, 2, 9, true, Some(&mut proof)).unwrap(), + PurgeOutcome::Purged + ); + let count = ctx.data.stats().range_deletes; + assert_eq!(count, 1); + for _ in 0..10 { + assert_eq!( + purge_if_eligible(ctx, 7, 100, 1, 2, 9, true, Some(&mut proof)).unwrap(), + PurgeOutcome::Empty + ); + } + assert_eq!(ctx.data.stats().range_deletes, count); + store::put_raw(ctx, &key(7), b"new"); + assert_eq!( + purge_if_eligible(ctx, 7, 100, 1, 2, 9, true, Some(&mut proof)).unwrap(), + PurgeOutcome::StaleEvidence + ); + assert_eq!(store::get_raw(ctx, &key(7)).unwrap(), b"new"); + assert_eq!( + ctx.db.get(&ctx.meta, &cold_marker_key(7)).unwrap(), + 1u64.to_be_bytes() + ); + }) + .await; + drop(store); + let _ = std::fs::remove_dir_all(path); + } + #[tokio::test] + async fn changed_generation_view_degraded_ownership_and_restart_fence_purge() { + let (path, store) = test_store(); + store + .run(7, |ctx| { + mark(ctx, 7); + store::put_raw(ctx, &key(7), b"safe"); + let mut proof = ColdEvidence { + generation: ctx.maintenance.generation(7), + view_epoch: 3, + rounds: 2, + }; + assert_eq!( + purge_if_eligible(ctx, 7, 100, 1, 2, 3, false, Some(&mut proof)).unwrap(), + PurgeOutcome::Ineligible + ); + assert_eq!( + purge_if_eligible(ctx, 7, 100, 1, 2, 4, true, Some(&mut proof)).unwrap(), + PurgeOutcome::StaleEvidence + ); + store::put_raw(ctx, &key(7), b"after-match"); + assert_eq!( + purge_if_eligible(ctx, 7, 100, 1, 2, 3, true, Some(&mut proof)).unwrap(), + PurgeOutcome::StaleEvidence + ); + // Even legacy persisted counters cannot substitute for runtime proof. + ctx.db + .put( + &ctx.meta, + &cold_ae_key(7), + &100u64.to_be_bytes(), + Duration::ZERO, + ) + .unwrap(); + assert_eq!( + purge_if_eligible(ctx, 7, 100, 1, 2, 3, true, None).unwrap(), + PurgeOutcome::StaleEvidence + ); + assert_eq!(ctx.data.stats().range_deletes, 0); + }) + .await; + drop(store); + let reopened = Store::open(&store::StoreConfig { + data_dir: path.to_string_lossy().into_owned(), + node_id: 1, + shard_threads: 2, + sync_mode: ondadb::SyncMode::Interval, + }) + .unwrap(); + reopened + .run(7, |ctx| { + assert_eq!( + purge_if_eligible(ctx, 7, 100, 1, 2, 3, true, None).unwrap(), + PurgeOutcome::StaleEvidence + ); + assert_eq!(store::get_raw(ctx, &key(7)).unwrap(), b"after-match"); + }) + .await; + drop(reopened); + let _ = std::fs::remove_dir_all(path); + } + #[tokio::test] + async fn queued_write_between_evidence_and_purge_is_rechecked_on_shard() { + let (path, store) = test_store(); + let proof = store + .run(7, |ctx| { + mark(ctx, 7); + store::put_raw(ctx, &key(7), b"before"); + ColdEvidence { + generation: ctx.maintenance.generation(7), + view_epoch: 1, + rounds: 3, + } + }) + .await; + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + store.spawn_on(7, move |ctx| { + store::put_raw(ctx, &key(7), b"after"); + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + }); + entered_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + let (done_tx, done_rx) = std::sync::mpsc::channel(); + store.spawn_on(7, move |ctx| { + let mut proof = proof; + done_tx + .send(purge_if_eligible(ctx, 7, 100, 1, 2, 1, true, Some(&mut proof)).unwrap()) + .unwrap(); + }); + assert!(done_rx.try_recv().is_err()); + release_tx.send(()).unwrap(); + assert_eq!( + done_rx.recv_timeout(Duration::from_secs(2)).unwrap(), + PurgeOutcome::StaleEvidence + ); + store + .run(7, |ctx| { + assert_eq!(store::get_raw(ctx, &key(7)).unwrap(), b"after") + }) + .await; + drop(store); + let _ = std::fs::remove_dir_all(path); + } + #[tokio::test] + async fn local_purge_never_emits_point_replication_or_removes_remote_records() { + let (path, store) = test_store(); + let (remote_path, remote) = test_store(); + let hooks = Arc::new(AtomicU64::new(0)); + let observed = hooks.clone(); + store.set_commit_hook(Some(Arc::new(move |_, ops| { + observed.fetch_add(ops.len() as u64, Ordering::Relaxed); + }))); + for node in [&store, &remote] { + node.run(7, |ctx| store::put_raw(ctx, &key(7), b"replicated")) + .await; + } + let before = hooks.load(Ordering::Relaxed); + store + .run(7, |ctx| { + mark(ctx, 7); + let mut proof = ColdEvidence { + generation: ctx.maintenance.generation(7), + view_epoch: 1, + rounds: 2, + }; + assert_eq!( + purge_if_eligible(ctx, 7, 100, 1, 2, 1, true, Some(&mut proof)).unwrap(), + PurgeOutcome::Purged + ); + }) + .await; + assert_eq!(hooks.load(Ordering::Relaxed), before); + remote + .run(7, |ctx| { + assert_eq!(store::get_raw(ctx, &key(7)).unwrap(), b"replicated") + }) + .await; + drop(store); + drop(remote); + let _ = std::fs::remove_dir_all(path); + let _ = std::fs::remove_dir_all(remote_path); + } + + #[tokio::test] + async fn visible_commit_before_observer_cannot_overtake_cold_proof_check() { + let (path, store) = test_store(); + let proof = store + .run(7, |ctx| { + mark(ctx, 7); + store::put_raw(ctx, &key(7), b"before"); + ColdEvidence { + generation: ctx.maintenance.generation(7), + view_epoch: 1, + rounds: 2, + } + }) + .await; + let old_generation = store.partition_generation(7); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let release = std::sync::Mutex::new(release_rx); + store + .maintenance + .set_observer_barrier_for_tests(Some(Arc::new(move || { + entered_tx.send(()).unwrap(); + release.lock().unwrap().recv().unwrap(); + }))); + store.spawn_on(7, |ctx| { + store::put_raw(ctx, &key(7), b"visible-before-hook") + }); + entered_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + assert_eq!(store.partition_generation(7), old_generation); + assert_eq!( + store.db.get(&store.data, &key(7)).unwrap(), + b"visible-before-hook" + ); + let (done_tx, done_rx) = std::sync::mpsc::channel(); + store.spawn_on(7, move |ctx| { + let mut proof = proof; + done_tx + .send(purge_if_eligible(ctx, 7, 100, 1, 2, 1, true, Some(&mut proof)).unwrap()) + .unwrap(); + }); + assert!(done_rx.try_recv().is_err()); + release_tx.send(()).unwrap(); + assert_eq!( + done_rx.recv_timeout(Duration::from_secs(3)).unwrap(), + PurgeOutcome::StaleEvidence + ); + store.maintenance.set_observer_barrier_for_tests(None); + drop(store); + let _ = std::fs::remove_dir_all(path); + } +} + +#[cfg(test)] +mod cold_proof_protocol_tests { + use super::*; + #[test] + fn legacy_and_unknown_capabilities_do_not_enable_cold_cleanup_protocol() { + assert!(!supports_cold_proof(None)); + assert!(!supports_cold_proof(Some(0))); + assert!(!supports_cold_proof(Some( + marekvs_proto::features::COUNTER_FIELD + ))); + assert!(supports_cold_proof(Some(marekvs_proto::features::ALL))); + } + #[test] + fn malformed_peer_pids_are_rejected_before_generation_array_access() { + for pid in [4096, u16::MAX] { + for message in [ + PeerMsg::MerkleRoot { pid, root: 0 }, + PeerMsg::MerkleBuckets { + pid, + digests: vec![0; 256], + }, + PeerMsg::MerkleRootMatch { pid }, + PeerMsg::ColdProofRequest { + pid, + nonce: [0; 24], + }, + PeerMsg::ColdProofResponse { + pid, + nonce: [0; 24], + root: 0, + }, + ] { + assert!(!valid_message_partition(&message)); + } + } + assert!(valid_message_partition(&PeerMsg::MerkleRoot { + pid: 4095, + root: 0 + })); + } + #[test] + fn replaced_delayed_duplicate_unsolicited_and_wrong_peer_proofs_are_rejected() { + let mut requests = ColdProofRequests::new(); + let old = requests.issue(7, 2, 99, 10, 20).unwrap(); + let current = requests.issue(7, 2, 99, 11, 21).unwrap(); + assert_ne!(old, current); + assert!(requests.take(7, 2, old).is_none()); + assert!(requests.take(7, 3, current).is_none()); + assert!(requests.take(8, 2, current).is_none()); + let proof = requests.take(7, 2, current).unwrap(); + assert_eq!( + (proof.root, proof.generation, proof.view_epoch), + (99, 11, 21) + ); + assert!(requests.take(7, 2, current).is_none()); + } + #[test] + fn restart_changes_boot_nonce_and_lack_of_entropy_disables_proof() { + let mut first = ColdProofRequests::new(); + let mut second = ColdProofRequests::new(); + let old = first.issue(7, 2, 1, 0, 0).unwrap(); + let new = second.issue(7, 2, 1, 0, 0).unwrap(); + assert_ne!(old[..16], new[..16]); + assert!(second.take(7, 2, old).is_none()); + second.boot = None; + assert!(second.issue(8, 2, 1, 0, 0).is_none()); + } +} diff --git a/crates/marekvs-repl/src/mesh.rs b/crates/marekvs-repl/src/mesh.rs index 9ecc5d6..e2b401e 100644 --- a/crates/marekvs-repl/src/mesh.rs +++ b/crates/marekvs-repl/src/mesh.rs @@ -96,6 +96,8 @@ fn is_heavy_lane(msg: &PeerMsg) -> bool { msg, PeerMsg::MerkleRoot { .. } | PeerMsg::MerkleRootMatch { .. } + | PeerMsg::ColdProofRequest { .. } + | PeerMsg::ColdProofResponse { .. } | PeerMsg::MerkleBuckets { .. } | PeerMsg::BucketKeys { .. } | PeerMsg::RepairOps { .. } diff --git a/design/05-consistency-anti-entropy.md b/design/05-consistency-anti-entropy.md index 3d99d9a..3ec74cb 100644 --- a/design/05-consistency-anti-entropy.md +++ b/design/05-consistency-anti-entropy.md @@ -213,7 +213,7 @@ config). | mesh reconnect backoff | 100 ms → 5 s | const (marekvs-repl) | no | exponential | | gc_grace | 1 h | env `MAREKVS_GC_GRACE_SECS` | no | tombstone TTL; must be uniform cluster-wide; pull-only-until-synced rejoin rule **enforced** (above); `alive:last` heartbeat every min(gc_grace/4, 30 s) | | ttl_skew_grace | — | design (5 s) | — | expiry is materialized by the sweep as an ordinary tombstone write; digest-exclusion grace unimplemented | -| expiry sweep budget | 128 records | const (marekvs-engine) | no | incremental cursor walk between shard jobs | +| expiry sweep budget | 128 records / 8 partitions / 2 ms target | const (marekvs-engine) | no | fair own-partition discovery; park TTL-free partitions and schedule deadlines; commit generations invalidate proofs | | max_clock_drift | 5 s | const (marekvs-core `MAX_CLOCK_DRIFT_MS`) | no | remote HLC clamp + loud log | | repair_delay | — | design (30 s + jitter) | — | unimplemented; AE repairs fire on the next round | | bootstrap chunking | 256 ops/chunk, sequential | const (marekvs-repl) | no | lz4 bulk lane; donors refuse non-owned pids and dedup duplicate (peer, pid) streams within a 20 s window; design 8 concurrent streams unimplemented | @@ -221,7 +221,7 @@ config). | join gate timeout | 0 = wait forever | env `MAREKVS_JOIN_TIMEOUT_SECS` | no | operator escape hatch: forces Active with incomplete bootstrap (loud log + `marekvs_join_gate_timeouts_total`); gate progress visible via `marekvs_join_gate_pending_pids` | | disk high/low water | 90 % / 85 % | env `MAREKVS_DISK_HIGH_WATER_PCT` / `MAREKVS_DISK_LOW_WATER_PCT` | no | client writes (incl. DEL/EXPIRE/FLUSHALL — LSM deletes grow disk) get MISCONF at high-water; peer replication/AE/bootstrap and the REPLICAOF apply session are exempt; `marekvs_disk_write_stopped` | | disk min-avail floor | 1024 MiB | env `MAREKVS_DISK_MIN_AVAIL_MB` | no | write stop engages only when used% ≥ high-water AND available < floor (shared-fs false positives); releases at low-water or 2× the floor; statvfs + ondaDB DbStats polled every 2 s (`marekvs_disk_total_bytes` / `_avail_bytes`, `marekvs_db_total_bytes`) | -| cold_purge_delay | 15 m | env `MAREKVS_COLD_PURGE_SECS` | no | after the delay a partition this node no longer owns is dropped locally, reclaiming disk stranded by every scale event. Fenced by three further conditions, because this data is stranded-record AE's last-copy safety net: ≥ `MAREKVS_COLD_PURGE_CLEAN_ROUNDS` (3) stranded-AE exchanges must have returned `MerkleRootMatch` since the marker, the view must show a full `replicas_n` set of **Active** owners, and no rejoin may be active. The drop is a single ondaDB **range delete** over `[pid, pid+1)` — internal keys lead with the partition id big-endian, so a partition is exactly that interval — followed by a delete-only **excise** pass that unlinks wholly-covered SSTables by catalog edit without reading them. It is local-only, and structurally so: ondaDB never surfaces a range delete to a commit hook, so unlike the scan-and-tombstone predecessor it cannot reach the ring even if the `suppress_commit_hook()` discipline were lost. Signal: `marekvs_cold_purged_partitions_total` (`marekvs_cold_purged_records_total` is frozen — a range delete has no record count without reinstating the scan it replaced), with reclamation on `marekvs_db_excised_bytes` and `marekvs_db_range_fragments` | +| cold_purge_delay | 15 m | env `MAREKVS_COLD_PURGE_SECS` | no | after the delay a partition this node no longer owns is dropped locally, reclaiming disk stranded by every scale event. Fenced by three further conditions, because this data is stranded-record AE's last-copy safety net: ≥ `MAREKVS_COLD_PURGE_CLEAN_ROUNDS` (3) feature-gated nonce cold-proof exchanges must match the local root, current data generation and send-time view epoch, the view must show a full `replicas_n` set of **Active** owners, and no rejoin may be active. Runtime proof is rebuilt after restart; legacy AE responses do not authorize cleanup. Eligibility, an empty probe and deletion are serialized on the owning shard under a stable placement view. An empty partition emits no tombstone. The nonempty drop is a single ondaDB **range delete** over `[pid, pid+1)` — internal keys lead with the partition id big-endian, so a partition is exactly that interval — followed by a delete-only **excise** pass that unlinks wholly-covered SSTables by catalog edit without reading them. It is local-only, and structurally so: ondaDB never surfaces a range delete to a commit hook, so unlike the scan-and-tombstone predecessor it cannot reach the ring even if the `suppress_commit_hook()` discipline were lost. Signal: `marekvs_cold_purged_partitions_total` (`marekvs_cold_purged_records_total` is frozen — a range delete has no record count without reinstating the scan it replaced), with reclamation on `marekvs_db_excised_bytes` and `marekvs_db_range_fragments` | | mesh peer GC | 5 m | env `MAREKVS_MESH_PEER_GC_SECS` | no | dial loops for a node absent from the view this long are torn down (previously redialed until process exit, accumulating one pair per node ever seen). Reversible: a node that returns — including on a new address — is dialed again; `marekvs_mesh_peers_forgotten_total` | | terminationGracePeriodSeconds | 60 | manifest (k8s/statefulset.yaml) | k8s edit | drain typically completes in ~3 s | | listen addresses | :6379 / :7373 / :7946 / :9121 | env `MAREKVS_{RESP,MESH,GOSSIP,METRICS}_ADDR` | no | RESP / mesh / gossip(UDP) / metrics+probes | diff --git a/docs/consistency.md b/docs/consistency.md index bc30e4d..acf4591 100644 --- a/docs/consistency.md +++ b/docs/consistency.md @@ -119,16 +119,12 @@ off the storage engine's per-key TTL: may hold data whose covering tombstone was already purged elsewhere; merging it back would resurrect the delete. -```planned -**The pull-only-until-synced rejoin rule is designed but not yet enforced.** - -The intended enforcement: on rejoin, if `now − last_alive > gc_grace`, the -node's home partitions become **pull-only** — it receives AE repairs but never -pushes, until each partition completes a full Merkle sync against a current home -(its local data is a warm base; only the diff is pulled). Only then does it -regain push eligibility. This is the precise rule that prevents resurrection -across a long absence; today it is not wired into the rejoin path. -``` +The rejoin gate is enforced. An `alive:last` heartbeat persists while the node +is Active/Leaving. After an absence longer than `gc_grace`, the node stays +Joining while its data-bearing home partitions synchronize with their +pre-outage co-owners. Stale extras are dropped locally without replication; +only after synchronization does the node regain normal push eligibility. + Interest replicas cannot resurrect by construction: they never push AE, and lease-gated reads revalidate against homes. @@ -151,6 +147,38 @@ repairs around the deadline — is **unimplemented**. Today expiry is materializ by the sweep as an ordinary tombstone write, with no digest-exclusion grace. ``` +## Idle expiry and cold data cleanup + +Each shard discovers TTLs only in its own partitions, with a quantum of at most +128 records, eight partitions and a 2 ms elapsed-time target. One iterator step +or storage commit can exceed that target; the elapsed budget is checked between processed records. +Discovery also runs between foreground jobs, so busy clients cannot starve it. + +A completed partition with no TTLs parks without opening more expiry iterators. +Partitions with future TTLs wait for their earliest deadline. Committed writes, +replication, bootstrap and local range deletion invalidate the affected partition; +restart requires fresh discovery. Wall-clock changes are rechecked within one +second. This preserves expiry while eliminating repeated scans of persistent data. + +Cold data remains available for stranded-record anti-entropy after ownership +loss. Cleanup requires the configured retention delay (15 minutes by default), +a healthy full owner set, no active rejoin, and clean proof exchanges (three by default, configured with +`MAREKVS_COLD_PURGE_CLEAN_ROUNDS`) for +the current data generation and membership epoch. A feature-negotiated request +nonce binds each reply to its peer and request; stale, duplicate and prior-boot +replies cannot authorize deletion. Older peers continue ordinary replication and +anti-entropy, but cannot supply cleanup proofs until upgraded. + +The final eligibility check, bounded empty probe and local range delete run on +the owning shard under a stable placement view. An already empty partition emits +no new range tombstone. Clean proofs are rebuilt after restart. + +Observe `marekvs_expiry_iterators_total` and `marekvs_expiry_records_total` to +check that discovery settles. `marekvs_db_range_fragments` counts catalogued SST +fragments only; the separate memtable and range-cache memory gauges expose +in-memory range tombstones and retained fragment snapshots. See +[Testing](../testing/) for deterministic CI regressions and disposable benchmarks. + ## HLC discipline The full layout is in the [data model](../data-model/#hybrid-logical-clock); @@ -222,13 +250,13 @@ config). | ring high-water persist | 1 s | const (marekvs-repl) | restart resumes seq space +1,000,000 above persisted HW | | mesh writer queue | 4096 msgs | const (marekvs-repl) | per-peer, per-lane | | mesh reconnect backoff | 100 ms → 5 s | const (marekvs-repl) | exponential | -| gc_grace | 1 h | const (marekvs-engine `GC_GRACE`) | tombstone TTL; _Planned_ — the pull-only-until-synced rejoin rule is **not yet enforced** | +| gc_grace | 1 h | const (marekvs-engine `GC_GRACE`) | tombstone TTL; rejoin stays Joining until synchronization with pre-outage co-owners | | ttl_skew_grace | — | design (5 s) | _Planned_ — expiry is materialized by the sweep as an ordinary tombstone write; digest-exclusion grace unimplemented | -| expiry sweep budget | 128 records | const (marekvs-engine) | incremental cursor walk between shard jobs | +| expiry sweep budget | 128 records / 8 partitions / 2 ms target | const (marekvs-engine) | fair partition discovery and deadline scheduling between shard jobs | | max_clock_drift | 5 s | const (marekvs-core `MAX_CLOCK_DRIFT_MS`) | remote HLC clamp + loud log | | repair_delay | — | design (30 s + jitter) | _Planned_ — unimplemented; AE repairs fire on the next round | | bootstrap chunking | 256 ops/chunk, sequential | const (marekvs-repl) | lz4 bulk lane; _Planned_ — design 8 streams / 64 MiB/s rate cap unimplemented | -| cold_purge_delay | — | design (15 m) | _Planned_ — unimplemented; data kept after losing ownership (feeds stranded-record AE) | +| cold_purge_delay | 15 m | env `MAREKVS_COLD_PURGE_SECS` | generation/epoch-fenced local cleanup after fresh clean proofs; see above | | terminationGracePeriodSeconds | 60 | manifest (k8s/statefulset.yaml) | drain typically completes in ~3 s | | listen addresses | :6379 / :7373 / :7946 / :9121 | env `MAREKVS_{RESP,MESH,GOSSIP,METRICS}_ADDR` | RESP / mesh / gossip(UDP) / metrics+probes | | node identity | hostname ordinal, else 0 | env `MAREKVS_NODE_ID` | `marekvs-3` → 3; StatefulSet needs no per-pod config | @@ -265,10 +293,6 @@ Notes); the design target is what is missing. - **`repair_delay`** (30 s + jitter) — repairs fire on the next AE round. - **bootstrap 8 streams / 64 MiB/s rate cap** — chunking is 256 ops/chunk, sequential. -- **`cold_purge_delay`** (15 m) — cold data is kept after ownership loss (feeds - stranded-record AE). -- **`gc_grace` pull-only-until-synced rejoin rule** — the resurrection-prevention - gate is not yet enforced on rejoin. ``` ## Where to go next diff --git a/docs/investigations/2026-09-14-annotix-idle-cpu.md b/docs/investigations/2026-09-14-annotix-idle-cpu.md new file mode 100644 index 0000000..c6dc35b --- /dev/null +++ b/docs/investigations/2026-09-14-annotix-idle-cpu.md @@ -0,0 +1,124 @@ +# Annotix marekvs idle CPU investigation — 2026-09-14 + +## Finding + +The evidence points to repeated range-tombstone fragmentation during idle expiry +scans, amplified by repeated cold-partition purges. This is storage maintenance +CPU, rather than active Annotix requests or DIFF computation. + +No production container was restarted, reconfigured or modified during this +investigation. Temporary profiling containers attached to the hottest process; +short syscall tracing adds overhead, so the CPU measurements below also include +samples taken before tracing and an independent sampling profile. Temporary +synthetic databases were created outside the repository and removed afterward. + +## Live evidence + +The Annotix Docker Compose stack contains three marekvs nodes sharing node 0's +network namespace. Each has ten `mkv-shard-*` threads. All run image ID +`sha256:886bb1948d39f2573864d74b51c11aefdf64f8cc4a53ba3e17c80b869357652d`. + +| Node | First Docker CPU sample | Data-CF range-delete count | Connected clients | Active DIFF requests | +|---|---:|---:|---:|---:| +| 0 | 9.94% | 0 | 0 | 0 | +| 1 | 58.79% | 8,331 | 0 | 0 | +| 2 | 394.92% | 18,117 | 0 | 0 | + +Docker CPU uses 100% per logical core. A later node-2 sample reached 651.83%; +it was not a steady-state average. `perf stat` independently measured 4.3 CPU +cores over three seconds. A five-second 49 Hz user-space sampling profile +collected 934 samples, all attributed to the ten `mkv-shard-*` threads. +Compaction, WAL-sync, and DIFF workers were effectively idle. + +Over a 39–40 second metrics interval, no Redis command counters changed and no +replicated operation counters advanced. Background anti-entropy continued. +All nodes reported zero SSTable bytes, zero L0 tables, zero open SST readers, +and zero compaction debt. These reported gauges do not prove that no SST files +existed: a separate read-only inventory reported similar on-disk shapes on all +three nodes (58 files, 14 tables, roughly 24–34 MB). The synthetic experiment +below isolates the memtable cost independently of this reporting discrepancy. + +Complete retained logs showed repeated cold purges: + +- Node 1: 10,504 logged purges over 24 distinct partition IDs; up to 466 purges + for one partition. +- Node 2: 19,426 logged purges over 466 distinct partition IDs; up to 484 purges + for one partition. + +Retained-log totals cover container history and need not equal the current +process counters. During the short investigation window the range-delete +counters were stable; already accumulated tombstones were sufficient to keep +CPU high. + +## Source trace + +1. `crates/marekvs-repl/src/lib.rs:1525`, `spawn_cold_purge`, checks cold + partitions every 60 seconds. After a successful purge it increments a metric + but does not record that the current cold generation has already been purged + or consume its cleanup eligibility. Already-empty ranges can be deleted again. +2. `crates/marekvs-engine/src/store.rs:877`, `delete_partition_range`, writes + an ondaDB range tombstone unconditionally for the partition interval. +3. `crates/marekvs-engine/src/store.rs:669`, `shard_loop`, runs an expiry sweep + after each 100 ms idle receive timeout, independently on every shard. +4. `sweep_expired` constructs an **unbounded data-CF iterator** before processing + at most 128 visible records. It filters shard ownership only after iteration; + every shard pays whole-database iterator construction. The record budget does + not bound that construction cost or invisible records skipped by the iterator. +5. In the sibling ondaDB checkout, `src/column_family.rs:2257`, + `iterator_range_mask`, fragments the current in-memory range sets for every + newly constructed iterator. +6. `src/range_tombstone.rs:328`, `fragment_spans`, iterates every distinct + boundary interval and scans every span for covering ranges. The work scales + approximately with **range-span count × distinct boundaries**, plus sorting + sequence stacks. Unchanged tombstones incur this cost on every new iterator. + +The production binary is stripped: the sampling profile identifies hot threads +but does not directly symbolize their inner Rust functions. The source trace, +range-count correlation and isolated reproduction support the specific +fragmentation explanation. + +## Isolated reproduction + +A temporary release-mode Rust program linked the clean local ondaDB 0.9.0 +checkout. It issued only range deletions into an otherwise empty database, +then timed ten iterations of `begin → new_iterator → seek_to_first → drop`. +No client data, replica traffic or DIFF algorithms were involved. + +| Range deletes | Distinct partition intervals | Mean empty-iterator time | +|---:|---:|---:| +| 0 | 0 | 8 µs | +| 1,024 | 1,024 | 6,562 µs | +| 8,192 | 1,024 | 52,167 µs | +| 18,432 | 1,024 | 220,908 µs | + +A second run approximated the live counts and distinct-partition counts: + +| Range deletes | Distinct intervals | Before flush | After explicit memtable flush | +|---:|---:|---:|---:| +| 8,331 | 24 | 2,851 µs | 5 µs | +| 18,117 | 466 | 75,710 µs | 292 µs | + +The explicit flush was performed **only in the synthetic databases**. It moves +fragmentation out of repeated iterator construction, explaining the large +reduction. These macOS release measurements isolate the cost; they are not an +exact prediction of Linux VM CPU utilization. + +Temporary reproduction: `/tmp/marekvs-idle-probe/src/main.rs`. +Results: `/tmp/marekvs-idle-probe.log`, `/tmp/marekvs-idle-probe-shaped.log`. +Profiles: `/tmp/annotix-mkv-profile.txt`, `/tmp/annotix-mkv-perf.txt`. +Metrics snapshots: `/tmp/annotix-mkv-metrics-{0,1,2}-{a,b}.txt`. + +## Fix direction + +- Make cold cleanup idempotent for a cold ownership/data generation. Re-arm it + when ownership changes or newly arriving data requires cleanup; a blanket + permanent “already purged” flag would be unsafe. +- Reuse immutable range-fragment snapshots until range state changes, or use a + more efficient fragmentation algorithm. Preserve MVCC sequence visibility. +- Avoid ten independent unbounded expiry scans of the same database. Bound + iterator construction as well as visible-record processing, and scan only + the owning shard's partition ranges. +- Consider flushing range-heavy memtables as a controlled mitigation. A restart + alone is not a durable fix because WAL replay can restore the same tombstones. + +No fix or operational mitigation has been applied to the Annotix stack. diff --git a/docs/investigations/2026-09-14-idle-cpu-results.md b/docs/investigations/2026-09-14-idle-cpu-results.md new file mode 100644 index 0000000..dcb1104 --- /dev/null +++ b/docs/investigations/2026-09-14-idle-cpu-results.md @@ -0,0 +1,95 @@ +# Idle maintenance fix: release validation + +The final process fixture seeds 13,765 persistent keys and 18,117 range deletions +across 466 partitions in live memtables. Each run warms for 60 seconds and then +measures process CPU for 60 seconds, excluding setup and shutdown. CPU is percent +of one core. Measurements used the same macOS host and release toolchain; +background Docker/build activity means these are diagnostic measurements, not +controlled capacity claims. + +| Shards | Baseline CPU | Fixed CPU | Fixed idle iterator growth | +| --- | ---: | ---: | ---: | +| 1 | 29.198% | 0.382% | 0 | +| 2 | 70.442% | 0.341% | 0 | +| 10 | 337.494% | 0.370% | 0 | + +Baseline: marekvs `bd405d5`, ondaDB `bcd2de8`. Fixed behavior: marekvs +`c821053`, with annotation-only lint follow-up `e943840`, and ondaDB +`8afa06618ba4320df33b5a8c004adf2b85c003e8`. The final release dependency is +`0f4ebc67434551c9d3d4cba899829234760f4215` (ondaDB PR #3), which adds +only a Linux portability lint annotation to the reviewed cache implementation. +It is pinned in Cargo.toml and the canonical Git-source Cargo.lock. Exact Linux +Rust/Clippy 1.97.1 passes both default and unsafe-fastpath all-target lint gates. + +## Regression gates + +- Final marekvs `just ci` against the published dependency: 605 passed, zero + failed, three existing ignored tests; formatting, Clippy and grudge self-test + passed. New scheduler, cleanup, nonce correlation and INFO tests run in CI. +- ondaDB default suite: 1,052 passed, zero failed, 32 ignored. Unsafe-fastpath: + 1,047 passed, zero failed, 32 ignored. Both Clippy gates passed. A pre-existing + one-second flush timing check needed a focused retry and a full rerun with + four test threads under concurrent benchmark load; its timeout was unchanged. +- Documentation generator: 19 pages and landing page built successfully. +- Independent implementation reviews approved range snapshot reuse, expiry + invalidation/scheduling, cold cleanup serialization/proofs and INFO reporting. + +ondaDB's 80-case cold/warm benchmark and 20 structural cases passed. For the +empty ordinary-memtable 18,117-range fixture, warm iterator construction went +from about 117 ms to 0.604 microseconds; the first cold build still cost about +56 ms. These timings also had concurrent host load. Cache tests, rather than +wall-clock thresholds, enforce one shared build per unchanged range generation, +independent iterator positions, MVCC preservation and retained-memory accounting. + +## Docker acceptance + +The first final Docker matrix passed all fixed idle structural assertions: +zero expiry iterator and range-delete growth, no connected clients, and stable +ownership across the 60-second sample. The reproduction harness uses +three RF=2 nodes, unique persistent test volumes, separate client/peer networks, +180 seconds of warm-up and a 60-second no-client sample. It records individual +SET/GET latency and probes autonomous restart repair while the home is isolated +from peers before its first GET. The fixed invocation additionally requires +zero idle expiry iterator and range-delete growth with stable ownership. + +| Shards | Baseline aggregate CPU | Fixed aggregate CPU | Baseline ops/s | Fixed ops/s | Baseline p99 ms | Fixed p99 ms | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | 101.389% | 7.516% | 423.93 | 2,760.65 | 17.220 | 1.436 | +| 2 | 209.257% | 7.560% | 142.34 | 2,864.21 | 87.474 | 1.368 | +| 10 | 560.767% | 4.407% | 25.23 | 2,982.03 | 522.029 | 1.408 | + +CPU is the sum of the three node means; throughput and p99 are medians of three +runs of 1,000 SET/GET pairs, timing each command separately. Both matrices ran +three shard configurations concurrently on the same Docker runtime; background +build activity overlapped the baseline. The measurements meet the directional +throughput/p99 comparison but do not establish a controlled capacity benchmark. +The earlier pair-amortized latency reports were excluded. + +Baseline image: `sha256:886bb1948d39f2573864d74b51c11aefdf64f8cc4a53ba3e17c80b869357652d`. +Fixed acceptance image: `sha256:02fd6487b55c2428801532695abb512f45d5a21a193dce540a085db171f28b74`. +The first matrix's two-/ten-shard isolated restart probes timed out because +Docker stopped routing the published host port after peer-network removal. +A controlled reproduction showed the unchanged host port failing while localhost +PING/GET inside the node's network namespace succeeded; reconnecting the network +restored host access. The corrected harness uses a tracked redis-cli helper in +that namespace, with the peer network disconnected before the first GET. + +The complete corrected matrix passed for **all three shard counts**, including +TTL expiry, persistent-member retention, seed-data retention and autonomous +restart repair. All idle structural assertions passed again; every disposable +container, helper, volume and network was removed successfully. + +| Shards | Repeated fixed aggregate CPU | Ops/s | p99 ms | Behavior and structural checks | +| --- | ---: | ---: | ---: | --- | +| 1 | 6.899% | 2,901.74 | 1.478 | passed | +| 2 | 7.210% | 2,738.78 | 1.500 | passed | +| 10 | 6.535% | 3,041.14 | 1.400 | passed | + +The baseline two-shard run failed an immediate asynchronous member read, and the +baseline ten-shard probe encountered the same Docker transport issue. Those +failures were retained, not counted as passing baseline behavior checks. The +corrected harness waits for asynchronous member convergence and verifies local +repair without relying on host-port routing. + +No live Annotix deployment or volume was changed. Publishing v0.3.3 does not +perform a live rollout. diff --git a/docs/superpowers/plans/2026-09-14-idle-expiry-cpu.md b/docs/superpowers/plans/2026-09-14-idle-expiry-cpu.md new file mode 100644 index 0000000..9bfe52a --- /dev/null +++ b/docs/superpowers/plans/2026-09-14-idle-expiry-cpu.md @@ -0,0 +1,194 @@ +# Idle expiry and cold-purge CPU Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Remove repeated idle scanning and range-mask reconstruction while preserving TTL visibility, replication safety, and bounded foreground latency. + +**Architecture:** First fix range-fragment reuse in ondaDB, then make marekvs cold cleanup avoid repeated empty-range deletes. Replace independent whole-database expiry walks with generation-checked, partition-bounded discovery and deadline scheduling. Correct INFO reporting and verify the complete change in a disposable three-node workload before changing Annotix. + +**Tech Stack:** Rust 2021, marekvs workspace, ondaDB 0.9.0 sibling checkout, existing crossbeam/parking_lot/Tokio primitives, Docker test harness. + +**Spec:** [CPU investigation](../../investigations/2026-09-14-annotix-idle-cpu.md), the user's additional shard-loop investigation, and the correctness contracts below. Existing expiry and replication contracts in `design/05-consistency-anti-entropy.md` remain authoritative. + +## Execution status (2026-09-14) + +Tasks 2–6 are implemented and independently reviewed. The ondaDB +change is published in PR #2 (`8afa06618ba4320df33b5a8c004adf2b85c003e8`), +with default and unsafe-fastpath full suites passing. The final pin is +`0f4ebc67434551c9d3d4cba899829234760f4215` (PR #3), adding only the +Linux clock portability lint annotation validated on Rust 1.97.1. Tasks 4/5 additionally +require the nonce protocol amendment below. Task 1's disposable process and +Docker fixtures are available. The final Docker 1/2/10-shard matrix passed idle +structural assertions, TTL behavior and autonomous restart repair; see +[release validation](../../investigations/2026-09-14-idle-cpu-results.md). +Release publication is tracked in Task 7. The detailed checklist retains the original +acceptance contract; the final results report records any deviations explicitly. +Live Annotix rollout remains outside this release task. + +## Global constraints + +- Rust floor: 1.89. No new runtime dependencies, storage-format bits, Redis commands or record encodings. Review established that safe cleanup needs feature-negotiated internal proof request/reply messages; see the execution amendment below. +- Two repositories: `marekvs` and `../ondadb`. Keep their changes independently testable and reviewable. Create isolated execution worktrees; preserve all unrelated working files. +- Do not restart, flush, compact, reconfigure, load-test or replace the live Annotix stack during implementation. Use synthetic databases and disposable containers. Production rollout is a separate explicit operation. +- Preserve the budget-record exemption, deterministic expiry tombstone clocks, RGA dead anchors, head/delete-clock visibility, and passive ondaDB TTL backstop. +- Never infer absence from an incomplete/erroring scan. Never suppress active expiry based on `INFO expires`. +- No blocking shard submission or recursive storage writes from a commit callback. Replication-hook suppression must not suppress local maintenance invalidation. +- Do not solve this by simply increasing the 100 ms interval, reducing shard count, disabling expiry, or discarding historical range sequences. + +## Evidence corrections and acceptance contract + +1. `cmd/server.rs` currently formats `expires=0,avg_ttl=0` literally. The cited INFO result does **not** prove that these stores contain no TTLs. Even accurate key-level INFO would not count all per-member expirations. +2. The sweeper parses every internal key before its ownership check; envelope decoding is inside that check. All shards still pay iterator construction and traversal for the complete data CF. +3. `marekvs_db_range_deletes` is cumulative since open; `marekvs_db_range_fragments` reports catalogued SST fragments, not live memtable spans. Neither alone measures the current in-memory mask. +4. The isolated release experiment confirms expensive **iterator construction**, including on an empty database. Step cost may contribute, but it is not necessary to reproduce the fault. With 18,117 spans over 466 intervals, construction averaged 75.7 ms before a synthetic flush and 0.292 ms afterward. +5. Reported zero SST gauges do not establish that no files exist. The additional investigation reports similar on-disk shapes. Inventory read-only files and record both observations; do not make the remedy depend on an all-memory assumption. + +Success requires all of the following: + +- After initial discovery, a stable TTL-free store opens **zero expiry iterators** over a deterministic 60-second interval, regardless of shard count. +- Expiry reads only partitions assigned to the executing shard. Empty discovery is bounded by a partition count as well as a record count. +- A completed TTL-free proof cannot survive a relevant concurrent write unnoticed; restart/import/replication/repair cannot introduce a missed TTL. +- Due TTL work makes progress under continuously arriving jobs as well as when idle. +- Repeated reads of an unchanged range set reuse fragment storage and perform no re-fragmentation. Range mutations invalidate subsequent reads without changing existing iterator snapshots. +- Repeated eligible purges of an empty partition do not increment range-delete counts. New data must receive fresh cleanup safety evidence before a new purge. +- The representative three-node idle workload drops aggregate CPU by at least 90% relative to its pre-fix baseline and averages below 5% of one core per node over 60 seconds after warm-up. Measure on the same host/runtime, outside profiling; use structural counters as the CI oracle rather than flaky CPU assertions. + +## File and ownership map + +| Repository / files | Responsibility | +|---|---| +| marekvs `crates/marekvs-engine/src/store.rs`, new `src/store/expiry.rs` | Store-owned maintenance invalidation, expiry discovery and scheduling | +| marekvs `crates/marekvs-engine/src/cmd/server.rs`, `cmd/generic.rs` | Accurate key-level expiry statistics using existing visibility logic | +| marekvs `crates/marekvs-engine/src/metrics.rs` | Expiry work counters and live range-state measurements | +| marekvs `crates/marekvs-repl/src/lib.rs` | Cold-purge eligibility, generation-bound evidence, empty-range check | +| marekvs `crates/marekvs-engine/tests/expiry_maintenance.rs`, `tests/info_expiry.rs`; repl inline tests | Behavioral and race regressions | +| ondaDB `src/range_tombstone.rs`, `src/column_family.rs`, `src/unified.rs` | Immutable cached fragment snapshots and integration | +| ondaDB `tests/range_delete.rs`, new `tests/range_mask_reuse.rs` | MVCC, invalidation, bounds and allocation regressions | +| marekvs new `tests/idle_maintenance/` | Reproducible workload, metrics collection and CPU report | +| marekvs `docs/testing.md`, `design/05-consistency-anti-entropy.md`, investigation | Operating limits, evidence and rollout guidance | + +Dependency order: **1 → 2 → 3 → 4 → 5 → 6 → 7**. Each task ends with its focused tests and a separate commit. Do not begin later tasks with failing earlier gates. + +## Task 1 — Preserve the reproduction and add honest observability + +- [ ] Add `tests/idle_maintenance/README.md` and a synthetic fixture generator based on the recorded experiment: zero ranges; 1,024 unique ranges; 8,331 deletions across 24 partitions; 18,117 deletions across 466 partitions. Include a live non-expiring key and a completely empty variant. Construct data through APIs, never private production volumes. +- [ ] Add test-only counters at expiry iterator creation, visited records, partition transitions and range fragmentation. Reset/read counters within an isolated fixture; do not share global counters across parallel tests without isolation. +- [ ] Expose process metrics `marekvs_expiry_passes_total`, `marekvs_expiry_iterators_total`, `marekvs_expiry_records_total`, `marekvs_expiry_scan_errors_total`, `marekvs_expiry_tombstones_total`, and `marekvs_expiry_tick_seconds`. Use fixed labels only. Record only successful tombstone writes as emitted mutations. +- [ ] Add ondaDB statistics for current memtable range spans and cached fragment bytes, distinct from cumulative range deletes and catalogued fragments. Carry these through marekvs metrics using the existing DB statistics collection path. Instrument cache builds/hits in test/debug performance counters. +- [ ] Save an unchanged-baseline 60-second report: CPU, thread distribution, connected clients, command deltas, expiry iterator counts, range spans, range deletes, SST inventory and compaction activity. Sweep/iterator counters must distinguish discovery from due work. +- [ ] Verify baseline reproduction fails the acceptance contract: iterator count increases indefinitely without writes or TTLs. Preserve the failure as an ignored/manual performance baseline until the behavioral regression exists in Task 5. +- [ ] Commit the harness and instrumentation, with no behavior change. + +Reproduction must use both per-CF memtables and unified WAL/memtable configurations. Existing databases and default configurations remain readable. + +## Task 2 — Cache immutable range-fragment snapshots in ondaDB + +**Interfaces:** add an internal `RangeTombstoneSet::fragment_snapshot() -> Arc<[Fragment]>`; keep the existing clipped `fragments(lower, upper)` behavior for flush/compaction callers. Change `FragCursor` to own shared fragment storage plus its own cursor/range window. Expose no new public wire or on-disk format. + +- [x] Add a failing test: construct a range set, create/drop 100 iterators without mutations, and assert fragment construction happens once, shared backing allocation is reused, and every iterator has independent cursor state. +- [x] Add mutation/snapshot tests: hold an old iterator, add an overlapping range, create a new iterator; old traversal remains stable and the new iterator honors the new sequence. Check fixed read snapshots at before/equal/after tombstone sequences and point reinsertions after a range deletion. +- [x] Add bounds tests for inclusive/exclusive endpoints, reverse traversal, seek direction changes, custom comparator, overlapping sources, and a range extending outside an SST's point-key bounds. +- [x] Store an optional cached `Arc<[Fragment]>` alongside the protected span set. Invalidate it **under the same write lock** that changes spans; build/publish the first snapshot under exclusive access after rechecking the cache. This prevents concurrent shard scans from independently rebuilding the same cold cache or publishing stale results. Existing iterators keep their immutable Arcs. +- [x] Build the full snapshot once per range-set generation, then select overlapping fragments through binary search and a per-iterator window. Do not cache a vector for every arbitrary query bound and do not clone the complete vector on each iterator. Account one resident cache plus snapshots retained by live readers; expose retained-memory behavior in the benchmark. +- [x] Adapt `iterator_range_mask` to consume shared snapshots. Adapt unified mode without concatenating and sorting all cached fragments per read: retain per-source fragment lists, clip to the CF-prefixed interval, and translate keys consistently. Keep transaction overlay ranges isolated from the committed cache. +- [x] Preserve every sequence required by MVCC; do not deduplicate repeated deletions by keeping only their newest sequence. Ensure cache-only computation cannot swallow read/storage errors. +- [x] Run `cargo test --test range_delete --test range_mask_reuse` in ondaDB, then its normal full CI/test gate and feature variants covering unified and ordinary memtables. Rerun the reproduction: unchanged-iterator fragmentation count must stay constant after warm-up. +- [x] Commit in ondaDB. Do not optimize `fragment_spans` into a new sweep-line algorithm in this change: caching removes repeated work with a smaller correctness surface. Measure cold-build time separately and retain it as a known cost. + +## Task 3 — Integrate the dependency change reproducibly + +- [x] Add the reviewed ondaDB commit to the dependency provenance used by marekvs; update `Cargo.lock` against the canonical Git source once that commit is available there. Keep a local path patch only for development. Do not claim the lockfile is reproducible while it references an unpublished sibling-only change. +- [x] Validate that marekvs with the local patch and a clean checkout without the patch use the same reviewed ondaDB revision and pass the existing scan/TTL/merge suites. +- [x] Record pre/post **warm and cold** iterator timings and cache memory on the four fixtures. Ensure the optimization does not replace repeated CPU work with unbounded per-query cache growth. +- [x] Commit the marekvs dependency integration separately from behavior changes. + +Publishing the dependency is an execution handoff if not already authorized. Implementation and local verification can proceed with the path patch; remote publication must not be invented as completed. + +## Task 4 — Stop duplicate cold purges and bind safety evidence to data state + +**Interfaces:** extract a testable `purge_if_eligible` path returning `PurgeOutcome::{Purged, Empty, StaleEvidence, Ineligible}`. Keep ownership age persisted. Bind clean-AE counts to an in-process data generation; discard old clean evidence on restart, while retaining the conservative ownership-loss timestamp. + +- [x] Add a failing replica test: mark a cold partition safe, populate it, purge it, then evaluate cleanup ten more times. Assert the first pass emits one range delete and subsequent passes emit none. +- [x] Add tests for an initially empty partition; an incomplete empty probe; new data after a successful purge; writes after the last clean-AE exchange; ownership loss/regain/loss; restart; degraded owners. Errors must never be treated as proof of emptiness or permission to delete. +- [x] Introduce store-owned `partition_generation(pid) -> u64` invalidation shared with Task 5. Increment the affected partition after every successfully committed data mutation, including replication/AE/bootstrap and direct batch paths. Hook ordering is not commit ordering: do not assign an older sequence over a newer generation. Range operations need explicit invalidation because ondaDB omits range deletes from point commit hooks. +- [x] Bind a clean-AE exchange to the generation used for its root: capture before root computation, require the same generation after computing and when processing the peer's match. A cached root must carry the generation at which it was computed; sampling a new generation around an old cached root is insufficient. Count the match only for the same cold ownership epoch and data generation. Restart clears runtime proof so it cannot reuse a reset generation with old persisted counts. +- [x] Inside the owning shard job, recheck generation and cleanup eligibility, then use `new_iterator_bounded` on `[pid, pid+1)` to probe for live data. If complete and empty, return `Empty` without issuing a range delete. If nonempty, issue the existing local-only range deletion and clear clean evidence after success. Recheck owner-view epoch/health before deletion; a view change requires another eligibility evaluation. +- [x] Establish a serialization boundary covering the final proof check, empty probe and range deletion against all same-partition data writers. Use the owning shard when the ingress audit proves every writer uses it; otherwise route bypassing writers through that boundary. A post-commit callback alone cannot close the gap between data becoming visible and invalidation, or prevent a write between the final check and deletion. Add a barrier-controlled race test for both gaps. +- [x] Do not introduce a permanent “purged” flag. Later data or ownership changes must naturally require fresh proof. Keep cold age metadata separate from proof resets, so frequent writes do not accidentally defeat the retention safety window. +- [x] Retain best-effort excise after a successful nonempty purge; do not force flush/compact a live store to hide the root cause. Add `cold_purge_empty_skips_total` and `cold_purge_stale_proofs_total`. +- [x] Run existing repl tests plus the new cases; prove remote replicas retain records, same-key later data is not incorrectly purged, and repeated empty passes leave the range-delete count unchanged. Commit. + +## Task 5 — Replace perpetual whole-store expiry walks with safe discovery + +**State:** fixed-size per-partition generation counters (4,096 entries), dirty bits, and shard-local discovery records. Each partition is `Unknown`, `Scanning`, `NoTtl { generation }`, or `Due { generation, deadline_ms }`. State is advisory and rebuilt on restart; it is not a persisted TTL index. + +**Interfaces:** new `store/expiry.rs` owns `ExpiryScheduler`, with `next_wait(now_ms) -> Duration`, `invalidate(pid)`, and `poll(ctx, now_ms, record_budget, partition_budget, elapsed_budget) -> Result`. Use existing configured shard ownership and no new runtime dependencies. Use an injected clock/counters in unit tests. `ExpiryProgress` reports `records_visited: usize`, `partitions_completed: usize`, `iterator_opens: usize`, `tombstones_written: usize` and `has_more_work: bool`. `ScanIncomplete` retains the failed partition ID and storage error for retry/metrics; budget exhaustion is ordinary progress, not an error. + +- [x] Before coding the scheduler, inventory **every** data ingestion path: `put_raw`, `write_merged`, `put_many_lww`, replication batches, AE, bootstrap, restore, partition ingest/move/import, range deletes and physical derived-index deletes. Identify bypasses of the point commit hook. Route invalidation through one store-owned observer composed with the replication observer; never install two competing hooks or perform metadata writes in the callback. +- [x] Start every partition `Unknown`, including after restart. A data mutation sets its dirty bit and advances its generation without blocking. Conservative invalidation for non-TTL writes is acceptable; no false-negative invalidation is acceptable. +- [x] Discover only `pid % shard_count == shard_index`. For each partition use declared byte bounds `[pid, pid+1)` and a cursor within those bounds. Process at most 128 surfaced records and 8 partition transitions per poll, with a 2 ms elapsed-work check between steps. Construction/one underlying iterator step is not preemptible; document this and record its actual duration. +- [x] A multi-tick discovery pass captures its generation. If any mutation occurs before completion, discard its absence/deadline proof and rescan; a write behind the cursor must not be missed. Publish `NoTtl` only after a complete error-free scan with matching generation. Keep `Unknown` on errors and retry with bounded backoff; never clear dirtiness unconditionally after a concurrent commit. Apply the writer serialization contract from Task 4 to proof publication, or demonstrate an equivalent ordering protocol that closes the visible-commit/before-callback gap. Add a deterministic race test that pauses there. +- [x] For non-budget, live envelopes, remember the earliest positive TTL deadline during a complete scan. Keep future TTLs scheduled even when none are due now. At a due deadline, rescan that partition and apply the existing deterministic expiry behavior. TTL extension, PERSIST, collection replacement, member TTLs and merges invalidate the old proof. Native GC TTLs and budget payload deadlines are not interchangeable with envelope TTLs. +- [x] Park partitions proven `NoTtl` until mutation. For unfinished discovery or dirty partitions, schedule the next bounded poll within 100 ms; rotate partitions fairly so one busy partition cannot monopolize discovery. Otherwise cap waiting at 1 second to observe dirty bits and wall-clock changes without opening an iterator. Use wall clock for expiry decisions and a monotonic clock for elapsed-work limits. Forward clock jumps are noticed within that cap; backward jumps must not expire records early. +- [x] Change `shard_loop` to perform a bounded due/discovery poll after a job when its maintenance deadline has arrived, as well as after a receive timeout. Never let a continuously nonempty command queue starve expiration; never run unbounded catch-up work after a pause. +- [x] Add deterministic tests using the injected clock: no-TTL store performs discovery once and zero additional iterator opens over 60 seconds; 1/2/10 shards never visit another shard's partition; empty databases honor the partition budget; writes during discovery and replica TTL arrivals invalidate absence; future TTL, PERSIST, expired overwrite and backward/forward clock changes behave correctly; busy queues still expire due records; read failure leaves the partition armed. +- [x] Reuse `head_del`, JSON, per-member TTL, RGA-anchor and budget tests as visibility regressions. Add restart/replay and native-TTL filtering cases: discovery must not claim that a vanished expired record was actively tombstoned, and must preserve the existing passive expiry/replication contract. +- [x] Run `cargo test -p marekvs-engine --test expiry_maintenance`, the relevant existing suites, then full workspace tests. Commit. + +If an ingestion route cannot participate in invalidation, that route must leave affected partitions `Unknown` and trigger rediscovery. Do not enable a no-TTL skip for partitions whose writes cannot be observed. This is a correctness gate, not an optional optimization. + +## Task 6 — Make INFO expiry reporting truthful + +- [x] Add `tests/info_expiry.rs`: one persistent string, one future-TTL string, one future-TTL collection head, an already-expired key, and a persistent collection with only a per-member TTL. Assert `keys` counts live keys, `expires` counts only the two live key-level TTLs, and `avg_ttl` is computed from positive remaining key TTLs with a deterministic test clock/tolerance. +- [x] Replace `keyspace_count` with a shared visibility-aware `keyspace_stats` result containing `{ keys, expires, avg_ttl_ms }`. Reuse generic key enumeration/type and head-clock rules; avoid double-counting physical record families for one logical key. +- [x] Preserve INFO section filtering. Do not turn metrics scraping or expiry scheduling into repeated INFO scans; this remains explicit command work. Document that a multi-shard result is an observed aggregate, not a global snapshot. +- [x] Explain that `INFO expires=0` does not rule out per-member TTLs, and cannot act as the maintenance scheduler's safety proof. Run INFO/generic/type regression tests and commit. + +## Task 7 — End-to-end acceptance, docs and rollout handoff + +- [x] Run `just ci` on the final marekvs tree and the full ondaDB gate on its reviewed revision. Keep a TTL-heavy suite, range-deletion MVCC suite and cold-purge re-arrival tests as separate reported results. +- [x] In disposable Docker nodes, run the same no-TTL, key-TTL, member-TTL, mixed, restart and range-heavy fixtures before/after. Use fixed shard counts 1/2/10, persistent volumes scoped to the test, RF=2 across three nodes, and a stable ownership view. +- [x] Warm caches and complete initial discovery, then collect a 60-second no-client interval. Assert zero expiry iterator growth for stable no-TTL partitions and no range-delete growth for empty cold partitions. Record CPU mean/peak, mask builds/hits/bytes, discovery/due time, and replication repair counters. Observe at least three cold-purge ticks to catch duplicate cleanup. +- [x] Repeat under normal client traffic: report throughput and p50/p95/p99 latency; require no more than 5% throughput loss or p99 increase against the same-host baseline. Separate first-use fragmentation and restart discovery latency from steady state. If noisy, extend measurement instead of weakening correctness assertions. +- [x] Verify convergence after TTL deadlines, replica disconnect/reconnect, new writes into a previously purged partition, and a node restart. No lost last-copy data, stale mask reuse, skipped TTL or range resurrection is acceptable. +- [x] Update `docs/testing.md`, the defaults/design documentation, and the investigation with actual results and any unmet acceptance criteria. Record the resolved `expires=0` misconception and the distinction between iterator build work and visited-record budget. +- [ ] Produce reviewed, separate ondaDB and marekvs commits/PR descriptions with dependency ordering. Finish implementation locally before requesting any publication or live rollout authorization not already granted. +- [ ] Live Annotix handoff: identify the exact image digest and dependency revision; retain existing volumes; use its existing rollout scripts only after explicit authorization. Recheck readiness, replication health, TTL behavior and the 60-second idle CPU window. A rolling restart without the code change is not the remedy. + +## Execution amendment — cleanup response correlation + +Independent implementation review found that even equal-content Merkle buckets +can be delayed from an earlier ownership period. Assigning the current epoch +when a response arrives cannot prove that the exchange belongs to that epoch. +The implementation therefore adds feature-negotiated cold-proof request/reply +messages with unique request IDs. Pending evidence records the local root, +data generation, ownership epoch and peer at request time; receipt and deletion +recheck that provenance. Legacy root matches and bucket messages remain normal +anti-entropy traffic and cannot authorize cleanup. Mixed-version peers continue +replication; destructive cleanup waits for an owner supporting correlated proofs. +This is an internal protocol extension, with no Redis API or storage-format change. + +Review also requires a fairness regression with at least eight repeatedly dirty +partitions, elapsed-budget checks between expiry commits, and rejection of +out-of-range peer partition IDs before indexing generation arrays. + +## Acceptance execution notes + +The process fixture isolates the reported live-memtable, persistent-key worst +case for 1/2/10 shards. The ondaDB 80-case timing matrix and deterministic engine +suites cover the additional empty, range-shape and TTL cases. Baseline binaries +do not have the new maintenance counters; red/green tests provide the structural +comparison. Docker reports retain counter snapshots and warm for 180 seconds; +repeated-empty cleanup is enforced deterministically in CI, rather than inferred +from an unexported purge-tick count. Throughput/p99 comparisons improved in every +shard configuration, with background-host-load limitations stated in the report. +The initial Docker isolation transport error was reproduced and corrected, then +the complete fixed matrix reran successfully. Live rollout remains unperformed. + +## Plan review checklist + +- [x] Every successful data ingress invalidates expiry proofs, even when replication forwarding is suppressed. +- [x] Empty/error, future/already-expired, key/member/budget TTLs are distinct in tests. +- [x] Cached fragments preserve comparator ordering, source boundaries, MVCC stacks and independent cursors. +- [x] Cold proof is invalidated by new data and restart; empty cleanup adds no new tombstone. +- [x] No runtime/production action is claimed completed by this planning document. diff --git a/docs/testing.md b/docs/testing.md index b4fe1e9..654cbac 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -34,10 +34,10 @@ score-index key order matches f64 order (including ±0, subnormals, infinities). These run against a real ondaDB instance, not a mock. -- **Commit-hook contract** — hooks fire exactly once per committed batch, in - publish order, with the full op list. Hammered with concurrent committers - across shards, asserting the ring sees a gap-free, ordered seq stream. This - test is the canary for ondaDB upgrades. +- **Commit-hook contract** — hooks fire exactly once per committed batch, with the full op list and + unique commit sequence numbers. Concurrent callbacks can arrive out of + commit order, and metadata writes create sequence gaps; neither ordering nor + gap-free delivery is assumed. This test is the canary for ondaDB upgrades. - Shard-thread RMW atomicity (INCR storms on one key); TTL sweeper vs lazy expiry vs compaction GC; prefix-scan boundary discipline (the iterator stops at prefix end); crash-restart WAL replay with `SyncMode::Interval` (bounded loss window, @@ -46,6 +46,37 @@ These run against a real ondaDB instance, not a mock. golden-file suite for RESP2/RESP3 framing (HELLO switching, map/set/push frames, downgrades). +## Idle maintenance regressions + +The PR CI workflow runs the workspace tests and the range-deletion tests in the +exact ondaDB revision recorded in `Cargo.lock`. Image and patch-release builds +wait for that same test gate before publishing. + +The expiry tests use controlled clocks and work counters: after a complete +TTL-free discovery, advancing 60 seconds must open no further expiry iterators. +They also exercise shard ownership, writes behind a discovery cursor, replicated +TTL arrivals, future deadlines, PERSIST, busy command queues and restart. A scan +error must leave the partition scheduled for another attempt. + +Cold-cleanup tests check that an empty partition adds no range tombstone and +that new data invalidates older cleanup evidence. Delayed anti-entropy responses +must not authorize deletion of a newer generation. ondaDB tests independently +check cached fragment reuse, snapshot lifetime, bounds, overlapping sources, +forward/reverse traversal and MVCC visibility after range deletion/reinsertion. + +`INFO keyspace` tests distinguish live key-level deadlines from per-member TTLs +and exclude deleted collection residue. `expires=0` does not imply that no +collection member has a deadline, and INFO is never the scheduler's proof that +expiry work can stop. + +CPU thresholds belong to an isolated performance run, not timing-sensitive CI +assertions. The reproducible harness in `tests/idle_maintenance/` measures +release-mode process CPU and disposable three-node Docker workloads. It records +warm-up separately, samples an idle 60-second window, and checks TTL and restart +behavior without accessing deployment volumes. Initial discovery and the first +fragment build still cost work; the regression contract removes repeated work +on unchanged data. + ## Membership churn & chaos (Jepsen-style) ```success Implemented diff --git a/tests/idle_maintenance/README.md b/tests/idle_maintenance/README.md new file mode 100644 index 0000000..03e8cf9 --- /dev/null +++ b/tests/idle_maintenance/README.md @@ -0,0 +1,57 @@ +# Idle maintenance reproduction + +Run only against disposable synthetic data. The fixture creates and removes its +own temporary database; it never opens an existing deployment's volume. + +```sh +cargo run --release -p marekvs-engine --example idle_maintenance -- 10 18117 466 13765 60 60 +``` + +Arguments are shard count, range-delete count, distinct partition ranges, live +persistent key count, warm-up seconds and sample seconds. Output contains +Prometheus snapshots bracketing the sample and process CPU time (100% = one core). +Warm-up, seeding and process shutdown are excluded from measured CPU. + +Compare the same release toolchain, host, arguments and ondaDB provenance before +and after the fix. Test shards 1/2/10; range fixtures 0/0, 1024/1024, 8331/24 and +18117/466; and keys 0 and 13765. Use at least a 60-second sample. No TTLs are +installed here: key/member TTL, restart and cold-purge safety belong to the +behavioral regression suites, which run under the ordinary CI test command. + +The process fixture isolates expiry maintenance and database worker cost. It +is not a three-node networking or client-throughput benchmark. Record that +separately using disposable containers, stable RF=2 ownership and no live +Annotix data. ondaDB's range-cache benchmark covers ordinary and unified +memtables, cold/warm iterator construction and retained cache memory. + +## Disposable Docker acceptance + +```sh +python3 tests/idle_maintenance/cluster.py --image marekvs:idle-baseline --shards 10 --output /tmp/idle-baseline.json +python3 tests/idle_maintenance/cluster.py --image marekvs:idle-fixed --expect-fixed --shards 10 --output /tmp/idle-fixed.json +``` + +The script assigns unique peer and client networks, containers, volumes and dynamic loopback +ports, and removes only those resources when finished. It seeds one owner before +joining two peers (RF=2), uses shortened cold-retention settings **only in these +test containers**, warms up for at least 180 seconds, then records CPU and +metrics for 60 seconds without client traffic. After sampling it measures three +SET/GET throughput/latency runs and verifies key/member expiry and restart repair. +Repeat with `--shards 1` and `--shards 2`. Reports retain the image digest, +configuration, metric snapshots, raw CPU samples, behavior outcome and logs. + +Compare aggregate mean CPU and per-node means, plus the median throughput and +p99 across the three runs. Latencies measure individual SET and GET commands. Check the structural expiry/range counters alongside +CPU: low CPU alone cannot show that a scheduler still expires records correctly. +`--expect-fixed` waits for discovery to settle and requires zero growth in expiry +iterator and range-delete counters during the idle sample. Restart repair is +checked with the restarted home disconnected from the peer network before its +first GET. A short-lived `redis-cli` helper shares that container's network +namespace and probes localhost, avoiding published-port routing changes during +mesh disconnect. It is tracked and removed with the test containers. The default +helper image is `redis:alpine` (override with `--probe-image`); ensure it is available +locally before running the workload. Reports record its image digest. Host client +connections are recreated after mesh connectivity is restored. + +The script records failure in its JSON report and exits unsuccessfully; it does +not replace behavioral CI tests with a timing threshold. diff --git a/tests/idle_maintenance/cluster.py b/tests/idle_maintenance/cluster.py new file mode 100644 index 0000000..97be811 --- /dev/null +++ b/tests/idle_maintenance/cluster.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Disposable Docker acceptance workload. Uses only its uniquely named resources.""" +import argparse +import ipaddress +import json +import socket +import statistics +import subprocess +import time +import urllib.request +import uuid +from pathlib import Path + + +def docker(*args): + return subprocess.check_output(["docker", *args], text=True).strip() + + +class Client: + def __init__(self, port): + self.sock = socket.create_connection(("127.0.0.1", port), timeout=15) + self.file = self.sock.makefile("rb") + + def close(self): + self.file.close() + self.sock.close() + + def command(self, *args): + args = [str(a).encode() if not isinstance(a, bytes) else a for a in args] + self.sock.sendall(b"*%d\r\n" % len(args) + b"".join( + b"$%d\r\n" % len(a) + a + b"\r\n" for a in args)) + return self.read() + + def read(self): + line = self.file.readline() + if not line: + raise EOFError("RESP connection closed") + kind, value = line[:1], line[1:-2] + if kind == b"-": + raise RuntimeError(value.decode()) + if kind == b"+": + return value.decode() + if kind == b":": + return int(value) + if kind == b"$": + n = int(value) + if n == -1: + return None + data = self.file.read(n + 2) + assert len(data) == n + 2 and data[-2:] == b"\r\n" + return data[:-2] + if kind == b"*": + return [self.read() for _ in range(int(value))] + raise AssertionError(line) + + +def eventually(check, timeout=90): + end = time.monotonic() + timeout + error = None + while time.monotonic() < end: + try: + if check(): + return + except (OSError, RuntimeError, EOFError) as exc: + error = exc + time.sleep(0.5) + raise AssertionError(f"condition timed out: {error}") + + +def metrics(port): + with urllib.request.urlopen(f"http://127.0.0.1:{port}/metrics", timeout=15) as r: + body = r.read().decode() + out = {} + for line in body.splitlines(): + if line and not line.startswith("#"): + name, value = line.split() + out[name] = float(value) + return out + + +def local_probe(container, key, name, probe_image, tracked_containers): + """Probe localhost without Docker's published-port/gateway routing.""" + helper = docker("create", "--name", name, "--network", f"container:{container}", + "--tmpfs", "/data", "--entrypoint", "redis-cli", probe_image, + "-e", "-t", "5", "--raw", "-h", "127.0.0.1", "GET", key) + tracked_containers.append(helper) + # A socket/command timeout must not leave the helper untracked. Outer + # teardown removes it even if docker start or this bounded probe fails. + return subprocess.check_output(["docker", "start", "--attach", helper], + text=True, timeout=15).strip() + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--image", required=True) + ap.add_argument("--probe-image", default="redis:alpine", help="redis-cli image for the isolated localhost repair check") + ap.add_argument("--shards", type=int, default=10) + ap.add_argument("--keys", type=int, default=13765) + ap.add_argument("--warm", type=int, default=180) + ap.add_argument("--sample", type=int, default=60) + ap.add_argument("--output", type=Path, required=True) + ap.add_argument("--expect-fixed", action="store_true", help="assert idle maintenance stops after discovery") + args = ap.parse_args() + assert 1 <= args.shards <= 4096 and args.sample >= 60 and args.warm >= 180 + prefix = "mkv-idle-" + uuid.uuid4().hex[:10] + names, clients, ports, metric_ports, volumes = [], [], [], [], [] + report = {"image": args.image, "shards": args.shards, "keys": args.keys, + "warm_seconds": args.warm, "sample_seconds": args.sample, "prefix": prefix} + network = False + edge_network = False + cleanup_errors = [] + try: + docker("network", "create", prefix) + network = True + docker("network", "create", prefix + "-edge") + edge_network = True + subnet = json.loads(docker("network", "inspect", prefix))[0]["IPAM"]["Config"][0]["Subnet"] + net = ipaddress.ip_network(subnet) + ips = [str(net.network_address + 10 + i) for i in range(3)] + seeds = ",".join(f"{ip}:7946" for ip in ips) + for i in range(3): + name, volume = f"{prefix}-{i}", f"{prefix}-data-{i}" + volumes.append(volume) + docker("volume", "create", volume) + name = docker("create", "--name", name, "--network", prefix + "-edge", "-p", "127.0.0.1::6379", "-p", "127.0.0.1::9121", + "-v", f"{volume}:/data", "-e", f"MAREKVS_NODE_ID={i}", + "-e", f"MAREKVS_ADVERTISE_IP={ips[i]}", "-e", f"MAREKVS_SEEDS={seeds}", + "-e", "MAREKVS_REPLICAS_N=2", "-e", f"MAREKVS_SHARDS={args.shards}", + "-e", "MAREKVS_DATA_DIR=/data", "-e", "MAREKVS_COLD_PURGE_SECS=1", + "-e", "MAREKVS_COLD_PURGE_CLEAN_ROUNDS=1", "-e", "RUST_LOG=warn", args.image) + names.append(name) + docker("network", "connect", "--ip", ips[i], prefix, name) + docker("start", name) + info = json.loads(docker("inspect", name))[0] + ports.append(int(info["NetworkSettings"]["Ports"]["6379/tcp"][0]["HostPort"])) + metric_ports.append(int(info["NetworkSettings"]["Ports"]["9121/tcp"][0]["HostPort"])) + def ready(): + c = Client(ports[i]) + try: + return c.command("PING") == "PONG" + finally: + c.close() + eventually(ready) + if i == 0: + c = Client(ports[0]) + try: + # Populate the sole owner, then join peers. Ownership loss + # creates cold copies and exercises real cleanup rounds. + for start in range(0, args.keys, 128): + pairs = [v for k in range(start, min(start + 128, args.keys)) + for v in (f"idle-key-{k}", "value")] + assert c.command("MSET", *pairs) == "OK" + finally: + c.close() + eventually(lambda: all(metrics(p).get("marekvs_cluster_members") == 3 for p in metric_ports)) + print(f"{prefix}: three nodes ready; warming {args.warm}s", flush=True) + time.sleep(args.warm) + if args.expect_fixed: + def stable_discovery(): + first = [metrics(p) for p in metric_ports] + if any(m.get("marekvs_expiry_partitions_completed_total", 0) < 4096 for m in first): + return False + time.sleep(5) + second = [metrics(p) for p in metric_ports] + return all(a["marekvs_expiry_iterators_total"] == b["marekvs_expiry_iterators_total"] + and a["marekvs_cluster_owned_partitions"] == b["marekvs_cluster_owned_partitions"] + and b["marekvs_cluster_members"] == 3 + and b["marekvs_join_gate_pending_pids"] == 0 + for a, b in zip(first, second)) + eventually(stable_discovery, 120) + before = [metrics(p) for p in metric_ports] + report["metrics_before"] = before + report["image_digest"] = json.loads(docker("inspect", names[0]))[0]["Image"] + samples = [] + end = time.monotonic() + args.sample + while time.monotonic() < end: + rows = [json.loads(line) for line in docker("stats", "--no-stream", "--format", "{{json .}}", *names).splitlines()] + samples.append([float(next(r["CPUPerc"] for r in rows if n.startswith(r["ID"])).rstrip("%")) for n in names]) + time.sleep(min(2, max(0, end - time.monotonic()))) + after = [metrics(p) for p in metric_ports] + report["metrics_after"] = after + if args.expect_fixed: + for a, b in zip(before, after): + assert b["marekvs_expiry_iterators_total"] == a["marekvs_expiry_iterators_total"], "expiry work continued while idle" + assert b["marekvs_db_range_deletes"] == a["marekvs_db_range_deletes"], "empty cleanup added range tombstones" + assert b["marekvs_cluster_members"] == a["marekvs_cluster_members"] == 3 + assert b["marekvs_cluster_owned_partitions"] == a["marekvs_cluster_owned_partitions"] + assert b["marekvs_connected_clients"] == a["marekvs_connected_clients"] == 0 + report["structural_checks"] = "passed: completed discovery, zero idle expiry/range work, stable ownership" + report["cpu_samples_percent"] = samples + report["cpu_mean_percent"] = [statistics.mean(v) for v in zip(*samples)] + report["cpu_peak_percent"] = [max(v) for v in zip(*samples)] + print(f"{prefix}: idle CPU means {report['cpu_mean_percent']}", flush=True) + clients = [Client(p) for p in ports] + latency_runs = [] + for _ in range(3): + latencies = [] + start = time.perf_counter() + for i in range(1000): + t = time.perf_counter() + assert clients[0].command("SET", f"load-{i % 100}", "value") == "OK" + latencies.append((time.perf_counter() - t) * 1000) + t = time.perf_counter() + assert clients[0].command("GET", f"load-{i % 100}") == b"value" + latencies.append((time.perf_counter() - t) * 1000) + elapsed = time.perf_counter() - start + latencies.sort() + latency_runs.append({"ops_per_second": 2000 / elapsed, "p50_ms": latencies[999], + "p95_ms": latencies[1899], "p99_ms": latencies[1979]}) + report["client_runs"] = latency_runs + assert clients[0].command("SET", "ttl-key", "value", "PX", 3000) == "OK" + eventually(lambda: all(c.command("GET", "ttl-key") == b"value" for c in clients), 2) + eventually(lambda: all(c.command("GET", "ttl-key") is None for c in clients), 15) + assert clients[0].command("HSET", "ttl-hash", "f", "value", "persistent", "value") == 2 + assert clients[0].command("HPEXPIRE", "ttl-hash", 3000, "FIELDS", 1, "f") == [1] + eventually(lambda: all(c.command("HGET", "ttl-hash", "f") is None for c in clients), 15) + eventually(lambda: all(c.command("HGET", "ttl-hash", "persistent") == b"value" for c in clients)) + # Choose a home key for node 2, so its repair must be autonomous. + slots = clients[0].command("CLUSTER", "SLOTS") + node2_id = clients[2].command("CLUSTER", "MYID") + repair_key = None + for candidate in range(100): + key = f"repair-after-restart-{candidate}" + slot = clients[0].command("CLUSTER", "KEYSLOT", key) + if any(row[0] <= slot <= row[1] and any(owner[2] == node2_id for owner in row[2:]) for row in slots): + repair_key = key + break + assert repair_key is not None + # Stop/restart only our own node with its own persistent test volume. + clients[2].close() + docker("stop", names[2]) + assert clients[0].command("SET", repair_key, "value") == "OK" + docker("start", names[2]) + ports[2] = int(json.loads(docker("inspect", names[2]))[0]["NetworkSettings"]["Ports"]["6379/tcp"][0]["HostPort"]) + eventually(lambda: socket.create_connection(("127.0.0.1", ports[2]), timeout=1).close() is None) + metric_ports[2] = int(json.loads(docker("inspect", names[2]))[0]["NetworkSettings"]["Ports"]["9121/tcp"][0]["HostPort"]) + eventually(lambda: metrics(metric_ports[2]).get("marekvs_join_gate_pending_pids") == 0) + time.sleep(20) # allow the normal AE bound before a local-only probe + # Remove peer connectivity before the first GET. Published host ports + # can stop routing when Docker changes gateways on network disconnect; + # a helper in the same network namespace reaches localhost directly. + docker("network", "disconnect", prefix, names[2]) + try: + time.sleep(4) + value = local_probe(names[2], repair_key, prefix + "-local-probe", args.probe_image, names) + assert value == "value", "autonomous repair missing local home record" + report["probe_image_digest"] = json.loads(docker("inspect", names[-1]))[0]["Image"] + report["local_repair_check"] = "passed via container-namespace localhost with mesh disconnected before first GET" + finally: + docker("network", "connect", "--ip", ips[2], prefix, names[2]) + ports[2] = int(json.loads(docker("inspect", names[2]))[0]["NetworkSettings"]["Ports"]["6379/tcp"][0]["HostPort"]) + def reconnected(): + c = Client(ports[2]) + try: + if c.command("PING") != "PONG": + return False + clients[2] = c + return True + finally: + if clients[2] is not c: + c.close() + eventually(reconnected) + assert clients[2].command("GET", "ttl-key") is None + assert clients[2].command("HGET", "ttl-hash", "f") is None + report["seed_reads"] = {} + for i in (0, args.keys // 2, args.keys - 1): + def seed_converged(): + values = [] + for port in ports: + c = Client(port) + try: + value = c.command("GET", f"idle-key-{i}") + values.append(None if value is None else value.decode()) + finally: + c.close() + report["seed_reads"][str(i)] = values + return values == ["value"] * 3 + eventually(seed_converged) + + report["behavior_checks"] = "passed: key/member TTL, persistent member, disconnect/restart repair, seed data retained" + except BaseException as exc: + report["error"] = repr(exc) + raise + finally: + try: + for c in clients: + try: + c.close() + except OSError as exc: + cleanup_errors.append(str(exc)) + report["logs"] = {} + for n in names: + try: + report["logs"][n] = docker("logs", "--tail", "50", n) + except (OSError, subprocess.CalledProcessError) as exc: + report["logs"][n] = str(exc) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n") + finally: + commands = [("rm", "-f", n) for n in names] + [("volume", "rm", v) for v in volumes] + if network: + commands.append(("network", "rm", prefix)) + if edge_network: + commands.append(("network", "rm", prefix + "-edge")) + for command in commands: + try: + docker(*command) + except (OSError, subprocess.CalledProcessError) as exc: + cleanup_errors.append(str(exc)) + if cleanup_errors: + report["cleanup_errors"] = cleanup_errors + args.output.write_text(json.dumps(report, indent=2) + "\n") + raise RuntimeError("test resource cleanup incomplete: " + "; ".join(cleanup_errors)) + + + +if __name__ == "__main__": + main()