From d399ee91b5008ce3737699deb3fd51bfd23e8077 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 14:50:30 +0200 Subject: [PATCH 1/7] feat(objio): list keys with their tags and delete them under a lease Bucket gains list_tags (keys with the tag get would report, one listing, no downloads) and delete_leased (delete each (key, tag) only if the key still holds that tag). The defaults are an error and an unconditional delete; GitBucket overrides both. GitBucket::delete_leased pushes ':refs/sc/K' with one --force-with-lease=refs/sc/K: per key (never '+', never --atomic) in chunks of at most 128 refs and 24 KiB of arguments. A key re-created since the caller's listing holds another oid and survives; after a refused push every ref of the chunk is re-read, and a key still at its lease is an error, so a hook refusal is never mistaken for a lost race. Contract suite covers the common part; the git tests cover a moved ref, a re-created ref, an update hook refusing one ref, bad keys and leases, and the command line (one lease per ref, no '+', under 32 KiB). --- crates/objio/src/git.rs | 267 +++++++++++++++++++++++++++++++++++++++- crates/objio/src/lib.rs | 41 ++++++ 2 files changed, 307 insertions(+), 1 deletion(-) diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index 33dce183..0d8ad8e6 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -50,6 +50,11 @@ const SMALL_CAP: usize = 1 << 20; const NS: &str = "refs/sc/"; /// Refspecs per `fetch` call in `list_meta` (keeps the command line short). const FETCH_BATCH: usize = 256; +/// Most refs one leased delete push carries, and the most command-line bytes it +/// may use (a lease plus a refspec per key; Windows caps a command line near +/// 32 KiB, so the budget stays well under it). +const DELETE_BATCH: usize = 128; +const DELETE_ARG_BUDGET: usize = 24 << 10; /// `-c` overrides on every git call: transport allowlist, no hooks, no signing, /// no background maintenance, bounded stalls. @@ -1167,6 +1172,84 @@ impl Bucket for GitBucket { }, } } + + fn list_tags(&self, prefix: &str) -> Result> { + check_prefix(prefix)?; + self.remote_list(prefix) + } + + /// One `git push --porcelain` per chunk, a `--force-with-lease=refs/sc/K:` + /// and a `:refs/sc/K` refspec per key (never `+`, never `--atomic`): the + /// host checks each lease under that ref's lock, so a key someone re-created + /// since the caller's listing survives. A refused push is settled by + /// re-reading every ref of the chunk: gone or at another value is fine, still + /// at its lease means the host refused it (hook, permissions) and is an error. + fn delete_leased(&self, items: &[(String, String)]) -> Result<()> { + for (key, tag) in items { + check_key(key)?; + if !is_tag(tag) { + return Err(Error::Backend(format!( + "lease for {key:?} is not a full object id" + ))); + } + } + let mut rest = items; + while !rest.is_empty() { + let mut n = 0; + let mut bytes = 0; + while let Some((key, _)) = rest.get(n) { + let cost = 2 * (NS.len() + key.len()) + 64; + if n > 0 && (n >= DELETE_BATCH || bytes + cost > DELETE_ARG_BUDGET) { + break; + } + bytes += cost; + n += 1; + } + let (chunk, tail) = rest.split_at(n); + rest = tail; + { + let mut st = self.lock(); + for (key, _) in chunk { + st.remove(key); + } + } + let leases: Vec = chunk + .iter() + .map(|(k, t)| format!("--force-with-lease={NS}{k}:{t}")) + .collect(); + let specs: Vec = chunk.iter().map(|(k, _)| format!(":{NS}{k}")).collect(); + let mut a = vec!["push", "--porcelain", "--no-verify"]; + a.extend(leases.iter().map(String::as_str)); + a.extend(["--end-of-options", &self.url]); + a.extend(specs.iter().map(String::as_str)); + let out = self.run(&a, None, &[], SMALL_CAP)?; + if out.success { + continue; + } + let first = out + .stdout + .split(|b| *b == b'\n') + .find(|l| l.first() == Some(&b'!')) + .map(|l| self.scrub(&String::from_utf8_lossy(l))) + .unwrap_or_default(); + let refusal = format!( + "git push failed (exit {}): {} {}", + out.code.map_or("signal".to_string(), |c| c.to_string()), + out.stderr, + first + ); + // Cannot re-read (network): report the push failure itself. + let now: HashMap = self + .remote_list("") + .map_err(|_| Error::Backend(refusal.clone()))? + .into_iter() + .collect(); + if chunk.iter().any(|(k, t)| now.get(k) == Some(t)) { + return Err(Error::Backend(refusal)); + } + } + Ok(()) + } } #[cfg(test)] @@ -1275,7 +1358,6 @@ mod tests { GitBucket::open_with(url, git.to_string(), CHUNK).unwrap() } - #[cfg(unix)] fn tag_of(b: &GitBucket, key: &str) -> String { let Fetched::New { tag, .. } = b.get(key, None).unwrap() else { panic!("expected New") @@ -2106,4 +2188,187 @@ mod tests { assert!(measured[1].1 <= 6, "{measured:?}"); assert!(measured[1].2 <= 4, "{measured:?}"); } + + /// Keys of a listing, in order. + fn names(v: &[(String, String)]) -> Vec<&str> { + v.iter().map(|(k, _)| k.as_str()).collect() + } + + #[test] + fn list_tags_reports_the_tag_get_reports_without_downloading() { + need_git!(); + let t = Tmp::new("tags-list"); + let (_, url) = hub(&t); + let b = open(&url); + for k in ["log/1", "logger/x", "manifest", "packs/a.pack"] { + b.put_new(k, k.as_bytes()).unwrap(); + } + let r = open(&url); // fresh cache: nothing is downloaded to list + let tags = r.list_tags("log").unwrap(); + assert_eq!(names(&tags), ["log/1", "logger/x"]); + for (k, tag) in &r.list_tags("").unwrap() { + assert_eq!(*tag, tag_of(&b, k), "{k}"); + } + assert!(r.lock().is_empty(), "a listing must not fetch values"); + assert!(matches!(r.list_tags("a b"), Err(Error::BadKey(_)))); + // a CAS moves the tag the listing reports + let old = tag_of(&b, "manifest"); + let new = b + .put_if_tag("manifest", b"v2", Some(&old)) + .unwrap() + .unwrap(); + let tags = r.list_tags("manifest").unwrap(); + assert_eq!(tags, vec![("manifest".to_string(), new)]); + } + + #[test] + fn delete_leased_removes_only_keys_still_at_their_lease() { + need_git!(); + let t = Tmp::new("leased"); + let (hubdir, url) = hub(&t); + let b = open(&url); + for k in ["keep", "log/1", "log/2", "log/3"] { + assert!(b.put_new(k, k.as_bytes()).unwrap()); + } + let listed = b.list_tags("").unwrap(); + // After the listing someone else replaces log/2 and re-creates log/3. + let other = open(&url); + let lease = |k: &str| listed.iter().find(|(n, _)| n == k).unwrap().1.clone(); + let new2 = other + .put_if_tag("log/2", b"theirs", Some(&lease("log/2"))) + .unwrap() + .unwrap(); + other.delete("log/3").unwrap(); + assert!(other.put_new("log/3", b"recreated").unwrap()); + let new3 = tag_of(&other, "log/3"); + // A key that was never there, with a plausible lease, is not an error. + let mut items: Vec<(String, String)> = listed + .iter() + .filter(|(k, _)| k.starts_with("log/")) + .cloned() + .collect(); + items.push(("log/9".into(), "1".repeat(40))); + b.delete_leased(&items).unwrap(); + + let now = open(&url).list_tags("").unwrap(); + assert_eq!( + now, + vec![ + ("keep".to_string(), lease("keep")), + ("log/2".to_string(), new2), + ("log/3".to_string(), new3), + ] + ); + let Fetched::New { bytes, .. } = open(&url).get("log/3", None).unwrap() else { + panic!() + }; + assert_eq!(bytes, b"recreated"); + assert!(!git(&hubdir, &["for-each-ref", "refs/sc/log/1"], "").contains("log/1")); + // Replaying the same call is idempotent. + b.delete_leased(&items).unwrap(); + assert_eq!(b.list("").unwrap(), vec!["keep", "log/2", "log/3"]); + b.delete_leased(&[]).unwrap(); + } + + #[test] + fn delete_leased_refuses_a_bad_key_or_lease_before_any_push() { + need_git!(); + let t = Tmp::new("leased-bad"); + let (_, url) = hub(&t); + let b = open(&url); + b.put_new("log/1", b"x").unwrap(); + let good = tag_of(&b, "log/1"); + for lease in ["", "zz", "A".repeat(40).as_str(), "0".repeat(41).as_str()] { + let r = b.delete_leased(&[("log/1".into(), lease.into())]); + assert!(matches!(r, Err(Error::Backend(_))), "{lease:?}"); + } + for key in ["a..b", "a b", "", "x.lock", "/a", "log/*"] { + let r = b.delete_leased(&[(key.into(), good.clone())]); + assert!(matches!(r, Err(Error::BadKey(_))), "{key:?}"); + } + // One bad item anywhere refuses the whole call: nothing was deleted. + let r = b.delete_leased(&[("log/1".into(), good.clone()), ("a..b".into(), good)]); + assert!(r.is_err()); + assert_eq!(b.list("").unwrap(), vec!["log/1"]); + } + + #[cfg(unix)] + #[test] + fn delete_leased_reports_a_ref_the_host_refuses_and_still_removes_the_rest() { + use std::os::unix::fs::PermissionsExt; + need_git!(); + let t = Tmp::new("leased-hook"); + let (hubdir, url) = hub(&t); + let b = open(&url); + for k in ["log/1", "log/2", "log/3"] { + b.put_new(k, k.as_bytes()).unwrap(); + } + let items = b.list_tags("log/").unwrap(); + // An `update` hook judges each ref on its own: it refuses log/2 only. + let hook = hubdir.join("hooks/update"); + std::fs::write( + &hook, + "#!/bin/sh\ncase \"$1\" in refs/sc/log/2) echo 'keep log two' >&2; exit 1;; esac\nexit 0\n", + ) + .unwrap(); + std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap(); + let e = b.delete_leased(&items).unwrap_err().to_string(); + assert!(e.contains("keep log two"), "{e}"); + // Not mistaken for a lost race: log/2 is still at its lease, the others went. + assert_eq!(names(&open(&url).list_tags("").unwrap()), ["log/2"]); + // Once the host allows it, the same call finishes the job. + std::fs::remove_file(&hook).unwrap(); + b.delete_leased(&items).unwrap(); + assert!(b.list("").unwrap().is_empty()); + } + + #[cfg(unix)] + #[test] + fn delete_leased_batches_with_one_lease_per_ref_and_never_plus_or_atomic() { + need_git!(); + let t = Tmp::new("leased-batch"); + let (hubdir, url) = hub(&t); + plant(&hubdir, "seed", false, &[("100644", "blob", "c0000")]); + let c = git(&hubdir, &["rev-parse", "refs/sc/seed"], ""); + const N: usize = 300; + let cmds: String = (0..N) + .map(|i| format!("create refs/sc/log/{i:04} {c}\n")) + .collect(); + git(&hubdir, &["update-ref", "--stdin"], &cmds); + let log = t.0.join("pushes.log"); + let sh = shim( + &t, + "pushes.sh", + &format!( + "for x in \"$@\"; do if [ \"$x\" = push ]; then echo \"$*\" >> {}; break; fi; done", + log.display() + ), + ); + let b = open_git(&url, &sh); + let items: Vec<(String, String)> = b + .list_tags("log/") + .unwrap() + .into_iter() + .map(|(k, _)| (k, c.clone())) + .collect(); + assert_eq!(items.len(), N); + b.delete_leased(&items).unwrap(); + assert!(b.list("log/").unwrap().is_empty()); + let text = std::fs::read_to_string(&log).unwrap(); + let pushes: Vec<&str> = text.lines().collect(); + assert!(pushes.len() >= N.div_ceil(DELETE_BATCH), "{}", pushes.len()); + let mut leases = 0; + for p in &pushes { + assert!(p.len() < 32 * 1024, "command line of {} bytes", p.len()); + assert!(!p.contains("--atomic"), "{p}"); + for tok in p.split(' ') { + assert!(!tok.starts_with('+'), "plus refspec: {tok}"); + if let Some(l) = tok.strip_prefix("--force-with-lease=refs/sc/log/") { + assert!(l.ends_with(&format!(":{c}")), "{tok}"); + leases += 1; + } + } + } + assert_eq!(leases, N, "one lease per ref"); + } } diff --git a/crates/objio/src/lib.rs b/crates/objio/src/lib.rs index d43e5ba3..39145547 100644 --- a/crates/objio/src/lib.rs +++ b/crates/objio/src/lib.rs @@ -76,6 +76,29 @@ pub trait Bucket: Send { fn exists(&self, key: &str) -> Result; /// Remove `key`. Idempotent: an absent key is success. fn delete(&self, key: &str) -> Result<()>; + /// Keys starting with the raw string `prefix`, each with the tag + /// [`Bucket::get`] reports for its current value, sorted by key, from one + /// listing and no value downloads. A backend that cannot list tags cheaply + /// keeps this default error: compaction asks only a Git carrier for it, + /// where the tag is the ref's object id and the listing already holds it. + fn list_tags(&self, _prefix: &str) -> Result> { + Err(Error::Backend( + "this backend cannot list keys with their tags".into(), + )) + } + /// Remove each `(key, expected_tag)` whose key still holds that tag. A key + /// that now holds another value (or is already gone) is left alone and is + /// not an error; a key that still holds its expected tag but could not be + /// removed is an error. Several keys go in one call so a backend can batch + /// them. This default has no lease to check, so it deletes each key + /// unconditionally (what the backends without conditional deletes do); a + /// backend that can lease must override it (the Git carrier does). + fn delete_leased(&self, items: &[(String, String)]) -> Result<()> { + for (key, _) in items { + self.delete(key)?; + } + Ok(()) + } } /// Reject traversal and absolute keys before any backend touches them. @@ -167,6 +190,24 @@ pub fn contract_suite(b: &dyn Bucket) { assert_eq!(b.list_meta("packs/").unwrap().len(), 2); // a deleted key can be created again assert!(b.put_new("packs/ab12.pack", b"again").unwrap()); + // delete_leased: removes keys holding their expected tag, is idempotent, + // touches only its keys. Whether a stale tag survives is up to the backend + // (the default cannot lease), so that is checked in each lease-capable + // backend's own tests. + assert!(b.put_new("lease/a", b"1").unwrap()); + assert!(b.put_new("lease/b", b"2").unwrap()); + assert!(b.put_new("lease/keep", b"3").unwrap()); + let tag = |k: &str| match b.get(k, None).unwrap() { + Fetched::New { tag, .. } => tag, + _ => panic!("{k} should exist"), + }; + let (ta, tb) = (tag("lease/a"), tag("lease/b")); + b.delete_leased(&[("lease/a".into(), ta.clone()), ("lease/b".into(), tb)]) + .unwrap(); + assert_eq!(b.list("lease/").unwrap(), vec!["lease/keep".to_string()]); + b.delete_leased(&[("lease/a".into(), ta)]).unwrap(); + b.delete_leased(&[]).unwrap(); + assert_eq!(b.list("lease/").unwrap(), vec!["lease/keep".to_string()]); } #[cfg(test)] From 05dad781f888cfd99f5d57afcc3bc0423db49086 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 14:50:30 +0200 Subject: [PATCH 2/7] feat(repo): compact sc+git:: remotes with a first-sighting age record A Git host reports no trustworthy object age (committer time is writer-controlled), so v1 refused 'sc remote compact' there. Compaction now keeps its own clock. Age source: refs/sc/compact/sightings holds ' ' per packs/ and checkpoints/ key, generation-aware by oid, strictly decoded (a bad record, or a time over five minutes ahead of the compactor, blocks the run before anything is written). It supplies ages only; the candidates still come from the listing minus the live chain and retained structure. It is written with a CAS before any delete decision uses it; a dry run never writes it. Sizes: the view keeps each live idx's declared pack length, so the merge plan never calls list_meta (which downloads every value on a carrier); a pack far over what its idx declares is left unmerged. Deletes: leased to the oid the one listing showed (Bucket::delete_leased), idx before bodies. Before every destructive batch, on every backend, the manifest is read with no cache and its tag compared (a carrier answers a conditional read from its 2 s window with no network call). The v1 refusals (repo compact(), refuse_git_compact, the CLI check) are removed. Tests run over a real GitBucket on file:// hubs with an injected clock and cover two runs, backdated, malformed, future-dated and manifest- naming records, dry run, a re-created key, a push landing before the manifest check, a key re-created before its leased delete, two compactors racing on the record, crashes at each delete call, a refusing host, and every operation boundary over an in-memory double with Git's tag semantics. The CLI scenario compacts twice and then clones. --- crates/cli/src/main.rs | 11 +- crates/cli/tests/git_bucket.rs | 52 +- crates/repo/src/bucket_transport.rs | 53 +- crates/repo/src/bucket_transport/compact.rs | 414 ++++-- .../src/bucket_transport/compact/sightings.rs | 332 +++++ .../src/bucket_transport/compact/tests.rs | 2 + .../src/bucket_transport/compact/tests/git.rs | 1166 +++++++++++++++++ .../src/bucket_transport/git_remote_tests.rs | 24 +- crates/repo/src/lib.rs | 4 +- 9 files changed, 1915 insertions(+), 143 deletions(-) create mode 100644 crates/repo/src/bucket_transport/compact/sightings.rs create mode 100644 crates/repo/src/bucket_transport/compact/tests/git.rs diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 52e1dd06..48a36efe 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -915,16 +915,20 @@ enum RemoteOp { }, /// List configured remotes. List, - /// Compact a bucket remote (`sc+wal://`, `sc+s3://`; not `sc+git::` yet): merge small packs, + /// Compact a bucket remote (`sc+wal://`, `sc+s3://`, `sc+git::`): merge small packs, /// then delete log entries, checkpoints and packs no reader or writer can - /// still need (ADR-0057). Needs bucket list and delete rights. + /// still need (ADR-0057). Needs bucket list and delete rights (on a Git + /// repository, push access). Run it from one host. Compact { name: String, /// Plan and report; write and delete nothing. #[arg(long)] dry_run: bool, /// Delete only what has been unreferenced for at least this long - /// (e.g. 24h, 7d). Keep it well above any push's duration. + /// (e.g. 24h, 7d). Keep it well above any push's duration. On an + /// `sc+git::` remote the age is counted from this command's first + /// sighting of an object, so the first run deletes nothing it did not + /// already see (with a zero grace, a first sighting is immediately old). #[arg(long, default_value = "24h")] prune_expire: String, /// Allow a --prune-expire under 1h. Only safe while nothing pushes to @@ -4584,7 +4588,6 @@ fn run_remote(op: RemoteOp) -> Result<()> { .find(|(n, _)| *n == name) .map(|(_, u)| u) .ok_or_else(|| anyhow::anyhow!("no such remote: {name}"))?; - scl_repo::refuse_git_compact(&url)?; if !scl_repo::is_bucket_url(&url) { anyhow::bail!("remote '{name}' is not a bucket remote ({url}); nothing to compact"); } diff --git a/crates/cli/tests/git_bucket.rs b/crates/cli/tests/git_bucket.rs index 4f78cb57..1c3c656e 100644 --- a/crates/cli/tests/git_bucket.rs +++ b/crates/cli/tests/git_bucket.rs @@ -141,21 +141,12 @@ fn sc_git_is_never_a_git_bridge_remote_and_the_opt_in_is_sc_git_only() { std::fs::remove_dir_all(&a).unwrap(); } -/// Deferred in v1: `sc remote compact` and `sc serve --store` refuse an -/// `sc+git::` store with a clear message, before any I/O. +/// Deferred in v1: `sc serve --store` refuses an `sc+git::` store with a clear +/// message, before any I/O. #[test] -fn compact_and_serve_store_refuse_sc_git() { +fn serve_store_refuses_sc_git() { let a = tmp("deferred"); init_repo(&a); - // Written by hand so the test does not depend on GitBucket validation. - std::fs::write( - a.join(".sc/config"), - "[remote.origin]\nurl = \"sc+git::file:///nonexistent/carrier.git\"\n", - ) - .unwrap(); - let e = err_text(&sc(&a, &["remote", "compact", "origin", "--dry-run"])); - assert!(e.contains("does not support sc+git::"), "{e}"); - let home = tmp("deferred-home"); init_repo(&home); let e = err_text(&sc( @@ -385,6 +376,43 @@ fn scenario(fx: Fixture, gated: bool) { } ok(sc(a, &["switch", "main", "--identity", idp])); + // Compact the same remote twice, a grace window apart; the clones below then + // prove everything still reads. On a Git carrier the first run only records + // when it first saw each object, so it deletes nothing and the second run + // (past the grace) is the one that removes refs. + let remote = if gated { "opted" } else { "origin" }; + let compact = [ + "remote", + "compact", + remote, + "--prune-expire", + "1s", + "--allow-short-grace", + ]; + let first = ok(sc(a, &compact)); + let mid = refs_on_carrier(&fx.carrier).lines().count(); + if gated { + assert!(first.contains("first sighting"), "{first}"); + assert!( + first.contains("deleted 0 log entries, 0 checkpoint(s), 0 pack(s)"), + "{first}" + ); + assert!( + refs_on_carrier(&fx.carrier).contains("refs/sc/compact/sightings"), + "the age record is on the host" + ); + } + std::thread::sleep(std::time::Duration::from_millis(1500)); + let second = ok(sc(a, &compact)); + if gated { + assert!(!second.contains("deleted 0 log entries"), "{second}"); + let end = refs_on_carrier(&fx.carrier).lines().count(); + assert!( + end < mid, + "compaction removed refs from the host: {mid} -> {end}" + ); + } + // Keyless clone: public content and the branch marker only. let (keyless, out) = clone_into(&fx, "keyless", None, &[]); assert_eq!(notice(&out), gated); diff --git a/crates/repo/src/bucket_transport.rs b/crates/repo/src/bucket_transport.rs index afb99399..0f4d4371 100644 --- a/crates/repo/src/bucket_transport.rs +++ b/crates/repo/src/bucket_transport.rs @@ -42,6 +42,11 @@ struct WalView { /// is a pure copy: it never needs to re-walk the log or re-derive the /// pack list, just snapshot this field into the new checkpoint object. packs: Vec, + /// pack hash -> the pack body length its idx declares ([`declared_pack_len`]), + /// parsed per idx (not from `index`, where the last pack wins an id). A + /// planning hint only: it lets compaction size packs on a Git carrier + /// without downloading them, and the idx is writer-controlled. + pack_sizes: BTreeMap, } /// A [`Transport`] whose object graph and refs live entirely in an @@ -93,8 +98,9 @@ pub struct BucketTransport { /// Tag of the manifest the in-progress `refresh_once` is building from, so /// a failed attempt can say which manifest it failed on. attempt_tag: RefCell>, - /// Opened from an `sc+git::` URL: compaction is refused here, not only in - /// the CLI (a Git host gives it no trustworthy pack age, ADR-0060). + /// Opened from an `sc+git::` URL: a Git host reports no trustworthy object + /// age, so compaction keeps its own record of when it first saw each object + /// (`compact/sightings`) and deletes with leases (ADR-0057 addendum). git_carrier: bool, } @@ -142,6 +148,24 @@ fn capped_to(what: &str, bytes: Vec, max: usize) -> Result> { Ok(bytes) } +/// Bytes of pack body an idx describes: the end of its furthest record, which +/// starts at `offset` with a 4-byte length prefix (`scl_core::pack`), or the +/// 8-byte header of an empty pack. The idx is writer-controlled, so this is a +/// hint for planning, never a bound: arithmetic that overflows saturates, so a +/// hostile idx can only make a pack look large (never a merge candidate). +fn declared_pack_len(entries: &[IndexEntry]) -> u64 { + entries + .iter() + .map(|e| { + e.offset + .checked_add(4) + .and_then(|end| end.checked_add(e.length)) + .unwrap_or(u64::MAX) + }) + .max() + .unwrap_or(8) +} + /// Fold a checkpoint once the log tail exceeds this many entries past the /// last checkpoint (spec: "default 64 entries, one tunable constant"). const CHECKPOINT_INTERVAL: u64 = 64; @@ -219,24 +243,6 @@ pub fn refuse_git_store(url: &str) -> Result<()> { Ok(()) } -/// `sc remote compact` on an `sc+git::` remote is refused (deferred): the -/// compactor has no trustworthy age source for refs in a Git repository, so -/// it could not tell a stale pack from one an in-flight push just uploaded. -pub fn refuse_git_compact(url: &str) -> Result<()> { - if is_git_bucket_url(url) { - return Err(git_compact_refusal()); - } - Ok(()) -} - -fn git_compact_refusal() -> Error { - Error::InvalidArgument( - "sc remote compact does not support sc+git:: remotes yet (deferred): a Git \ - repository gives the compactor no trustworthy age for stored packs" - .into(), - ) -} - /// A parsed `sc+wal://`, `sc+s3:///` or /// `sc+git::` remote URL. /// `sc+wal` treats everything after the scheme as a directory path (so @@ -493,12 +499,15 @@ impl BucketTransport { // Index build: over the FULL cumulative pack list (checkpoint // packs + tail packs), fetching each idx exactly as today. let mut index = BTreeMap::new(); + let mut pack_sizes = BTreeMap::new(); for hash in &packs { let Fetched::New { bytes, .. } = self.bucket.get(&idx_key(hash), None)? else { return Err(self.gone(format!("pack {hash} on chain but idx absent"))); }; let bytes = capped("pack idx", bytes)?; - for IndexEntry { id, offset, length } in parse_index(&bytes)? { + let entries = parse_index(&bytes)?; + pack_sizes.insert(hash.clone(), declared_pack_len(&entries)); + for IndexEntry { id, offset, length } in entries { index.insert(id, (hash.clone(), offset, length)); } } @@ -513,6 +522,7 @@ impl BucketTransport { refs, index, packs, + pack_sizes, }); Ok(()) } @@ -2378,6 +2388,7 @@ mod tests { refs, index: BTreeMap::new(), packs: Vec::new(), + pack_sizes: BTreeMap::new(), }); // The triggering push's own commit already landed (that's what put diff --git a/crates/repo/src/bucket_transport/compact.rs b/crates/repo/src/bucket_transport/compact.rs index 2e57b963..2595a5c5 100644 --- a/crates/repo/src/bucket_transport/compact.rs +++ b/crates/repo/src/bucket_transport/compact.rs @@ -8,13 +8,24 @@ //! ever deleted that the current manifest references, and the manifest itself //! is never deleted, so a failure between any two operations leaves a bucket a //! cold start reads. +//! +//! On a Git carrier (`sc+git::`) the host reports no trustworthy object age, so +//! the sweep takes ages from the first-sighting record (`sightings`) instead of +//! the bucket's modification times, sizes packs from their idx instead of +//! downloading them, and deletes with leases (`Bucket::delete_leased`) so a key +//! someone re-created after the listing survives (ADR-0057 addendum). + +mod sightings; -use super::{capped, BucketTransport}; +use super::{capped, declared_pack_len, BucketTransport}; use crate::error::{Error, Result}; -use crate::walfmt::{checkpoint_key, log_key, pack_key, Checkpoint, LogEntry, Manifest, MAX_LIST}; -use scl_core::pack::parse_pack_reader; +use crate::walfmt::{ + checkpoint_key, idx_key, log_key, pack_key, Checkpoint, LogEntry, Manifest, MAX_LIST, +}; +use scl_core::pack::{parse_index, parse_pack_reader}; use scl_core::ObjectId; use scl_objio::{Fetched, ObjectMeta}; +use sightings::Sightings; use std::collections::{BTreeMap, BTreeSet}; use std::time::{Duration, SystemTime}; @@ -84,6 +95,13 @@ fn seq_of(key: &str) -> Option { key.rsplit('/').next()?.parse().ok() } +/// Whole seconds since the Unix epoch, the resolution of the age record. +fn unix_secs(t: SystemTime) -> Result { + t.duration_since(SystemTime::UNIX_EPOCH) + .map(|d| d.as_secs()) + .map_err(|_| Error::InvalidArgument("the clock is before 1970".into())) +} + /// `packs/.pack` / `packs/.idx` -> ``. fn hash_of(key: &str) -> Option<&str> { let name = key.strip_prefix("packs/")?; @@ -91,14 +109,32 @@ fn hash_of(key: &str) -> Option<&str> { .or_else(|| name.strip_suffix(".idx")) } -/// Bytes of the `.pack` and `.idx` objects behind these packs. -fn pack_bytes(packs: &[(&String, &Vec)]) -> u64 { +/// Bytes of the `.pack` and `.idx` objects behind these packs; a pack in +/// `declared` (a Git carrier, which lists no sizes) counts that many instead. +fn pack_bytes(packs: &[(&String, &Vec)], declared: &BTreeMap) -> u64 { packs .iter() - .flat_map(|(_, ms)| ms.iter().map(|m| m.size)) + .map(|(h, ms)| { + declared + .get(*h) + .copied() + .unwrap_or_else(|| ms.iter().map(|m| m.size).sum()) + }) .sum() } +/// What one sweep attempt sees in the bucket. +struct Listing { + ckpts: Vec<(u64, ObjectMeta)>, + logs: Vec<(u64, ObjectMeta)>, + /// `.pack` and `.idx` objects by pack hash. + by_hash: BTreeMap>, + /// Git carrier only: the tag each listed key held in this very listing, the + /// lease its delete is conditional on (the oid whose age was judged is the + /// oid deleted; nothing lists twice). + tags: BTreeMap, +} + /// Greedy size-tiered grouping: smallest candidates first, a group closes when /// the next pack would push it past `group_bytes`. Groups of one are dropped /// (there is nothing to merge). @@ -131,16 +167,20 @@ fn plan_groups( impl BucketTransport { /// Compact this bucket remote (ADR-0057): merge small packs, then delete /// log entries, checkpoints and packs no reader or writer can still need. - /// Safe to run while others push and fetch; run one at a time (concurrent - /// runs stay correct but waste work). Needs bucket list and delete rights. + /// Safe to run while others push and fetch; run one at a time, from one + /// host (concurrent runs stay correct but waste work, and on a Git carrier + /// runs that share the age record need clocks within far less than the grace + /// window). Needs bucket list and delete rights (on a Git repository, push + /// access). pub fn compact(&self, opts: &CompactOptions) -> Result { - if self.git_carrier { - return Err(super::git_compact_refusal()); - } let mut rep = CompactReport { dry_run: opts.dry_run, ..CompactReport::default() }; + if self.git_carrier { + // An unusable age record stops the run before phase 1 writes anything. + self.read_sightings(unix_secs(opts.now.unwrap_or_else(SystemTime::now))?)?; + } self.refresh()?; if self.view.borrow().is_none() { // No manifest yet: nothing to merge, but a first push that died after @@ -179,13 +219,23 @@ impl BucketTransport { .collect(); (live, v.refs.len()) }; - let sizes: BTreeMap = self - .bucket - .list_meta("packs/")? - .into_iter() - .filter(|m| m.key.ends_with(".pack")) - .filter_map(|m| hash_of(&m.key).map(|h| (h.to_string(), m.size))) - .collect(); + // A Git carrier's `list_meta` downloads every value, so its sizes + // come from the idx files the view already parsed (a hint: the idx is + // writer-controlled, which the body check below backs up). + let sizes: BTreeMap = if self.git_carrier { + self.view + .borrow() + .as_ref() + .map(|v| v.pack_sizes.clone()) + .unwrap_or_default() + } else { + self.bucket + .list_meta("packs/")? + .into_iter() + .filter(|m| m.key.ends_with(".pack")) + .filter_map(|m| hash_of(&m.key).map(|h| (h.to_string(), m.size))) + .collect() + }; let groups = plan_groups(&live, &sizes, opts); if groups.is_empty() { if live.len() > MAX_LIST { @@ -227,6 +277,17 @@ impl BucketTransport { missing_on = tag; continue 'attempt; }; + // The idx that sized this pack is the writer's word: a pack + // far over what it declared is not merged (nor decoded). + if self.git_carrier + && bytes.len() as u64 > opts.small_pack_bytes.saturating_mul(2) + { + rep.notes.push(format!( + "pack {hash} is {} bytes, far over what its idx declares: left unmerged", + bytes.len() + )); + continue; + } parse_pack_reader(std::io::Cursor::new(&bytes[..]), |id, obj| { let enc = obj.encode(); decoded += enc.len() as u64; @@ -396,7 +457,7 @@ impl BucketTransport { .checked_sub(opts.prune_expire) .unwrap_or(SystemTime::UNIX_EPOCH); let old = |m: &ObjectMeta| m.modified <= old_before; - for _ in 0..MAX_ATTEMPTS { + 'attempt: for _ in 0..MAX_ATTEMPTS { self.refresh()?; let (tag, ck_seq, live) = match self.view.borrow().as_ref() { Some(v) => ( @@ -406,18 +467,16 @@ impl BucketTransport { ), None => (None, 0, BTreeSet::new()), }; - let ckpts: Vec<(u64, ObjectMeta)> = self - .bucket - .list_meta("checkpoints/")? - .into_iter() - .filter_map(|m| seq_of(&m.key).map(|s| (s, m))) - .collect(); - let logs: Vec<(u64, ObjectMeta)> = self - .bucket - .list_meta("log/")? - .into_iter() - .filter_map(|m| seq_of(&m.key).map(|s| (s, m))) - .collect(); + // A lost race on the age record means someone else updated it: replan. + let Some(Listing { + ckpts, + logs, + by_hash, + tags, + }) = self.list_for_sweep(opts, now, rep)? + else { + continue; + }; let cutoff = ckpts .iter() .filter(|(s, m)| *s <= ck_seq && old(m)) @@ -442,14 +501,9 @@ impl BucketTransport { .map(|(_, m)| m) .collect(); - // Pack objects by hash; a hash is a candidate if the live chain - // does not name it and every one of its objects is old enough. - let mut by_hash: BTreeMap> = BTreeMap::new(); - for m in self.bucket.list_meta("packs/")? { - if let Some(h) = hash_of(&m.key) { - by_hash.entry(h.to_string()).or_default().push(m); - } - } + // A hash is a candidate if the live chain does not name it and every + // one of its objects (a lone `.pack` after an idx-first crash is + // one) is old enough. let unnamed: Vec<(&String, &Vec)> = by_hash.iter().filter(|(h, _)| !live.contains(*h)).collect(); let (aged, young): (Vec<_>, Vec<_>) = @@ -477,21 +531,23 @@ impl BucketTransport { // the manifest does meanwhile: the cutoff only ever grows, so they // stay unreachable. Counters add up across replans. rep.cutoff = rep.cutoff.max(cutoff); - rep.logs_deleted += del_logs.len(); - rep.checkpoints_deleted += del_ckpts.len(); - rep.bytes_deleted += del_logs - .iter() - .chain(del_ckpts.iter()) - .map(|m| m.size) - .sum::(); - if !opts.dry_run { - for m in del_logs.iter().chain(del_ckpts.iter()) { - self.bucket.delete(&m.key)?; - } - } - rep.packs_deleted += del_packs.len(); - rep.bytes_deleted += pack_bytes(&del_packs); + // Git carrier: pack sizes for the report, from the idx (the listing + // carries none); a pack with no readable idx counts as 0 bytes. + let declared = if self.git_carrier { + self.declared_sizes(&del_packs, rep) + } else { + BTreeMap::new() + }; if opts.dry_run { + rep.logs_deleted += del_logs.len(); + rep.checkpoints_deleted += del_ckpts.len(); + rep.bytes_deleted += del_logs + .iter() + .chain(del_ckpts.iter()) + .map(|m| m.size) + .sum::(); + rep.packs_deleted += del_packs.len(); + rep.bytes_deleted += pack_bytes(&del_packs, &declared); if rep.packs_created > 0 { rep.notes.push( "dry run: the sweep figures describe the bucket as it is now; the merge \ @@ -502,41 +558,57 @@ impl BucketTransport { } return Ok(()); } - // A pack nothing names can still be named by a push that lands - // while we plan: if the manifest moved, replan the pack set - // (nothing is deleted against a stale picture). - let unmoved = matches!( - (&tag, self.bucket.get("manifest", tag.as_deref())?), - (Some(_), Fetched::Unchanged) | (None, Fetched::Absent) - ); - if !unmoved { - rep.packs_deleted -= del_packs.len(); - rep.bytes_deleted -= pack_bytes(&del_packs); - continue; + // The manifest is read without any cache before anything is deleted + // (a Git carrier answers a conditional read from its freshness + // window without asking the host): a push that landed since this + // attempt read the view may name what it would delete. + if !(del_logs.is_empty() && del_ckpts.is_empty()) { + if !self.manifest_unmoved(&tag)? { + continue; + } + rep.logs_deleted += del_logs.len(); + rep.checkpoints_deleted += del_ckpts.len(); + rep.bytes_deleted += del_logs + .iter() + .chain(del_ckpts.iter()) + .map(|m| m.size) + .sum::(); + let keys: Vec<&str> = del_logs + .iter() + .chain(del_ckpts.iter()) + .map(|m| m.key.as_str()) + .collect(); + self.delete_keys(&keys, &tags)?; } - // The delete loop can run long (one request per object), so look at - // the manifest again between batches: a push that lands meanwhile - // may have re-used one of these old packs byte for byte. + // A pack nothing names can still be named by a push that lands + // while we plan, and the delete loop can run long, so the manifest + // is looked at before every batch: if it moved before the first, the + // plan is stale (replan); later, what is left is kept. A push that + // lands meanwhile may have re-used one of these old packs byte for + // byte. for (i, batch) in del_packs.chunks(PACK_DELETE_BATCH).enumerate() { - if i > 0 - && !matches!( - (&tag, self.bucket.get("manifest", tag.as_deref())?), - (Some(_), Fetched::Unchanged) | (None, Fetched::Absent) - ) - { - let left = del_packs.len() - i * PACK_DELETE_BATCH; - rep.packs_deleted -= left; - rep.bytes_deleted -= pack_bytes(&del_packs[i * PACK_DELETE_BATCH..]); + if !self.manifest_unmoved(&tag)? { + if i == 0 { + continue 'attempt; + } rep.notes.push(format!( - "the manifest moved during pack deletion: {left} unreferenced pack(s) \ - kept, rerun" + "the manifest moved during pack deletion: {} unreferenced pack(s) kept, \ + rerun", + del_packs.len() - i * PACK_DELETE_BATCH )); return Ok(()); } - for (hash, _) in batch { - // idx first: a half-deleted pack is then plain debris. - self.bucket.delete(&format!("packs/{hash}.idx"))?; - self.bucket.delete(&pack_key(hash))?; + rep.packs_deleted += batch.len(); + rep.bytes_deleted += pack_bytes(batch, &declared); + // idx first: a half-deleted pack is then plain debris. + for suffix in [".idx", ".pack"] { + let keys: Vec<&str> = batch + .iter() + .flat_map(|(_, ms)| ms.iter()) + .map(|m| m.key.as_str()) + .filter(|k| k.ends_with(suffix)) + .collect(); + self.delete_keys(&keys, &tags)?; } } return Ok(()); @@ -547,6 +619,184 @@ impl BucketTransport { Ok(()) } + /// Whether the manifest is still the one `tag` names (`None`: still absent), + /// read with no cache so the answer comes from the bucket itself. + fn manifest_unmoved(&self, tag: &Option) -> Result { + Ok(match (tag, self.bucket.get("manifest", None)?) { + (Some(t), Fetched::New { tag: now, .. }) => *t == now, + (None, Fetched::Absent) => true, + _ => false, + }) + } + + /// Delete `keys`. On a Git carrier each goes with the lease its listing + /// gave it (`tags`), so a key that was re-created since survives; elsewhere + /// it is a plain delete. A key with no lease is an error, never an + /// unleased delete. + fn delete_keys(&self, keys: &[&str], tags: &BTreeMap) -> Result<()> { + if !self.git_carrier { + for key in keys { + self.bucket.delete(key)?; + } + return Ok(()); + } + let items = keys + .iter() + .map(|k| match tags.get(*k) { + Some(t) => Ok((k.to_string(), t.clone())), + None => Err(Error::Wal(format!( + "no lease for {k}: refusing to delete it" + ))), + }) + .collect::>>()?; + Ok(self.bucket.delete_leased(&items)?) + } + + /// Pack body sizes of `packs` as their idx declares them (a Git carrier's + /// listing carries no sizes), for the report only. A pack whose idx cannot be + /// read is left out and counts as 0 bytes; the report says so. + fn declared_sizes( + &self, + packs: &[(&String, &Vec)], + rep: &mut CompactReport, + ) -> BTreeMap { + let mut sizes = BTreeMap::new(); + for (hash, _) in packs { + if let Ok(Fetched::New { bytes, .. }) = self.bucket.get(&idx_key(hash), None) { + if let Ok(entries) = parse_index(&bytes) { + sizes.insert((*hash).clone(), declared_pack_len(&entries)); + } + } + } + let note = "Git carrier: byte counts are the pack bodies their idx declares; log entries, \ + checkpoints and idx files count as 0, as does a pack with no readable idx"; + if !packs.is_empty() && !rep.notes.iter().any(|n| n == note) { + rep.notes.push(note.into()); + } + sizes + } + + /// The age record as the bucket holds it, with the tag a compare-and-swap + /// write must lead with (`None`: no record yet). A record that does not + /// decode is an error, never "everything is old". + fn read_sightings(&self, now_secs: u64) -> Result<(Sightings, Option)> { + match self.bucket.get(sightings::KEY, None)? { + Fetched::Absent => Ok((Sightings::default(), None)), + Fetched::New { bytes, tag } => Ok(( + Sightings::decode(&capped("age record", bytes)?, now_secs)?, + Some(tag), + )), + Fetched::Unchanged => Err(Error::Wal( + "age record: an unconditional read answered Unchanged".into(), + )), + } + } + + /// What this attempt sees in the bucket. On a Git carrier it is one listing + /// with each key's tag (no value is downloaded), aged by the first-sighting + /// record: the record is brought up to this listing and written back (when + /// it changed, and not for a dry run) BEFORE any decision uses it, so a run + /// at least the grace window after a key was first seen is the earliest that + /// can delete it. `None` means the record's compare-and-swap lost to another + /// compactor: replan. + fn list_for_sweep( + &self, + opts: &CompactOptions, + now: SystemTime, + rep: &mut CompactReport, + ) -> Result> { + if !self.git_carrier { + let metas = |prefix: &str| -> Result> { + Ok(self + .bucket + .list_meta(prefix)? + .into_iter() + .filter_map(|m| seq_of(&m.key).map(|s| (s, m))) + .collect()) + }; + let mut by_hash: BTreeMap> = BTreeMap::new(); + for m in self.bucket.list_meta("packs/")? { + if let Some(h) = hash_of(&m.key) { + by_hash.entry(h.to_string()).or_default().push(m); + } + } + return Ok(Some(Listing { + ckpts: metas("checkpoints/")?, + logs: metas("log/")?, + by_hash, + tags: BTreeMap::new(), + })); + } + let now_secs = unix_secs(now)?; + let listed = self.bucket.list_tags("")?; + let (before, record_tag) = self.read_sightings(now_secs)?; + let seen = before.observe(&listed, now_secs); + let added = seen.added_since(&before); + if seen != before { + if opts.dry_run { + if added > 0 { + rep.notes.push(format!( + "dry run: {added} object(s) not seen before would be recorded by a real \ + run; ages here are a lower bound" + )); + } + } else if self + .bucket + .put_if_tag(sightings::KEY, &seen.encode()?, record_tag.as_deref())? + .is_none() + { + return Ok(None); + } else if added > 0 { + let note = format!( + "Git carrier: first sighting of {added} object(s) recorded at \ + refs/sc/{}; an object is deleted only by a run at least the grace window \ + after that", + sightings::KEY + ); + if !rep.notes.contains(¬e) { + rep.notes.push(note); + } + } + } + let aged = |key: &str| { + SystemTime::UNIX_EPOCH + Duration::from_secs(seen.first_seen(key).unwrap_or(now_secs)) + }; + let (mut ckpts, mut logs) = (Vec::new(), Vec::new()); + let mut by_hash: BTreeMap> = BTreeMap::new(); + let mut tags = BTreeMap::new(); + for (key, oid) in listed { + let meta = |modified| ObjectMeta { + key: key.clone(), + size: 0, + modified, + }; + if key.starts_with("checkpoints/") { + if let Some(s) = seq_of(&key) { + ckpts.push((s, meta(aged(&key)))); + } + } else if key.starts_with("log/") { + // Never aged: log entries go by sequence behind the cutoff. + if let Some(s) = seq_of(&key) { + logs.push((s, meta(now))); + } + } else if let Some(h) = hash_of(&key) { + by_hash + .entry(h.to_string()) + .or_default() + .push(meta(aged(&key))); + } else { + continue; + } + tags.insert(key, oid); + } + Ok(Some(Listing { + ckpts, + logs, + by_hash, + tags, + })) + } + /// Pack hashes named by structure the sweep keeps: checkpoints with /// `seq >= cutoff` and log entries with `seq > cutoff` (everything when /// `cutoff` is 0). Any record that fails to decode aborts: deleting packs diff --git a/crates/repo/src/bucket_transport/compact/sightings.rs b/crates/repo/src/bucket_transport/compact/sightings.rs new file mode 100644 index 00000000..6de796e7 --- /dev/null +++ b/crates/repo/src/bucket_transport/compact/sightings.rs @@ -0,0 +1,332 @@ +//! The first-sighting record: compaction's age source on a Git carrier +//! (ADR-0057 addendum, ADR-0060). +//! +//! A Git host reports no usable object age: the only time is a commit's +//! committer date, which the writer chooses and which predates publication. So +//! on a carrier the compactor keeps its own clock. Bucket key +//! `compact/sightings` (ref `refs/sc/compact/sightings`) holds one line per +//! tracked key, ` \n`, sorted by key, where +//! `oid` is the value's tag (a Git object id) when the compactor first listed it +//! and `first_seen` is the compactor's own clock then. A key whose listed oid +//! differs from the recorded one is a new generation (a deleted key that was +//! uploaded again) and starts its age over. Only `packs/` and `checkpoints/` +//! keys are tracked: log entries are deleted by sequence behind the cutoff, +//! never by age. +//! +//! The record supplies ages and nothing else: it never names something to +//! delete. Anyone who can push can write it, and so can already delete the +//! store directly, so it adds no authority; a malformed or future-dated record +//! is refused, never read as "everything is old". + +use crate::error::{Error, Result}; +use std::collections::BTreeMap; +use std::fmt::Display; + +/// Bucket key of the record. +pub(super) const KEY: &str = "compact/sightings"; + +/// How far past the reader's own clock a recorded first-seen time may lie +/// before the record is refused: compactors that share it must keep clocks +/// within far less than the grace window of each other. +const MAX_FUTURE_SECS: u64 = 5 * 60; + +/// Largest record read or written: the metadata cap of every non-pack value. +const MAX_BYTES: usize = scl_core::DEFAULT_MAX_OBJECT_SIZE; + +/// First sightings by key: `(first_seen_unix_secs, oid)`. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub(super) struct Sightings { + seen: BTreeMap, +} + +/// An unusable record. It blocks compaction (nothing is deleted on a guess), +/// and the message names the way out. +fn unusable(why: impl Display) -> Error { + Error::Wal(format!( + "the compaction age record refs/sc/{KEY} is unusable ({why}); delete that ref on the \ + Git host and rerun (every object's age then starts over)" + )) +} + +/// Whether `key` is one the record may name: a `packs/` or `checkpoints/` key +/// that the Git carrier itself would accept (its ref-safe charset and shape; +/// a listed key always passes, so a record this module wrote always decodes). +fn tracked(key: &str) -> bool { + (key.starts_with("packs/") || key.starts_with("checkpoints/")) + && key.len() <= 1024 + && !key.contains("..") + && key + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'/' | b'-')) + && key.split('/').all(|c| { + !c.is_empty() && !c.starts_with('.') && !c.ends_with('.') && !c.ends_with(".lock") + }) +} + +/// A full lowercase hex object id (40 digits). +fn is_oid(s: &str) -> bool { + s.len() == 40 && s.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) +} + +impl Sightings { + /// Strict canonical decode of a record read at `now` (the reader's clock, + /// unix seconds). Anything outside the canonical form is an error: plain + /// decimal seconds, a 40-digit lowercase oid, a tracked key, keys sorted and + /// unique, a final newline, and no first-seen time more than five minutes + /// ahead of `now` (compactor clocks disagree). + pub(super) fn decode(bytes: &[u8], now: u64) -> Result { + let text = std::str::from_utf8(bytes).map_err(|_| unusable("not UTF-8"))?; + let mut seen = BTreeMap::new(); + if text.is_empty() { + return Ok(Sightings { seen }); + } + let body = text + .strip_suffix('\n') + .ok_or_else(|| unusable("no final newline"))?; + let mut prev: Option<&str> = None; + for (i, line) in body.split('\n').enumerate() { + let at = |why: &str| unusable(format!("line {}: {why}", i + 1)); + let mut f = line.splitn(3, ' '); + let (Some(secs), Some(oid), Some(key)) = (f.next(), f.next(), f.next()) else { + return Err(at("expected ` `")); + }; + let first_seen: u64 = secs.parse().map_err(|_| at("bad seconds"))?; + if first_seen.to_string() != secs { + return Err(at("seconds are not plain decimal")); + } + if !is_oid(oid) { + return Err(at("oid is not 40 lowercase hex digits")); + } + if !tracked(key) { + return Err(at("key is not a tracked packs/ or checkpoints/ key")); + } + if prev.is_some_and(|p| p >= key) { + return Err(at("keys are not sorted and unique")); + } + if first_seen > now.saturating_add(MAX_FUTURE_SECS) { + return Err(Error::Wal(format!( + "compactor clocks disagree: the age record refs/sc/{KEY} has a first sighting \ + at {first_seen}, more than five minutes after this clock ({now}). Run \ + compaction from one host, or fix the clocks; to discard the record, delete \ + that ref on the Git host" + ))); + } + prev = Some(key); + seen.insert(key.to_string(), (first_seen, oid.to_string())); + } + Ok(Sightings { seen }) + } + + /// Canonical bytes, or an error when the record would pass the size cap + /// (it is never truncated: dropping entries would reset ages). + pub(super) fn encode(&self) -> Result> { + self.encode_within(MAX_BYTES) + } + + fn encode_within(&self, max: usize) -> Result> { + let mut out = String::new(); + for (key, (first_seen, oid)) in &self.seen { + out.push_str(&format!("{first_seen} {oid} {key}\n")); + if out.len() > max { + return Err(Error::Wal(format!( + "the compaction age record would exceed {max} bytes ({} tracked keys)", + self.seen.len() + ))); + } + } + Ok(out.into_bytes()) + } + + /// The record after this listing, taken at `now`: a tracked listed key keeps + /// its first-seen time while its oid is the recorded one, and starts at + /// `now` when it is unrecorded or holds a different oid; a key no longer + /// listed is dropped. Untracked keys are ignored. + pub(super) fn observe(&self, listed: &[(String, String)], now: u64) -> Sightings { + let seen = listed + .iter() + .filter(|(key, _)| tracked(key)) + .map(|(key, oid)| { + let first_seen = match self.seen.get(key) { + Some((t, recorded)) if recorded == oid => *t, + _ => now, + }; + (key.clone(), (first_seen, oid.clone())) + }) + .collect(); + Sightings { seen } + } + + /// When `key` was first seen, if it is recorded. + pub(super) fn first_seen(&self, key: &str) -> Option { + self.seen.get(key).map(|(t, _)| *t) + } + + /// The recorded keys, sorted (for tests). + #[cfg(test)] + pub(super) fn keys(&self) -> Vec<&str> { + self.seen.keys().map(String::as_str).collect() + } + + /// How many keys are recorded here that `before` does not hold at the same + /// oid: the sightings a run adds. + pub(super) fn added_since(&self, before: &Sightings) -> usize { + self.seen + .iter() + .filter(|(k, v)| before.seen.get(*k) != Some(*v)) + .count() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const A: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + const B: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + + fn rec(lines: &[(&str, u64, &str)]) -> Sightings { + Sightings { + seen: lines + .iter() + .map(|(k, t, o)| (k.to_string(), (*t, o.to_string()))) + .collect(), + } + } + + #[test] + fn a_record_round_trips_in_one_canonical_form() { + let r = rec(&[ + ("checkpoints/00000000000000000065", 10, A), + ("packs/ab12.idx", 20, B), + ("packs/ab12.pack", 0, A), + ]); + let bytes = r.encode().unwrap(); + assert_eq!( + String::from_utf8(bytes.clone()).unwrap(), + format!( + "10 {A} checkpoints/00000000000000000065\n20 {B} packs/ab12.idx\n0 {A} packs/ab12.pack\n" + ) + ); + assert_eq!(Sightings::decode(&bytes, 100).unwrap(), r); + assert_eq!(Sightings::decode(b"", 100).unwrap(), Sightings::default()); + assert!(Sightings::default().encode().unwrap().is_empty()); + } + + #[test] + fn anything_but_the_canonical_form_is_an_error_that_names_the_fix() { + let ok = format!("5 {A} packs/x.pack\n"); + Sightings::decode(ok.as_bytes(), 100).unwrap(); + let upper = A.to_uppercase(); + let short = &A[..39]; + let long = format!("{A}a"); + let bad: Vec<(String, &str)> = vec![ + (format!("5 {A} packs/x.pack"), "no final newline"), + (format!("5 {A} packs/x.pack\n\n"), "blank line"), + (format!("05 {A} packs/x.pack\n"), "leading zero"), + (format!("+5 {A} packs/x.pack\n"), "plus sign"), + (format!("-5 {A} packs/x.pack\n"), "negative"), + (format!("5.0 {A} packs/x.pack\n"), "fraction"), + (format!(" 5 {A} packs/x.pack\n"), "space"), + ( + format!("99999999999999999999999 {A} packs/x.pack\n"), + "overflow", + ), + (format!("5 {upper} packs/x.pack\n"), "uppercase oid"), + (format!("5 {short} packs/x.pack\n"), "short oid"), + (format!("5 {long} packs/x.pack\n"), "long oid"), + ("5 zz packs/x.pack\n".to_string(), "non-hex oid"), + (format!("5 {A}\n"), "no key"), + (format!("5 {A} manifest\n"), "manifest is not tracked"), + ( + format!("5 {A} log/00000000000000000001\n"), + "logs are not tracked", + ), + ( + format!("5 {A} compact/sightings\n"), + "the record is not tracked", + ), + (format!("5 {A} packs/../x\n"), "dotdot"), + (format!("5 {A} packs/a b\n"), "space in key"), + (format!("5 {A} packs/.x\n"), "hidden"), + (format!("5 {A} packs/x.lock\n"), "lock suffix"), + (format!("5 {A} packs//x\n"), "empty component"), + (format!("5 {A} packs/é\n"), "non-ascii key"), + ( + format!("5 {A} packs/b.pack\n5 {A} packs/a.pack\n"), + "unsorted", + ), + ( + format!("5 {A} packs/a.pack\n6 {B} packs/a.pack\n"), + "duplicate", + ), + (format!("5 {A} packs/x.pack\r\n"), "crlf"), + ]; + for (text, why) in &bad { + let e = Sightings::decode(text.as_bytes(), 100) + .expect_err(why) + .to_string(); + assert!( + e.contains("unusable") && e.contains("delete that ref"), + "{why}: {e}" + ); + } + let e = Sightings::decode(&[0xff, 0xfe], 100) + .unwrap_err() + .to_string(); + assert!(e.contains("not UTF-8"), "{e}"); + } + + #[test] + fn a_first_sighting_in_the_future_is_refused_beyond_five_minutes() { + let at = |t: u64| format!("{t} {A} packs/x.pack\n"); + // within the allowance (a compactor whose clock runs a little ahead) + Sightings::decode(at(100 + MAX_FUTURE_SECS).as_bytes(), 100).unwrap(); + let e = Sightings::decode(at(100 + MAX_FUTURE_SECS + 1).as_bytes(), 100) + .unwrap_err() + .to_string(); + assert!(e.contains("clocks disagree"), "{e}"); + assert!(e.contains("delete that ref"), "{e}"); + } + + #[test] + fn observe_keeps_ages_per_generation_and_drops_what_is_gone() { + let before = rec(&[ + ("packs/a.idx", 10, A), + ("packs/b.pack", 20, A), + ("packs/gone.pack", 30, A), + ]); + let listed = vec![ + ("log/00000000000000000001".to_string(), A.to_string()), + ("manifest".to_string(), A.to_string()), + ("packs/a.idx".to_string(), A.to_string()), // same oid: keeps its age + ("packs/b.pack".to_string(), B.to_string()), // re-created: a new generation + ("packs/new.pack".to_string(), B.to_string()), // first sighting + ]; + let after = before.observe(&listed, 1000); + assert_eq!( + after, + rec(&[ + ("packs/a.idx", 10, A), + ("packs/b.pack", 1000, B), + ("packs/new.pack", 1000, B), + ]) + ); + assert_eq!(after.first_seen("packs/a.idx"), Some(10)); + assert_eq!(after.first_seen("manifest"), None); + assert_eq!(after.added_since(&before), 2); + assert_eq!( + after.observe(&listed, 5000), + after, + "unchanged listing is a fixed point" + ); + } + + #[test] + fn a_record_past_the_cap_is_an_error_not_a_truncation() { + let r = rec(&[("packs/a.pack", 1, A), ("packs/b.pack", 2, A)]); + let full = r.encode().unwrap(); + assert_eq!(r.encode_within(full.len()).unwrap(), full); + let e = r.encode_within(full.len() - 1).unwrap_err().to_string(); + assert!(e.contains("would exceed"), "{e}"); + } +} diff --git a/crates/repo/src/bucket_transport/compact/tests.rs b/crates/repo/src/bucket_transport/compact/tests.rs index 97cc032f..a587459b 100644 --- a/crates/repo/src/bucket_transport/compact/tests.rs +++ b/crates/repo/src/bucket_transport/compact/tests.rs @@ -1032,3 +1032,5 @@ fn candidates_that_are_each_large_once_decoded_are_not_merged_one_by_one() { t2.get_object(&tip).unwrap(); } } + +mod git; diff --git a/crates/repo/src/bucket_transport/compact/tests/git.rs b/crates/repo/src/bucket_transport/compact/tests/git.rs new file mode 100644 index 00000000..1b228fa0 --- /dev/null +++ b/crates/repo/src/bucket_transport/compact/tests/git.rs @@ -0,0 +1,1166 @@ +//! Compaction on a Git carrier (ADR-0057 addendum, ADR-0060): the first-sighting +//! age record, leased deletes, and the races and crashes around them. Most tests +//! run over a real `GitBucket` on a `file://` bare hub with an injected clock +//! (`CompactOptions.now`); the exhaustive failure-boundary loop and the +//! idx-sizing test run over an in-memory double with Git's tag semantics (a new +//! tag for every write, real lease checks), which is fast enough to cut at every +//! operation. +//! +//! Every wrapper here forwards `list_tags` and `delete_leased` explicitly: a +//! wrapper that did not would silently get the trait defaults, an unleased +//! delete, and a lease test through it would pass without testing the lease. + +use super::super::sightings::{self, Sightings}; +use super::*; +use scl_objio::GitBucket; +use std::path::Path; +use std::process::Command; +use std::sync::atomic::AtomicUsize; + +fn have_git() -> bool { + Command::new("git") + .arg("--version") + .output() + .is_ok_and(|o| o.status.success()) +} + +macro_rules! need_git { + () => { + if !have_git() { + eprintln!("git not available; skipping"); + return; + } + }; +} + +const GRACE: Duration = Duration::from_secs(3600); +const DAY: Duration = Duration::from_secs(24 * 3600); + +/// The wall clock in whole seconds, the resolution the record keeps. +fn t0() -> SystemTime { + let secs = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + SystemTime::UNIX_EPOCH + Duration::from_secs(secs) +} + +fn secs(t: SystemTime) -> u64 { + t.duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs() +} + +/// A run at `now` with the grace of one hour. +fn at(now: SystemTime) -> CompactOptions { + CompactOptions { + prune_expire: GRACE, + now: Some(now), + ..CompactOptions::default() + } +} + +// ---- a bare Git hub --------------------------------------------------------- + +/// A bare hub repository on disk, removed on drop. +struct Hub { + dir: PathBuf, +} + +fn git_in(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env_remove("GIT_DIR") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_NOSYSTEM", "1") + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +fn copy_dir(from: &Path, to: &Path) { + std::fs::create_dir_all(to).unwrap(); + for e in std::fs::read_dir(from).unwrap().flatten() { + let (p, t) = (e.path(), to.join(e.file_name())); + if p.is_dir() { + copy_dir(&p, &t); + } else { + std::fs::copy(&p, &t).unwrap(); + } + } +} + +impl Hub { + fn new(tag: &str) -> Hub { + let root = scratch(tag); + std::fs::create_dir_all(&root).unwrap(); + let dir = root.join("hub.git"); + git_in(&root, &["init", "--bare", "-q", dir.to_str().unwrap()]); + Hub { dir } + } + + /// An independent copy of this hub (a new compaction experiment from the same state). + fn copy(&self, tag: &str) -> Hub { + let root = scratch(tag); + let dir = root.join("hub.git"); + copy_dir(&self.dir, &dir); + Hub { dir } + } + + fn url(&self) -> String { + format!("file://{}", self.dir.display()) + } + + /// A compactor/reader over this hub as an `sc+git::` remote. + fn carrier(&self) -> BucketTransport { + BucketTransport::open(&format!("sc+git::{}", self.url())).unwrap() + } + + /// A compactor whose bucket calls pass through `hook(op, key, after)`. + fn hooked(&self, hook: Hook) -> BucketTransport { + BucketTransport::build( + Box::new(Wrapped { + inner: Box::new(GitBucket::open(&self.url()).unwrap()), + hook, + }), + true, + ) + .unwrap() + } + + /// `refs/sc/*` names on the hub, sorted. + fn refs(&self) -> Vec { + git_in( + &self.dir, + &["for-each-ref", "--format=%(refname)", "refs/sc/"], + ) + .lines() + .map(str::to_string) + .collect() + } + + /// `refs/sc/*` with their object ids, sorted. + fn refs_with_oids(&self) -> Vec { + git_in( + &self.dir, + &[ + "for-each-ref", + "--format=%(refname) %(objectname)", + "refs/sc/", + ], + ) + .lines() + .map(str::to_string) + .collect() + } + + fn count(&self, needle: &str) -> usize { + self.refs().iter().filter(|r| r.contains(needle)).count() + } + + /// The age record, if there is one. + fn record(&self) -> Option { + match self.bucket().get(sightings::KEY, None).unwrap() { + Fetched::New { bytes, .. } => Some(Sightings::decode(&bytes, u64::MAX / 2).unwrap()), + _ => None, + } + } +} + +impl Store for Hub { + fn bucket(&self) -> Box { + Box::new(GitBucket::open(&self.url()).unwrap()) + } +} + +impl Drop for Hub { + fn drop(&mut self) { + if let Some(root) = self.dir.parent() { + let _ = std::fs::remove_dir_all(root); + if !std::thread::panicking() { + assert!(!root.exists(), "scratch dir must be removed"); + } + } + } +} + +/// 12 one-commit pushes with a checkpoint at 10 on a fresh hub, and the refs +/// they hold. +fn seeded_hub(tag: &str) -> (Hub, Vec<(String, ObjectId)>) { + let hub = Hub::new(tag); + let refs = hand_built(&hub, 12, 10); + (hub, refs) +} + +/// An orphan pack: uploaded, never committed. Its hash is the one a push of +/// `commit(label)` uploads, so a later push reuses it byte for byte. +fn orphan(s: &impl Store, label: &str) -> String { + let (_, objects) = commit(label); + let (pack, _) = scl_core::pack::build_pack(&objects).unwrap(); + let hash = hex::encode(blake3::hash(&pack).as_bytes()); + open(s).put_pack(&mut std::io::Cursor::new(pack)).unwrap(); + hash +} + +/// The refs a store is left with once everything unreferenced is gone: the +/// manifest, the one checkpoint, the one merged pack, and the age record. +fn tight(hub: &Hub) -> Vec { + let refs = hub.refs(); + assert_eq!(refs.len(), 5, "{refs:?}"); + assert_eq!(hub.count("refs/sc/checkpoints/"), 1); + assert_eq!(hub.count("refs/sc/packs/"), 2); + assert_eq!(hub.count("refs/sc/log/"), 0); + refs +} + +// ---- wrappers and doubles --------------------------------------------------- + +/// `(operation, key or prefix, after)`: called before (`false`) and after +/// (`true`) each bucket operation; an error stops it. +type Hook = Box scl_objio::Result<()> + Send>; + +/// Forwards every operation, including `list_tags` and `delete_leased`, through +/// a hook. `list_meta` is refused: it downloads every value on a Git carrier, so +/// compaction must never ask for it. +struct Wrapped { + inner: Box, + hook: Hook, +} + +impl Wrapped { + fn around( + &self, + op: &str, + key: &str, + f: impl FnOnce() -> scl_objio::Result, + ) -> scl_objio::Result { + (self.hook)(op, key, false)?; + let out = f()?; + (self.hook)(op, key, true)?; + Ok(out) + } +} + +impl Bucket for Wrapped { + fn get(&self, key: &str, tag: Option<&str>) -> scl_objio::Result { + self.around("get", key, || self.inner.get(key, tag)) + } + fn put_new(&self, key: &str, bytes: &[u8]) -> scl_objio::Result { + self.around("put_new", key, || self.inner.put_new(key, bytes)) + } + fn put_if_tag( + &self, + key: &str, + bytes: &[u8], + tag: Option<&str>, + ) -> scl_objio::Result> { + self.around("put_if_tag", key, || self.inner.put_if_tag(key, bytes, tag)) + } + fn list(&self, prefix: &str) -> scl_objio::Result> { + self.around("list", prefix, || self.inner.list(prefix)) + } + fn list_meta(&self, _prefix: &str) -> scl_objio::Result> { + Err(scl_objio::Error::Backend( + "list_meta downloads every value on a git carrier and must not be called".into(), + )) + } + fn exists(&self, key: &str) -> scl_objio::Result { + self.around("exists", key, || self.inner.exists(key)) + } + fn delete(&self, key: &str) -> scl_objio::Result<()> { + self.around("delete", key, || self.inner.delete(key)) + } + fn list_tags(&self, prefix: &str) -> scl_objio::Result> { + self.around("list_tags", prefix, || self.inner.list_tags(prefix)) + } + fn delete_leased(&self, items: &[(String, String)]) -> scl_objio::Result<()> { + let first = items.first().map_or("", |(k, _)| k.as_str()); + self.around("delete_leased", first, || self.inner.delete_leased(items)) + } +} + +/// In-memory bucket with Git's tag semantics: every write gets a new tag (a +/// commit with a nonce), so a key that is deleted and uploaded again is a new +/// generation even with identical bytes, and `delete_leased` honors its lease. +#[derive(Default)] +struct GitLike { + map: Mutex, String)>>, + writes: AtomicU64, +} + +impl GitLike { + fn tag_for(&self, bytes: &[u8]) -> String { + let n = self.writes.fetch_add(1, Ordering::SeqCst); + let mut h = blake3::Hasher::new(); + h.update(bytes); + h.update(&n.to_le_bytes()); + hex::encode(h.finalize().as_bytes())[..40].to_string() + } + + fn fork(&self) -> Arc { + Arc::new(GitLike { + map: Mutex::new(self.map.lock().unwrap().clone()), + writes: AtomicU64::new(self.writes.load(Ordering::SeqCst)), + }) + } +} + +struct LikeHandle(Arc); + +impl Bucket for LikeHandle { + fn get(&self, key: &str, cached: Option<&str>) -> scl_objio::Result { + Ok(match self.0.map.lock().unwrap().get(key) { + None => Fetched::Absent, + Some((_, t)) if cached == Some(t.as_str()) => Fetched::Unchanged, + Some((b, t)) => Fetched::New { + bytes: b.clone(), + tag: t.clone(), + }, + }) + } + fn put_new(&self, key: &str, bytes: &[u8]) -> scl_objio::Result { + let mut m = self.0.map.lock().unwrap(); + if m.contains_key(key) { + return Ok(false); + } + let tag = self.0.tag_for(bytes); + m.insert(key.into(), (bytes.to_vec(), tag)); + Ok(true) + } + fn put_if_tag( + &self, + key: &str, + bytes: &[u8], + expected: Option<&str>, + ) -> scl_objio::Result> { + let mut m = self.0.map.lock().unwrap(); + if m.get(key).map(|(_, t)| t.as_str()) != expected { + return Ok(None); + } + let tag = self.0.tag_for(bytes); + m.insert(key.into(), (bytes.to_vec(), tag.clone())); + Ok(Some(tag)) + } + fn list(&self, prefix: &str) -> scl_objio::Result> { + Ok(self + .list_tags(prefix)? + .into_iter() + .map(|(k, _)| k) + .collect()) + } + fn list_meta(&self, _prefix: &str) -> scl_objio::Result> { + Err(scl_objio::Error::Backend( + "list_meta must not be called on a git carrier".into(), + )) + } + fn exists(&self, key: &str) -> scl_objio::Result { + Ok(self.0.map.lock().unwrap().contains_key(key)) + } + fn delete(&self, key: &str) -> scl_objio::Result<()> { + self.0.map.lock().unwrap().remove(key); + Ok(()) + } + fn list_tags(&self, prefix: &str) -> scl_objio::Result> { + Ok(self + .0 + .map + .lock() + .unwrap() + .iter() + .filter(|(k, _)| k.starts_with(prefix)) + .map(|(k, (_, t))| (k.clone(), t.clone())) + .collect()) + } + fn delete_leased(&self, items: &[(String, String)]) -> scl_objio::Result<()> { + let mut m = self.0.map.lock().unwrap(); + for (key, tag) in items { + if m.get(key).is_some_and(|(_, t)| t == tag) { + m.remove(key); + } + } + Ok(()) + } +} + +impl Store for Arc { + fn bucket(&self) -> Box { + Box::new(LikeHandle(self.clone())) + } +} + +fn like_carrier(s: &Arc) -> BucketTransport { + BucketTransport::build(s.bucket(), true).unwrap() +} + +/// Push `commit(label)` to branch `label` through a fresh transport on `hub`. +fn push_label(hub: &Hub, label: &str) -> (String, ObjectId) { + let t = hub.carrier(); + let (tip, objects) = commit(label); + for (id, bytes) in &objects { + t.put_object(id, bytes).unwrap(); + } + t.update_ref(label, &tip, None).unwrap(); + (label.to_string(), tip) +} + +// ---- the age record gates every delete -------------------------------------- + +#[test] +fn the_first_run_deletes_nothing_it_never_saw_and_a_run_past_the_grace_deletes_what_aged() { + need_git!(); + let (hub, refs) = seeded_hub("git-two-runs"); + let start = t0(); + let before = hub.refs(); + + // Run 1: phase 1 merges the 12 small packs; the sweep sees everything for the first time. + let r1 = hub.carrier().compact(&at(start)).unwrap(); + assert_eq!((r1.packs_merged, r1.packs_created), (12, 1), "{r1:?}"); + assert_eq!( + (r1.logs_deleted, r1.checkpoints_deleted, r1.packs_deleted), + (0, 0, 0), + "{r1:?}" + ); + assert_eq!(r1.packs_held_back, 12, "{r1:?}"); + assert!( + r1.notes.iter().any(|n| n.contains("first sighting")), + "{r1:?}" + ); + let after1 = hub.refs(); + for r in &before { + assert!(after1.contains(r), "{r} vanished on the first run"); + } + // The record names every pack and checkpoint key, first seen now, and no log or manifest. + let rec = hub.record().expect("the first run wrote the record"); + let tracked: Vec<&str> = after1 + .iter() + .map(|r| r.strip_prefix("refs/sc/").unwrap()) + .filter(|k| k.starts_with("packs/") || k.starts_with("checkpoints/")) + .collect(); + assert_eq!(rec.keys(), tracked); + assert_eq!(tracked.len(), 2 * 13 + 2, "13 packs and 2 checkpoints"); + for k in &tracked { + assert_eq!(rec.first_seen(k), Some(secs(start)), "{k}"); + } + assert_readable(&hub, &refs); + + // Run 2, two days later: everything unreferenced has aged. + let r2 = hub.carrier().compact(&at(start + 2 * DAY)).unwrap(); + assert_eq!( + (r2.logs_deleted, r2.checkpoints_deleted, r2.packs_deleted), + (13, 1, 12), + "{r2:?}" + ); + assert_eq!(r2.cutoff, 13, "{r2:?}"); + assert!(r2.bytes_deleted > 0, "{r2:?}"); + let left = tight(&hub); + assert!(left.contains(&"refs/sc/compact/sightings".to_string())); + assert!(left.contains(&"refs/sc/manifest".to_string())); + assert_readable(&hub, &refs); + + // Run 3 deletes nothing more and drops the vanished keys from the record. + let r3 = hub.carrier().compact(&at(start + 2 * DAY)).unwrap(); + assert_eq!( + (r3.logs_deleted, r3.checkpoints_deleted, r3.packs_deleted), + (0, 0, 0), + "{r3:?}" + ); + assert_eq!( + hub.record().unwrap().keys().len(), + 3, + "one pack pair and one checkpoint" + ); + assert_eq!(tight(&hub), left); + // The remote still takes pushes and the next one reads back. + let mut all = refs; + all.push(push_label(&hub, "after")); + assert_readable(&hub, &all); +} + +#[test] +fn a_record_that_calls_everything_ancient_deletes_only_what_nothing_names() { + need_git!(); + let (hub, refs) = seeded_hub("git-backdated"); + let start = t0(); + hub.carrier().compact(&at(start)).unwrap(); + let live_before = hub.refs(); + // Anyone who can push can write this record. They can delete directly too, so it + // adds no authority; but it must never make a live object deletable. + let b = hub.bucket(); + let Fetched::New { bytes, tag } = b.get(sightings::KEY, None).unwrap() else { + panic!("no record") + }; + let text = String::from_utf8(bytes).unwrap(); + let backdated: String = text + .lines() + .map(|l| format!("0 {}\n", l.split_once(' ').unwrap().1)) + .collect(); + b.put_if_tag(sightings::KEY, backdated.as_bytes(), Some(&tag)) + .unwrap() + .unwrap(); + + // The very next run, at the same instant, takes the backdated ages at their word. + let rep = hub.carrier().compact(&at(start)).unwrap(); + assert_eq!(rep.packs_deleted, 12, "{rep:?}"); + tight(&hub); + // The live merged pack, the current checkpoint and the manifest survived it. + for needle in ["refs/sc/manifest", "refs/sc/checkpoints/"] { + assert!(live_before.iter().any(|r| r.contains(needle))); + assert!(hub.refs().iter().any(|r| r.contains(needle)), "{needle}"); + } + assert_readable(&hub, &refs); +} + +#[test] +fn a_record_that_names_the_manifest_or_a_log_is_refused_and_nothing_is_deleted() { + need_git!(); + let (hub, refs) = seeded_hub("git-names-manifest"); + let start = t0(); + hub.carrier().compact(&at(start)).unwrap(); + let before = hub.refs_with_oids(); + let b = hub.bucket(); + let Fetched::New { bytes, tag } = b.get(sightings::KEY, None).unwrap() else { + panic!("no record") + }; + let text = String::from_utf8(bytes).unwrap(); + // `manifest` sorts between `checkpoints/` and `packs/`; put it there so the + // only thing wrong with the record is that it names the manifest. + let mut lines: Vec = text.lines().map(str::to_string).collect(); + let pos = lines.iter().position(|l| l.contains(" packs/")).unwrap(); + lines.insert(pos, format!("0 {} manifest", "a".repeat(40))); + let hostile_text = lines.join("\n") + "\n"; + b.put_if_tag(sightings::KEY, hostile_text.as_bytes(), Some(&tag)) + .unwrap() + .unwrap(); + let hostile = hub.refs_with_oids(); + let err = hub + .carrier() + .compact(&at(start + 2 * DAY)) + .unwrap_err() + .to_string(); + assert!( + err.contains("unusable") && err.contains("delete that ref"), + "{err}" + ); + assert_eq!( + hub.refs_with_oids(), + hostile, + "a refused record deletes nothing" + ); + assert_ne!(before, hostile); + assert_readable(&hub, &refs); +} + +#[test] +fn a_malformed_or_future_record_blocks_compaction_until_it_is_deleted() { + need_git!(); + let (hub, refs) = seeded_hub("git-malformed"); + let start = t0(); + let b = hub.bucket(); + assert!(b.put_new(sightings::KEY, b"garbage\n").unwrap()); + let planted = hub.refs_with_oids(); + let err = hub.carrier().compact(&at(start)).unwrap_err().to_string(); + assert!(err.contains("unusable"), "{err}"); + assert!(err.contains("refs/sc/compact/sightings"), "{err}"); + assert_eq!( + hub.refs_with_oids(), + planted, + "nothing was merged or deleted" + ); + + // A record from a clock an hour ahead is not read as "everything is young" either. + let Fetched::New { tag, .. } = b.get(sightings::KEY, None).unwrap() else { + panic!() + }; + let future = format!("{} {} packs/x.pack\n", secs(start) + 3600, "b".repeat(40)); + b.put_if_tag(sightings::KEY, future.as_bytes(), Some(&tag)) + .unwrap() + .unwrap(); + let err = hub.carrier().compact(&at(start)).unwrap_err().to_string(); + assert!(err.contains("clocks disagree"), "{err}"); + assert!(err.contains("delete that ref"), "{err}"); + assert_eq!(hub.count("refs/sc/log/"), 12); + + // The fix the message names works. + b.delete(sightings::KEY).unwrap(); + hub.carrier().compact(&at(start)).unwrap(); + assert!(hub.record().is_some()); + assert_readable(&hub, &refs); +} + +#[test] +fn a_dry_run_writes_no_record_and_reports_what_a_real_run_would_do() { + need_git!(); + let (hub, refs) = seeded_hub("git-dry"); + let start = t0(); + let before = hub.refs_with_oids(); + + // Before any real run: the plan merges and keeps everything; no record appears. + let dry = hub + .carrier() + .compact(&CompactOptions { + dry_run: true, + ..at(start) + }) + .unwrap(); + assert!(dry.dry_run); + assert_eq!((dry.packs_merged, dry.packs_created), (12, 1), "{dry:?}"); + assert_eq!(dry.packs_deleted, 0, "{dry:?}"); + assert!( + dry.notes.iter().any(|n| n.contains("lower bound")), + "{dry:?}" + ); + assert_eq!(hub.refs_with_oids(), before, "a dry run writes nothing"); + assert!(hub.record().is_none()); + + // After a real run, a dry run two days later reports the deletions the real run makes. + hub.carrier().compact(&at(start)).unwrap(); + let recorded = hub.refs_with_oids(); + let dry = hub + .carrier() + .compact(&CompactOptions { + dry_run: true, + ..at(start + 2 * DAY) + }) + .unwrap(); + assert_eq!( + (dry.logs_deleted, dry.checkpoints_deleted, dry.packs_deleted), + (13, 1, 12), + "{dry:?}" + ); + assert!(dry.bytes_deleted > 0); + assert_eq!( + hub.refs_with_oids(), + recorded, + "still nothing written or deleted" + ); + let real = hub.carrier().compact(&at(start + 2 * DAY)).unwrap(); + assert_eq!( + ( + real.logs_deleted, + real.checkpoints_deleted, + real.packs_deleted + ), + (13, 1, 12) + ); + assert_readable(&hub, &refs); +} + +// ---- generations, leases and the races around a delete ---------------------- + +#[test] +fn a_key_deleted_and_uploaded_again_starts_its_age_over() { + need_git!(); + let (hub, refs) = seeded_hub("git-recreated"); + let start = t0(); + let lone = orphan(&hub, "orphan-a"); + let keys = [format!("packs/{lone}.idx"), format!("packs/{lone}.pack")]; + let exists = |hub: &Hub| keys.iter().all(|k| hub.bucket().exists(k).unwrap()); + + hub.carrier().compact(&at(start)).unwrap(); + assert!(exists(&hub)); + let r2 = hub.carrier().compact(&at(start + 2 * DAY)).unwrap(); + assert!( + !keys.iter().any(|k| hub.bucket().exists(k).unwrap()), + "{r2:?}" + ); + + // A retried push uploads the same bytes again, without ever committing them. + assert_eq!(orphan(&hub, "orphan-a"), lone); + assert!(exists(&hub)); + // Past the old record's grace, but the key is a new generation: it survives. + let r3 = hub.carrier().compact(&at(start + 2 * DAY)).unwrap(); + assert!(exists(&hub), "{r3:?}"); + assert_eq!(r3.packs_held_back, 1, "{r3:?}"); + let rec = hub.record().unwrap(); + assert_eq!(rec.first_seen(&keys[1]), Some(secs(start + 2 * DAY))); + // It does age out like any other, from its own first sighting. + hub.carrier().compact(&at(start + 4 * DAY)).unwrap(); + assert!(!exists(&hub)); + assert_readable(&hub, &refs); +} + +#[test] +fn a_key_re_created_between_the_listing_and_the_delete_survives_its_lease() { + need_git!(); + let (hub, refs) = seeded_hub("git-lease"); + let start = t0(); + let lone = orphan(&hub, "orphan-b"); + let keys = [format!("packs/{lone}.idx"), format!("packs/{lone}.pack")]; + hub.carrier().compact(&at(start)).unwrap(); + + let url = hub.url(); + let fired = Arc::new(AtomicBool::new(false)); + let hook_keys = keys.to_vec(); + let hook: Hook = Box::new(move |op, key, after| { + // Just before the first pack delete: the key is deleted and uploaded + // again by someone else (the same bytes, a new commit, a new oid). + if op == "delete_leased" + && !after + && key.starts_with("packs/") + && !fired.swap(true, Ordering::SeqCst) + { + let other = GitBucket::open(&url).unwrap(); + for k in &hook_keys { + let Fetched::New { bytes, .. } = other.get(k, None)? else { + panic!("{k} should exist") + }; + other.delete(k)?; + assert!(other.put_new(k, &bytes)?); + } + } + Ok(()) + }); + let before: BTreeMap = hub + .bucket() + .list_tags("packs/") + .unwrap() + .into_iter() + .collect(); + let rep = hub.hooked(hook).compact(&at(start + 2 * DAY)).unwrap(); + // The other 12 old packs went; the re-created one did not, and nothing failed. + assert_eq!(hub.count("refs/sc/packs/"), 2 + 2, "{:?}", hub.refs()); + let after: BTreeMap = hub + .bucket() + .list_tags("packs/") + .unwrap() + .into_iter() + .collect(); + for k in &keys { + assert!(after.contains_key(k), "{k} was destroyed: {rep:?}"); + assert_ne!(after[k], before[k], "{k} should be the re-created value"); + } + assert_readable(&hub, &refs); +} + +/// The scenario of a pusher that reuses an aged orphan pack by identical bytes +/// and lands before the compactor looks at the manifest again. +#[test] +fn a_push_landing_after_the_listing_is_seen_by_the_manifest_check_and_its_pack_kept() { + need_git!(); + let (hub, mut refs) = seeded_hub("git-late-push"); + let start = t0(); + let lone = orphan(&hub, "late-0"); + let keys = [format!("packs/{lone}.idx"), format!("packs/{lone}.pack")]; + hub.carrier().compact(&at(start)).unwrap(); + + let url = hub.url(); + let state = Arc::new(AtomicUsize::new(0)); + let hook: Hook = Box::new(move |op, key, after| { + match (op, after, state.load(Ordering::SeqCst)) { + ("list_tags", true, 0) => state.store(1, Ordering::SeqCst), + // The first manifest read after the listing: a push has just landed. + ("get", false, 1) if key == "manifest" => { + state.store(2, Ordering::SeqCst); + let t = BucketTransport::open(&format!("sc+git::{url}")).unwrap(); + let (tip, objects) = commit("late-0"); + for (id, bytes) in &objects { + t.put_object(id, bytes).unwrap(); + } + t.update_ref("late-0", &tip, None).unwrap(); + } + _ => {} + } + Ok(()) + }); + let rep = hub.hooked(hook).compact(&at(start + 2 * DAY)).unwrap(); + for k in &keys { + assert!(hub.bucket().exists(k).unwrap(), "{k} was deleted: {rep:?}"); + } + refs.push(("late-0".to_string(), commit("late-0").0)); + assert_readable(&hub, &refs); + // The replan went on to remove what was still unreferenced. + assert!(rep.packs_deleted >= 12, "{rep:?}"); + assert_eq!( + hub.count("refs/sc/log/"), + 1, + "only the late push's entry is left" + ); +} + +#[test] +fn two_compactors_racing_on_the_record_converge() { + need_git!(); + let (hub, refs) = seeded_hub("git-race"); + let start = t0(); + let url = hub.url(); + let fired = Arc::new(AtomicBool::new(false)); + // Compactor A is about to write the record when compactor B runs to completion. + let hook: Hook = Box::new(move |op, key, after| { + if op == "put_if_tag" + && key == sightings::KEY + && !after + && !fired.swap(true, Ordering::SeqCst) + { + BucketTransport::open(&format!("sc+git::{url}")) + .unwrap() + .compact(&at(start)) + .unwrap(); + } + Ok(()) + }); + let rep = hub.hooked(hook).compact(&at(start)).unwrap(); + assert_eq!( + (rep.logs_deleted, rep.checkpoints_deleted, rep.packs_deleted), + (0, 0, 0), + "{rep:?}" + ); + // A's lost compare-and-swap replanned and read B's record: one consistent record. + let rec = hub.record().unwrap(); + for k in rec.keys() { + assert_eq!(rec.first_seen(k), Some(secs(start)), "{k}"); + } + assert_readable(&hub, &refs); + // Both agree that two days later it is all old. + let rep = hub.carrier().compact(&at(start + 2 * DAY)).unwrap(); + assert!(rep.packs_deleted > 0, "{rep:?}"); + tight(&hub); + assert_readable(&hub, &refs); +} + +#[test] +fn a_failure_between_the_record_and_each_delete_call_leaves_a_readable_store_a_rerun_finishes() { + need_git!(); + let (template, refs) = seeded_hub("git-crash"); + let start = t0(); + template.carrier().compact(&at(start)).unwrap(); + // Run 2 makes three delete calls: log entries and checkpoints, then the + // idx files of the aged packs, then their bodies. Fail before each one. + for fail_at in 0..3usize { + let hub = template.copy(&format!("git-crash-{fail_at}")); + let calls = Arc::new(AtomicUsize::new(0)); + let hook: Hook = Box::new(move |op, _, after| { + if op == "delete_leased" && !after && calls.fetch_add(1, Ordering::SeqCst) == fail_at { + return Err(scl_objio::Error::Backend("injected crash".into())); + } + Ok(()) + }); + let err = hub + .hooked(hook) + .compact(&at(start + 2 * DAY)) + .unwrap_err() + .to_string(); + assert!(err.contains("injected crash"), "{err}"); + // What the crash left is a bucket a cold start reads. + assert_readable(&hub, &refs); + match fail_at { + 0 => { + assert_eq!(hub.count("refs/sc/log/"), 13); + assert_eq!(hub.count("refs/sc/packs/"), 2 * 13); + } + 1 => { + assert_eq!(hub.count("refs/sc/log/"), 0); + assert_eq!(hub.count("refs/sc/packs/"), 2 * 13); + } + _ => { + // idx files gone, bodies left: plain debris, aged on its own next run + assert_eq!(hub.count(".idx"), 1); + assert_eq!(hub.count(".pack"), 13); + } + } + // A healthy rerun finishes the job. + hub.carrier().compact(&at(start + 2 * DAY)).unwrap(); + tight(&hub); + } +} + +#[cfg(unix)] +#[test] +fn a_host_that_refuses_the_deletes_fails_the_run_and_loses_nothing() { + use std::os::unix::fs::PermissionsExt; + need_git!(); + let (hub, refs) = seeded_hub("git-refused"); + let start = t0(); + hub.carrier().compact(&at(start)).unwrap(); + // An `update` hook that refuses to delete any pack body (a deletion pushes the zero oid). + let hook = hub.dir.join("hooks/update"); + std::fs::write( + &hook, + format!( + "#!/bin/sh\ncase \"$1\" in refs/sc/packs/*.pack) if [ \"$3\" = {} ]; then echo 'bodies are kept' >&2; exit 1; fi;; esac\nexit 0\n", + "0".repeat(40) + ), + ) + .unwrap(); + std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap(); + let err = hub + .carrier() + .compact(&at(start + 2 * DAY)) + .unwrap_err() + .to_string(); + assert!(err.contains("bodies are kept"), "{err}"); + assert_readable(&hub, &refs); + // The host relents: the rerun deletes the leftovers. + std::fs::remove_file(&hook).unwrap(); + hub.carrier().compact(&at(start + 2 * DAY)).unwrap(); + tight(&hub); + assert_readable(&hub, &refs); +} + +// ---- in-memory: every operation boundary, and sizes from the idx ------------ + +#[test] +fn a_failure_at_every_operation_boundary_on_a_git_carrier_leaves_a_readable_bucket() { + let template = Arc::new(GitLike::default()); + let refs = hand_built(&template, 12, 10); + let mut finished_at = None; + for budget in 0..5000i64 { + let work = template.fork(); + let left = Arc::new(AtomicI64::new(budget)); + let tick = left.clone(); + let hook: Hook = Box::new(move |_, _, after| { + if !after && tick.fetch_sub(1, Ordering::SeqCst) <= 0 { + return Err(scl_objio::Error::Backend("injected failure".into())); + } + Ok(()) + }); + let outcome = BucketTransport::build( + Box::new(Wrapped { + inner: work.bucket(), + hook, + }), + true, + ) + // A zero grace sees every first sighting as already old: one run deletes. + .and_then(|t| t.compact(&eager())); + + // Whatever happened, a fresh reader still sees everything. + assert_readable(&work, &refs); + if outcome.is_ok() && left.load(Ordering::SeqCst) > 0 { + finished_at = Some(budget); + break; + } + // And a healthy rerun finishes the job. + like_carrier(&work).compact(&eager()).unwrap(); + assert_readable(&work, &refs); + assert_eq!(keys(&work, "packs/").len(), 2, "budget {budget}"); + assert!(keys(&work, "log/").is_empty(), "budget {budget}"); + assert_eq!(keys(&work, "checkpoints/").len(), 1, "budget {budget}"); + } + let n = finished_at.expect("compaction never completed within the budget"); + assert!(n > 20, "the loop should cross many boundaries, got {n}"); +} + +/// The leased analogue of the pack-deletion stop on the plain backends: a push +/// lands while the first batch of packs is going, and the next batch's manifest +/// check keeps the rest. +#[test] +fn a_manifest_that_moves_during_leased_pack_deletion_stops_the_deletes() { + let mem = Arc::new(GitLike::default()); + let refs = push_commits(&open(&mem), "w", 70); + let fired = Arc::new(AtomicBool::new(false)); + let late = mem.clone(); + let hook: Hook = Box::new(move |op, key, after| { + // After the idx files of the first batch went: someone else pushes. + if op == "delete_leased" + && after + && key.starts_with("packs/") + && !fired.swap(true, Ordering::SeqCst) + { + push_commits(&open(&late), "late", 1); + } + Ok(()) + }); + let t = BucketTransport::build( + Box::new(Wrapped { + inner: mem.bucket(), + hook, + }), + true, + ) + .unwrap(); + let rep = t.compact(&eager()).unwrap(); + assert!( + rep.notes + .iter() + .any(|n| n.contains("moved during pack deletion")), + "{rep:?}" + ); + assert_eq!(rep.packs_deleted, PACK_DELETE_BATCH, "{rep:?}"); + assert!(keys(&mem, "packs/").len() > 4); + let mut all = refs; + all.push(("late-0".to_string(), commit("late-0").0)); + assert_readable(&mem, &all); + // A rerun finishes the job. + like_carrier(&mem).compact(&eager()).unwrap(); + assert_eq!(keys(&mem, "packs/").len(), 2); + assert_readable(&mem, &all); +} + +#[test] +fn the_double_leases_like_git_and_a_replayed_compaction_changes_nothing() { + // The double itself: a re-created key has a new tag and a stale lease leaves it. + let scratch_hub = Arc::new(GitLike::default()); + let b = scratch_hub.bucket(); + assert!(b.put_new("packs/a.pack", b"x").unwrap()); + let Fetched::New { tag: old, .. } = b.get("packs/a.pack", None).unwrap() else { + panic!() + }; + b.delete("packs/a.pack").unwrap(); + assert!(b.put_new("packs/a.pack", b"x").unwrap()); + b.delete_leased(&[("packs/a.pack".into(), old)]).unwrap(); + assert!(b.exists("packs/a.pack").unwrap()); + + // A zero-grace pass leaves a tight store, and a second one changes nothing. + let mem = Arc::new(GitLike::default()); + let refs = hand_built(&mem, 12, 10); + like_carrier(&mem).compact(&eager()).unwrap(); + let settled = mem.map.lock().unwrap().clone(); + like_carrier(&mem).compact(&eager()).unwrap(); + assert!( + settled.keys().eq(mem.map.lock().unwrap().keys()), + "a replay must not add or remove a key" + ); + assert_readable(&mem, &refs); +} + +/// A push's objects and the tip it moves a branch to. +type Push = (Vec<(ObjectId, Vec)>, ObjectId); + +/// Build a one-pack-per-push chain by hand; `idx_of` may rewrite a pack's idx. +fn chain( + b: &dyn Bucket, + packs: &[Push], + idx_of: impl Fn(usize, Vec) -> Vec, +) -> Vec { + let mut hashes = Vec::new(); + for (i, (objects, tip)) in packs.iter().enumerate() { + let (pack, idx) = scl_core::pack::build_pack(objects).unwrap(); + let hash = hex::encode(blake3::hash(&pack).as_bytes()); + b.put_new(&pack_key(&hash), &pack).unwrap(); + b.put_new(&idx_key(&hash), &idx_of(i, idx)).unwrap(); + let seq = i as u64 + 1; + let entry = LogEntry { + seq, + parent_seq: seq - 1, + packs: vec![hash.clone()], + updates: vec![RefUpdate { + branch: format!("b-{seq}"), + old: None, + new: *tip, + }], + }; + b.put_new(&log_key(seq), &entry.encode()).unwrap(); + hashes.push(hash); + } + let m = Manifest { + head_seq: packs.len() as u64, + checkpoint_seq: 0, + head_branch: "b-1".into(), + }; + b.put_if_tag("manifest", &m.encode(), None) + .unwrap() + .unwrap(); + hashes +} + +#[test] +fn packs_are_sized_from_their_idx_and_one_far_over_its_idx_is_left_unmerged() { + // Deterministic bytes zstd cannot shrink, so the pack really is about this long. + let mut x = 0x9e37_79b9_7f4a_7c15u64; + let noise: Vec = (0..6000) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x as u8 + }) + .collect(); + let small = |label: &str| { + let (tip, objects) = commit(label); + (objects, tip) + }; + let big = { + let blob = Object::blob(noise); + let tree = Object::Tree(scl_core::Tree::new(vec![scl_core::TreeEntry { + name: "f".into(), + kind: scl_core::EntryKind::Blob, + id: blob.id(), + mode: scl_core::FileMode(0o644), + perms: 0, + }])); + let snap = Object::Snapshot(scl_core::Snapshot { + root: tree.id(), + parents: vec![], + author: "t".into(), + timestamp: 1, + message: "liar".into(), + secrets: Default::default(), + protection: Default::default(), + }); + let objects: Vec<(ObjectId, Vec)> = [&blob, &tree, &snap] + .iter() + .map(|o| (o.id(), o.encode())) + .collect(); + (objects, snap.id()) + }; + let mem = Arc::new(GitLike::default()); + let hashes = chain( + mem.bucket().as_ref(), + &[small("s-1"), small("s-2"), big], + // The third pack's idx claims it is a dozen bytes long. + |i, idx| { + if i != 2 { + return idx; + } + let entries = scl_core::pack::parse_index(&idx).unwrap(); + let mut lie = Vec::new(); + lie.extend_from_slice(b"SCIX"); + lie.extend_from_slice(&1u32.to_le_bytes()); + lie.extend_from_slice(&(entries.len() as u64).to_le_bytes()); + for e in &entries { + lie.extend_from_slice(e.id.as_bytes()); + lie.extend_from_slice(&8u64.to_le_bytes()); + lie.extend_from_slice(&1u64.to_le_bytes()); + } + lie + }, + ); + let opts = CompactOptions { + small_pack_bytes: 2000, + ..eager() + }; + // The idx sizes are what plans the merge: no list_meta (the double refuses it). + let rep = like_carrier(&mem).compact(&opts).unwrap(); + assert_eq!((rep.packs_merged, rep.packs_created), (2, 1), "{rep:?}"); + assert!( + rep.notes + .iter() + .any(|n| n.contains("left unmerged") && n.contains(&hashes[2])), + "{rep:?}" + ); + // The over-sized pack is untouched and still on the chain; the honest two are replaced. + assert!(mem.bucket().exists(&pack_key(&hashes[2])).unwrap()); + assert!(!mem.bucket().exists(&pack_key(&hashes[0])).unwrap()); + let ck = checkpoint_key(manifest(&mem).checkpoint_seq); + let Fetched::New { bytes, .. } = mem.bucket().get(&ck, None).unwrap() else { + panic!("no checkpoint") + }; + let packs = Checkpoint::decode(&bytes).unwrap().packs; + assert_eq!(packs.len(), 2); + assert!(packs.contains(&hashes[2])); +} + +#[test] +fn the_idx_length_is_the_end_of_the_furthest_record_and_saturates() { + use scl_core::pack::{build_pack, parse_index, IndexEntry}; + let (_, objects) = commit("sized"); + let (pack, idx) = build_pack(&objects).unwrap(); + let entries = parse_index(&idx).unwrap(); + assert_eq!(declared_pack_len(&entries), pack.len() as u64); + // an empty pack is its 8-byte header + let (empty, empty_idx) = build_pack(&[]).unwrap(); + assert_eq!( + declared_pack_len(&parse_index(&empty_idx).unwrap()), + empty.len() as u64 + ); + // a hostile idx can only make a pack look large + let id = objects[0].0; + let huge = |offset, length| IndexEntry { id, offset, length }; + assert_eq!(declared_pack_len(&[huge(u64::MAX, 1)]), u64::MAX); + assert_eq!(declared_pack_len(&[huge(8, u64::MAX)]), u64::MAX); + assert_eq!(declared_pack_len(&[huge(u64::MAX - 3, 0)]), u64::MAX); +} diff --git a/crates/repo/src/bucket_transport/git_remote_tests.rs b/crates/repo/src/bucket_transport/git_remote_tests.rs index 4730bfbe..b0feeb0a 100644 --- a/crates/repo/src/bucket_transport/git_remote_tests.rs +++ b/crates/repo/src/bucket_transport/git_remote_tests.rs @@ -1,5 +1,5 @@ //! `sc+git::` remote plumbing: URL parsing and the reserved prefix, the -//! private-branch gate, and the refusals (`serve --store`, `remote compact`). +//! private-branch gate, and the `serve --store` refusal. //! Nothing here reaches `GitBucket`'s real I/O except the URL-parse test. use super::*; @@ -163,14 +163,12 @@ fn push_of_a_private_branch_to_a_git_carrier_refuses_before_any_io() { } #[test] -fn serve_store_and_compact_refuse_git_carriers_before_parsing() { +fn serve_store_refuses_git_carriers_before_parsing() { let url = "sc+git::file:///srv/r.git"; assert!(invalid(refuse_git_store(url)).contains("sc serve --store")); - assert!(invalid(refuse_git_compact(url)).contains("remote compact")); // The reserved prefix is refused too, so a near miss never reaches a parser. assert!(refuse_git_store("sc+git:/x").is_err()); assert!(refuse_git_store("sc+wal:///tmp/b").is_ok()); - assert!(refuse_git_compact("sc+s3://b/p").is_ok()); // The wire layer enforces it for both `--stdio` and `--http` serving, and // the client sees why at the handshake. @@ -196,24 +194,6 @@ fn serve_store_and_compact_refuse_git_carriers_before_parsing() { std::fs::remove_dir_all(&home).unwrap(); } -/// The compaction refusal lives in the repo layer, not only in the CLI: a -/// transport opened from an `sc+git::` URL refuses `compact` outright. -#[test] -fn compact_refuses_a_git_carrier_transport() { - let d = root("compact-git"); - let open = |git: bool| { - BucketTransport::build( - Box::new(scl_objio::DirBucket::open(d.to_str().unwrap()).unwrap()), - git, - ) - .unwrap() - }; - let msg = invalid(open(true).compact(&CompactOptions::default())); - assert!(msg.contains("remote compact"), "{msg}"); - open(false).compact(&CompactOptions::default()).unwrap(); - std::fs::remove_dir_all(&d).unwrap(); -} - #[test] fn sc_git_urls_parse_with_the_inner_url_verbatim() { for inner in [ diff --git a/crates/repo/src/lib.rs b/crates/repo/src/lib.rs index 4b2023dc..0c2506cd 100644 --- a/crates/repo/src/lib.rs +++ b/crates/repo/src/lib.rs @@ -80,8 +80,8 @@ mod ws_refresh; pub use scl_objio::redact; pub use bucket_transport::{ - check_private_push_allowed, is_bucket_url, is_git_bucket_url, refuse_git_compact, - refuse_git_store, BucketScheme, BucketTransport, BucketUrl, CompactOptions, CompactReport, + check_private_push_allowed, is_bucket_url, is_git_bucket_url, refuse_git_store, BucketScheme, + BucketTransport, BucketUrl, CompactOptions, CompactReport, }; pub use conflicts::{ActiveOp, ConflictKind, ConflictVersions, ResolveSide, Side}; pub use error::{Error, Result}; From 225f0fcbaa8ef8e7bfca05daeaa01b678bb82a12 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 14:50:30 +0200 Subject: [PATCH 3/7] docs: compaction on sc+git:: (ADR-0057 addendum, ADR-0060, demo) ADR-0057 gets an addendum with the first-sighting record, idx sizing, leased deletes and the uncached manifest check; ADR-0060 drops "compaction refused", corrects the freshness-window sentence (a stale view is not harmless for a delete), and records that every pusher holds delete and that deleting a ref is not erasure. THREAT-MODEL, ROADMAP (follow-ons 1, 5a, 6, 7 addressed as far as they go; 3, 5b, 8 and S2 declined with reasons), CLAUDE.md, ARCHITECTURE.md, README and the sc-cli-reference skill follow. demo/run_git_carrier_demo.sh gains a compaction step in both legs (bare repo and sc-forge): two runs a grace window apart, printing the refs/sc/* count before and after, ahead of the keyless and keyed clones. --- .claude/skills/sc-cli-reference/SKILL.md | 24 +++-- ARCHITECTURE.md | 25 ++++-- CLAUDE.md | 17 +++- README.md | 7 +- ROADMAP.md | 59 ++++++++----- demo/run_git_carrier_demo.sh | 53 ++++++++--- docs/THREAT-MODEL.md | 25 +++++- docs/adr/0057-bucket-compaction.md | 97 +++++++++++++++++++++ docs/adr/0060-git-carried-bucket-remotes.md | 58 +++++++++--- 9 files changed, 296 insertions(+), 69 deletions(-) diff --git a/.claude/skills/sc-cli-reference/SKILL.md b/.claude/skills/sc-cli-reference/SKILL.md index ac57c3f0..6ea2bec0 100644 --- a/.claude/skills/sc-cli-reference/SKILL.md +++ b/.claude/skills/sc-cli-reference/SKILL.md @@ -431,9 +431,9 @@ cargo run --bin sc -- remote add sc+git:: [--allow-private-branc # `git ls-remote` lists them; the same repo can also # hold a `--git` mirror of public branches. Anyone # with push to the repo holds bucket put + delete; - # deleting a ref is not erasure. `remote compact` - # and `serve --store` refuse it in v1; clone - # --filter refused as for any bucket. Untested + # deleting a ref is not erasure. `serve --store` + # refuses it in v1; `remote compact` works (below); + # clone --filter refused as for any bucket. Untested # against real GitHub cargo run --bin sc -- remote compact [--dry-run] [--prune-expire 24h] [--allow-short-grace] # bucket remotes only (#216, ADR-0057): merge small @@ -451,7 +451,18 @@ cargo run --bin sc -- remote compact [--dry-run] [--prune-expire 24h] [-- # or fetches running). On a versioned S3 # bucket deletes only add delete markers (add a # noncurrent-version lifecycle rule). Relieves the - # 65536-pack checkpoint cap (not the ref cap) + # 65536-pack checkpoint cap (not the ref cap). + # On `sc+git::` (ADR-0057 addendum) a Git host has no + # trustworthy object age, so the age is counted from + # the compactor's first sighting of each pack and + # checkpoint (refs/sc/compact/sightings): the first + # run deletes nothing it did not already see, a later + # one at least --prune-expire on may (with a zero + # grace a first sighting is immediately old). Run it + # from ONE host (clocks); a malformed record blocks + # the run (delete that ref to start over). Every + # pusher holds delete there; deleting a ref is not + # erasure cargo run --bin sc -- serve --http --store sc+wal://|sc+s3:/// # serve a bucket remote over sc+http(s):// (P36c): # all content lives in the bucket (instance is @@ -463,7 +474,10 @@ bash demo/run_git_carrier_demo.sh # sc+git:: proof (P39): a bare Git # `git clone` sees no refs/sc, keyless clone sees # `(private, no access)` and no plaintext, keyed # clone reads all, public main also mirrored as a - # normal Git branch; second leg against sc-forge; + # normal Git branch; `remote compact` twice a grace + # window apart shrinks refs/sc/* (the first run + # deletes nothing) and the clones after it still + # read; second leg against sc-forge; # no sc-git-bucket-* temp leftovers bash demo/run_http_remote_demo.sh # sc-native HTTP transport proof (P26): real # loopback TCP (no shim, unlike ssh://), clone + diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 53017abc..fe95d213 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1372,17 +1372,30 @@ length. Git is the system binary (`SC_GIT` overrides) with a scrubbed environmen transport allowlist (`https`, `http`, `ssh`, `file`; `ext`, `fd` and `git` forced off), a URL allowlist and no userinfo on `http(s)` URLs (credentials go through git's credential helper); a private bare repo in `sc-git-bucket--` caches fetched -objects and is removed on `Drop`. v1 scaling limit: every `ls-remote` and push downloads the +objects and is removed on `Drop`. Scaling: every `ls-remote` and push downloads the whole ref advertisement (the host does not filter it server-side), and a cold read fetches -every `log/` and `checkpoints/` entry; with compaction refused, refs grow by at least three -per push. +every `log/` and `checkpoints/` entry behind the last checkpoint; refs grow by at least three +per push until `sc remote compact` runs. + +`sc remote compact` works on a carrier (ADR-0057 addendum). A Git host reports no +trustworthy object age (committer time is writer-controlled), so the compactor keeps its +own: a first-sighting record at `refs/sc/compact/sightings`, one ` ` +line per `packs/` and `checkpoints/` key, aged on the compactor's clock and +generation-aware by oid (a key deleted and uploaded again starts over). The record supplies +ages only, decodes strictly (a bad record blocks the run) and is written before any delete +decision uses it. Pack sizes for the merge plan come from each live idx (`offset + 4 + +length`), never from `list_meta`, which would download every value. Deletes are leased +(`Bucket::delete_leased`: one `git push` of `:refs/sc/K` with a +`--force-with-lease=refs/sc/K:` per key, the oid the one listing showed), and the +manifest is read uncached before every destructive batch. The first run after an object +appears never deletes it; run compaction from one host. A private branch goes to an `sc+git::` remote only under the per-remote opt-in `--allow-private-branches` (`sc remote set --allow-private-branches` for an existing remote), since its name, recipient ids and sizes become readable by -every reader of the repository and the ciphertext cannot be taken back. `sc remote -compact` (no trustworthy age for a stored pack) and `sc serve --store` (a daemon cannot -answer prompts, and it would bypass the opt-in) refuse an `sc+git::` store in v1. +every reader of the repository and the ciphertext cannot be taken back. +`sc serve --store` (a daemon cannot answer prompts, and it would bypass the opt-in) +refuses an `sc+git::` store in v1. sc-forge accepts exactly the `refs/sc/` prefix (ADR-0058 addendum). The dependency rule is amended: `objio` may spawn the system `git`, and only `gitio::bridge` and objio's git backend spawn it; `gix` stays in `gitio`. Proven by `crates/cli/tests/git_bucket.rs`, diff --git a/CLAUDE.md b/CLAUDE.md index 476ad1e3..c62c485f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -164,7 +164,7 @@ the code, those win. | P37c | Merge, cherry-pick and rebase on a partial clone: a tree-recursive engine (`merge_trees`) carries a subtree by id when the sides agree and no protection rule applies, reads only where they differ or a rule intersects, and on a lazy, online clone fills what it asks for by id outside the store lock (a non-lazy clone gets the gap error with `sc backfill `); a carried protected subtree keeps the complete input wrap union; a conflict inside the view is marked and resolved like on a full clone (`sc conflicts`/`sc resolve`/`merge --abort`/`sc commit`; the out-of-view part is grafted from the decided root), while a conflict outside the view, or a result that must be sealed outside it, is refused before any ref, merge-state or working-tree change (fetched objects stay). A subtree identical on all three sides is carried even under a protection rule. Cherry-pick and rebase use the same engine (`replay_commit`): the fold's accumulator may hold absent subtrees, the view is filled before the final write, and a stopped pick or rebase keeps its carried subtrees in the decided root. Mode-only edits now merge and a file/directory collision fails before any write, on full clones too. | [0049](docs/adr/0049-merge-on-partial-clones.md) | | P37d | Agent sessions on a partial clone: `sc ws fork`/`run`/`harvest`/`abandon` and `sc work` work; a workspace or agent sees the host's sparse view only (`sc work` no longer gets a complete tree there), the view closure is filled (lazy, online) or refused with the gap error before anything is written, a workspace commit grafts the out-of-view subtrees from the base without reading them, and harvest lands through the ADR-0049 merge (any conflict falls back to a `work-` branch as on a full clone) | [0050](docs/adr/0050-sessions-on-partial-clones.md) | | P38 | Gitea-compatible forge over bare Git and smart HTTP: recorded API subset, scoped auth, protected merges, maintenance and container operations; 24 recorded scenarios and six first-consumer acceptance scenarios pass. Hosted repos have none of the `.sc/` features; SSH and Actions remain outside v1. | [0058](docs/adr/0058-sc-forge-gitea-compatible-git-forge.md) | -| P39 | `sc+git::` remotes: the bucket WAL stored as one parentless commit per key at `refs/sc/` in any Git repository (system `git` via `objio::GitBucket`, lease CAS, no `+` refspec), so private branches, protected paths and secrets travel through a Git host (local bare repo, sc-forge; GitHub untested); private-branch push needs per-remote `--allow-private-branches`; `remote compact` and `serve --store` refuse it in v1; sc-forge accepts `refs/sc/` | [0060](docs/adr/0060-git-carried-bucket-remotes.md) | +| P39 | `sc+git::` remotes: the bucket WAL stored as one parentless commit per key at `refs/sc/` in any Git repository (system `git` via `objio::GitBucket`, lease CAS, no `+` refspec), so private branches, protected paths and secrets travel through a Git host (local bare repo, sc-forge; GitHub untested); private-branch push needs per-remote `--allow-private-branches`; `remote compact` works on it (first-sighting age record, leased deletes; ADR-0057 addendum) while `serve --store` refuses it in v1; sc-forge accepts `refs/sc/` | [0060](docs/adr/0060-git-carried-bucket-remotes.md) | ## Standing boundaries & gotchas @@ -228,8 +228,15 @@ transport-adjacent. The rest, imperatively: error on it and it never reaches the P18 bridge. Pushing a private branch needs the per-remote opt-in `--allow-private-branches` (`remote add`/`clone`; enforced in `repo`, refuses before any upload); protected paths and secrets - need none. `sc remote compact` and `sc serve --store` refuse it (deferred: no - trustworthy pack age, no prompts in a daemon). Every git call goes through + need none. `sc serve --store` refuses it (deferred: no prompts in a daemon). + `sc remote compact` works on it with its own age source (a Git host has no + trustworthy object age): a first-sighting record at + `refs/sc/compact/sightings` (` `, generation-aware, strict + decode: a bad record blocks the run, a time over five minutes ahead of the + compactor's clock is refused), leased deletes (`Bucket::delete_leased`), and + an uncached manifest read before every destructive batch. The first run + deletes nothing it did not already see (zero grace: a first sighting is + immediately old); run it from one host (clocks). Every git call goes through `objio::GitBucket` with a scrubbed environment; CAS is `--force-with-lease`, never a `+` refspec, and a rejected push is classified by re-reading the ref. **Everyone with push to the Git repository holds bucket put and delete** (no @@ -255,7 +262,9 @@ transport-adjacent. The rest, imperatively: not hold. It advances the manifest only through the normal CAS (one no-op entry plus a checkpoint) and never deletes what the current chain names. Keep `--prune-expire` well above any push's duration (default 24h): a writer stalled longer than that between uploading a pack and its - commit can still lose the pack (the leases follow-on closes it). + commit can still lose the pack (the leases follow-on closes it). On an + `sc+git::` remote the manifest is read uncached before every delete batch + (every backend) and every pusher holds delete. `--dry-run` writes and deletes nothing but still downloads and decodes every candidate pack, so on S3 its download egress and read requests match a real run (the uploads, CAS and deletes it skips are the rest). diff --git a/README.md b/README.md index eb30adab..d57a0298 100644 --- a/README.md +++ b/README.md @@ -142,9 +142,10 @@ stores the same bucket inside any (SHA-1) Git repository as `refs/sc/*`, so priv branches, protected paths and secrets travel through a Git host that a plain `git clone` never sees into (ADR-0060). Pushing a private branch there needs `--allow-private-branches` (its name and recipient ids become readable by -everyone who can read the repository); compaction and `serve --store` are not -supported on it yet. Proven by `demo/run_git_carrier_demo.sh`; not tested -against GitHub itself. +everyone who can read the repository). `sc remote compact` works on it (run it +from one host; the first run only records when it first saw each object); +`serve --store` is not supported on it yet. Proven by +`demo/run_git_carrier_demo.sh`; not tested against GitHub itself. ## How Phase 1 works diff --git a/ROADMAP.md b/ROADMAP.md index 6f30d08d..7a5b89f3 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -703,9 +703,11 @@ across every phase. sc-forge verified; real GitHub untested); a plain `git clone` never fetches `refs/sc/*`, and the same repository can also mirror public branches through a `--git` remote. A private-branch push needs the per-remote opt-in - `--allow-private-branches`; `sc remote compact` and `sc serve --store` refuse - `sc+git::` in v1; sc-forge accepts the `refs/sc/` prefix. No format, wire or - crypto change. Proven by `demo/run_git_carrier_demo.sh`. (ADR-0060.) + `--allow-private-branches`; `sc remote compact` works on `sc+git::` (a + first-sighting age record, leased deletes; ADR-0057 addendum) while + `sc serve --store` refuses it in v1; sc-forge accepts the `refs/sc/` prefix. No + format, wire or crypto change. Proven by `demo/run_git_carrier_demo.sh`. + (ADR-0060.) ## Active @@ -1016,27 +1018,36 @@ P28 review follow-ons, the port de-flake, the clippy allow-list) are recorded in issues a `has_object` round trip per object over S3; batching those probes into one `refresh()` plus local index lookups (instead of one S3 round trip per object) is deferred. -- **`sc+git::` follow-ons (P39, ADR-0060).** Deferred rather than built: (1) - `sc remote compact` on an `sc+git::` remote, which needs an age source the - compactor controls (a Git repository gives only writer-controlled committer - time, so `list_meta` reports observation time and v1 refuses the command); - (2) `sc serve --store sc+git::...` (a daemon cannot answer ssh or credential - prompts, and it would sit between clients and the private-branch opt-in); (3) - rollback and rewind of `refs/sc/manifest` by anyone with push access, tracked - under P34 follow-on (6) and ADR-0059 follow-on (2), now reachable by more - people; (4) validation against real GitHub (per-ref and per-push limits, - rulesets, hosts that refuse unknown ref namespaces), untested so far; (5) - batched deletes (one `git push` for many `:refs/sc/K`) and a batched - `has_objects`, since a Git host costs a round trip per probe or delete; (6) - `list_meta` downloads every listed value to read its size; (7) scaling: every - `ls-remote` and push downloads the whole ref advertisement (the host does not - filter it by prefix server-side, a packet trace showed the full list) and a - cold read fetches every `log/` and `checkpoints/` entry, and with compaction - refused (1) refs grow by at least three per push, so a long-lived store slows - down until compaction or a ref-filtering host exists; (8) SHA-256 Git - repositories (v1 pins its cache to SHA-1 and handles SHA-1 ids only); (9) a - download budget for fetches into the temp cache (`git fetch` has no input-size - limit, so an oversized value is written before the read caps refuse it). +- **`sc+git::` follow-ons (P39, ADR-0060).** Built: (1) `sc remote compact` on + an `sc+git::` remote (ADR-0057 addendum). A Git repository gives only + writer-controlled committer time, so the compactor keeps its own age source: a + first-sighting record at `refs/sc/compact/sightings`, generation-aware by oid, + with leased deletes and an uncached manifest check before every destructive + batch; (5a) batched deletes (`Bucket::delete_leased`, one `git push` per chunk + of refs); (6) compaction no longer calls `list_meta` on a carrier (pack sizes + come from the idx; `list_meta` itself still downloads every listed value); (7) + in part: compaction folds small packs and deletes what nothing names, which + bounds the ref growth of at least three per push and the log/checkpoint + entries a cold read fetches, but between runs every `ls-remote` and push still + downloads the whole ref advertisement (the host does not filter it by prefix + server-side, a packet trace showed the full list). Still deferred: (2) + `sc serve --store sc+git::...` (a daemon cannot answer ssh or credential + prompts, and it would sit between clients and the private-branch opt-in); (4) + validation against real GitHub (per-ref and per-push limits, rulesets, hosts + that refuse unknown ref namespaces), untested so far; (9) a download budget for + fetches into the temp cache (`git fetch` has no input-size limit, so an + oversized value is written before the read caps refuse it). Declined, with + reasons: (3) rollback and rewind of `refs/sc/manifest` by anyone with push + access is the cross-cutting signed-ref effort already tracked under P34 + follow-on (6) and ADR-0059 follow-on (2), now reachable by more people; a + client-side "highest manifest seq seen" floor was reviewed and judged a false + control (a fresh clone has no floor); (5b) a batched `has_objects`: the 2 s + freshness window already limits the per-object manifest probe to one network + call per 2 s; (8) SHA-256 Git repositories (v1 pins its cache to SHA-1): GitHub + does not support them and sc-forge is SHA-1, so the value is low and the cost + moderate (the cache's object format would have to be detected before the first + fetch); (S2, from review) the ssh user name that `remote list` prints is not a + credential, and `http(s)` userinfo is refused at validation. - **Serve-side persistent pack cache (P36c follow-on).** A bucket-backed serve instance re-downloads packs per connection; a content-addressed on-disk cache in the serve home would make warm instances cheap without diff --git a/demo/run_git_carrier_demo.sh b/demo/run_git_carrier_demo.sh index 642c8955..bf3df49b 100755 --- a/demo/run_git_carrier_demo.sh +++ b/demo/run_git_carrier_demo.sh @@ -13,11 +13,17 @@ # with --allow-private-branches; protected paths and secrets need no opt-in. # 3. CARRIER: `git ls-remote` lists refs/sc/*; a plain `git clone` of the same # repository fetches none of them and holds no sc objects. -# 4. KEYLESS: an sc clone sees `hotfix (private, no access)`, no protected +# 4. COMPACT: `sc remote compact` works on the carrier. A Git host reports no +# trustworthy object age, so the first run only records when it first saw +# each object (refs/sc/compact/sightings) and deletes nothing; a second run +# a grace window later deletes what aged, with leased ref deletions. The +# refs/sc/* count is printed before and after, and the clones below prove +# the compacted store still reads. +# 5. KEYLESS: an sc clone sees `hotfix (private, no access)`, no protected # plaintext, no embargoed file, and cannot read the secret value. -# 5. KEYED: an sc clone with alice's identity reads the protected file, the +# 6. KEYED: an sc clone with alice's identity reads the protected file, the # secret and the embargoed branch. -# 6. Zero residue: no leftover `sc-git-bucket-*` temp dirs (TMPDIR is private +# 7. Zero residue: no leftover `sc-git-bucket-*` temp dirs (TMPDIR is private # to each run, so any leftover is ours) and no .sc/tmp residue. # # Self-checking: every claim is an assertion; any failure exits non-zero before @@ -75,7 +81,7 @@ scenario() { local url="sc+git::$giturl" local a="$WORK/alice-$idx" local key="$WORK/alice.key" - local out before after n + local out before mid after n echo echo "=== [$label] 1: public main goes through the Git bridge; sealed content cannot ===" @@ -152,7 +158,31 @@ scenario() { echo "git ls-remote lists refs/sc/*; plain clone holds main only, no sc objects ✔" echo - echo "=== [$label] 4: KEYLESS — an sc clone sees ciphertext and markers only ===" + echo "=== [$label] 4: COMPACT — two runs a grace window apart shrink refs/sc/* ===" + cd "$a" + before=$(git ls-remote "$giturl" 'refs/sc/*' | wc -l | tr -d ' ') + out=$("$SC" remote compact opted --prune-expire 2s --allow-short-grace 2>&1) \ + || fail "the first compaction run failed (got: $out)" + echo "$out" | grep -q "first sighting" \ + || fail "the first run must record when it first saw each object (got: $out)" + echo "$out" | grep -q "deleted 0 log entries, 0 checkpoint(s), 0 pack(s)" \ + || fail "the first run must delete nothing it never saw before (got: $out)" + git ls-remote "$giturl" refs/sc/compact/sightings | grep -q "refs/sc/compact/sightings" \ + || fail "the age record must be on the Git host" + mid=$(git ls-remote "$giturl" 'refs/sc/*' | wc -l | tr -d ' ') + sleep 3 + out=$("$SC" remote compact opted --prune-expire 2s --allow-short-grace 2>&1) \ + || fail "the second compaction run failed (got: $out)" + echo "$out" | grep -q "deleted 0 log entries" \ + && fail "the second run, past the grace, must delete log entries (got: $out)" + after=$(git ls-remote "$giturl" 'refs/sc/*' | wc -l | tr -d ' ') + [ "$after" -lt "$mid" ] || fail "the second run must remove refs (before $before, mid $mid, after $after)" + git ls-remote "$giturl" refs/sc/manifest | grep -q "refs/sc/manifest" \ + || fail "compaction must never remove the manifest" + echo "refs/sc/*: $before before, $mid after run 1 (nothing deleted, first sightings recorded), $after after run 2 ✔" + + echo + echo "=== [$label] 5: KEYLESS — an sc clone sees ciphertext and markers only ===" "$SC" clone "$url" "$WORK/keyless-$idx" >/dev/null 2>&1 || fail "keyless sc clone must work" cd "$WORK/keyless-$idx" "$SC" branch list | grep -q "hotfix (private, no access)" \ @@ -169,7 +199,7 @@ scenario() { echo "keyless clone: branch marker, no protected plaintext, secret unreadable ✔" echo - echo "=== [$label] 5: KEYED — alice's clone reads all three ===" + echo "=== [$label] 6: KEYED — alice's clone reads all three ===" "$SC" clone "$url" "$WORK/keyed-$idx" --allow-private-branches >/dev/null 2>&1 \ || fail "keyed sc clone must work" cd "$WORK/keyed-$idx" @@ -184,7 +214,7 @@ scenario() { echo "keyed clone: protected file decrypts, secret injects, embargoed branch reads ✔" echo - echo "=== [$label] 6: zero residue ===" + echo "=== [$label] 7: zero residue ===" no_cache_residue for r in "$a" "$WORK/keyless-$idx" "$WORK/keyed-$idx"; do if [ -d "$r/.sc/tmp" ] && [ -n "$(ls -A "$r/.sc/tmp" 2>/dev/null)" ]; then @@ -263,6 +293,9 @@ echo "secret and a private branch crossed it losslessly, where the Git bridge" echo "refuses sealed content. The private branch needed the per-remote" echo "--allow-private-branches opt-in (the refused push uploaded nothing); a plain" echo "git clone fetched no refs/sc/* and held no sc objects while public main" -echo "rode along as an ordinary Git branch; a keyless sc clone saw only the" -echo "branch marker and ciphertext, a keyed clone read all three. Verified" -echo "across two independent runs, zero sc-git-bucket-* or .sc/tmp residue." +echo "rode along as an ordinary Git branch. sc remote compact then shrank" +echo "refs/sc/* on the host in two runs a grace window apart (the first only" +echo "recorded first sightings and deleted nothing), and after it a keyless sc" +echo "clone saw only the branch marker and ciphertext, a keyed clone read all" +echo "three. Verified across two independent runs, zero sc-git-bucket-* or" +echo ".sc/tmp residue." diff --git a/docs/THREAT-MODEL.md b/docs/THREAT-MODEL.md index ca5677b3..a3edb860 100644 --- a/docs/THREAT-MODEL.md +++ b/docs/THREAT-MODEL.md @@ -632,7 +632,8 @@ too, not just the src-control-side metadata. - **Compaction deletes data, so it is a privileged operation (#216, ADR-0057).** `sc remote compact` needs bucket **list and delete** rights in addition to the get/put/head permissions a pusher holds (pushes themselves need no list - or delete rights). Grant them to a dedicated operator or + or delete rights; on an `sc+git::` carrier every pusher holds them, see the carrier + section). Grant them to a dedicated operator or scheduled-job identity, not to ordinary pushers, read-only clients or the `sc serve --store` instances (they never need them: compaction is not a wire verb, so a served bucket's clients cannot trigger a delete). It cannot @@ -766,10 +767,26 @@ The bucket model above applies unchanged, with the Git repository as the bucket permission refusal is never reported as a lost race. The atomic CAS relies on the host checking the sent old value under its ref lock: stock `git receive-pack` and sc-forge do (verified); another host that does not cannot carry the store safely. -- **Not built in v1.** `sc remote compact` is refused (committer time is writer-controlled, so it - cannot be an age source) and so is `sc serve --store sc+git::` (a daemon cannot answer prompts +- **Compaction on a carrier has no trusted clock, so it keeps its own (ADR-0057 addendum).** + Committer time is writer-controlled and predates publication, so it is never an age. + `sc remote compact` instead records when it first saw each pack and checkpoint in + `refs/sc/compact/sightings` and ages objects from that (a re-created key is a new generation by + oid, so it starts over). The record is writer-trusted like the rest of the store: anyone who can + push can backdate it, but that person can already delete refs directly, so it adds no + authority. It supplies ages only: the candidates come from the listing minus everything the + live chain and the retained checkpoints and log entries name, so a record cannot make a live + object deletable. A malformed record, one naming a key outside `packs/` and `checkpoints/`, + or one dated more than five minutes ahead of the compactor's clock, blocks the run (it is + never read as "everything is old"). Deletes are leased to the oid the listing showed, so a key + re-created since survives, and the manifest is read uncached before every destructive batch. + Left open: a pusher reusing an aged orphan pack byte for byte can still lose it if the delete + lands after the manifest check and before its commit (no lease on the manifest), and a push + that stalls past the grace window loses its pack, as on any backend. **Every pusher holds + delete** (a push to `refs/sc/*` is a push); unlike S3, delete rights cannot be withheld from + them. Run compaction from one host (clocks). +- **Not built in v1.** `sc serve --store sc+git::` is refused (a daemon cannot answer prompts and would bypass the client-side private-branch opt-in). -- **Scaling.** Every `ls-remote` and push downloads the whole ref advertisement (the pattern given to `ls-remote` filters on the client; a packet trace showed the server sends every ref, protocol v2 `ref-prefix` or not), and a cold read fetches every `log/` and `checkpoints/` entry. With compaction refused, refs grow by at least three per push (a log entry, a pack and its idx), so a long-lived store gets slower to open until compaction or a ref-filtering host exists. +- **Scaling.** Every `ls-remote` and push downloads the whole ref advertisement (the pattern given to `ls-remote` filters on the client; a packet trace showed the server sends every ref, protocol v2 `ref-prefix` or not), and a cold read fetches every `log/` and `checkpoints/` entry behind the last checkpoint. Refs grow by at least three per push (a log entry, a pack and its idx); `sc remote compact` bounds that by merging small packs and deleting what nothing names, so a long-lived store should be compacted on a schedule. Between runs the advertisement still grows until a ref-filtering host exists. - **Local cache.** A temporary bare repository `sc-git-bucket--` (mode 0700) holds fetched store objects and is removed on exit; a killed process (SIGKILL, or SIGINT) can leave it behind, and there is no reaper. diff --git a/docs/adr/0057-bucket-compaction.md b/docs/adr/0057-bucket-compaction.md index 2ef32c1c..79c41e8e 100644 --- a/docs/adr/0057-bucket-compaction.md +++ b/docs/adr/0057-bucket-compaction.md @@ -274,3 +274,100 @@ The `Bucket` trait gains three methods, implemented by `DirBucket` and `crates/repo/src/bucket_transport/compact.rs` (`compact`, `CompactOptions`, `CompactReport`); reader restart and the pre-commit pack check in `bucket_transport.rs`; `sc remote compact` in `crates/cli`. + +## Addendum: compaction on an `sc+git::` remote (P39 follow-on, ADR-0060) + +ADR-0060 shipped the Git carrier with compaction refused: a Git host reports +no trustworthy age for a stored object. The only time it has is a commit's +committer date, which the writer chooses and which predates publication, so it +cannot say how long an unreferenced pack has been sitting there. Without +compaction `refs/sc/*` grows by at least three refs per push, every `ls-remote` +and push downloads the whole advertisement, and a cold read fetches every +`log/` and `checkpoints/` entry. This addendum lifts the refusal. The +algorithm above is unchanged; only the age source, the sizing, and the delete +primitive differ on a carrier, and the deletion rules around them get stricter. + +**Age source: first sightings.** On a carrier the compactor keeps its own +clock. Bucket key `compact/sightings` (`refs/sc/compact/sightings`) holds one +line per tracked key, ` \n`, sorted by key, +where `oid` is the value's tag when the compactor first listed it and +`first_seen` is the compactor's own clock then. Only `packs/` and +`checkpoints/` keys are tracked: log entries are deleted by sequence behind the +cutoff, never by age. An object's age is `now - first_seen`, so the first run +after an object appears never deletes it (with a zero grace a first sighting is +immediately old), and a later run at least `--prune-expire` after the first +sighting may. A listed key whose oid differs from the recorded one is a new +generation (a key that was deleted and uploaded again; every write is a new +commit, so the oid always differs) and starts its age over. + +The record supplies ages and nothing else. Deletion candidates still come from +the listing minus everything the live chain and the retained checkpoints and log +entries name, so a record that calls a live pack ancient deletes nothing live. +It decodes strictly: plain decimal seconds, a 40-digit lowercase oid, a +`packs/` or `checkpoints/` key the carrier accepts, keys sorted and unique, no +other bytes. Anything else is an error that blocks the run before phase 1 +writes anything, and the message names the fix (delete +`refs/sc/compact/sightings`; every object's age then starts over); a record is +never read as "everything is old". A first-seen time more than five minutes +after the compactor's own clock is refused ("compactor clocks disagree"). The +read cap is the 256 MiB value cap; a write that would pass it is an error, never +a truncation. The record is written with a compare-and-swap, only when it +changed, before any delete decision uses it (a lost CAS replans, so two +compactors converge). A dry run never writes it and reports ages as a lower +bound. + +**Sizes without downloads.** `list_meta` downloads every value on a carrier, so +the compactor never calls it there. A live pack's size for the merge plan is the +end of its furthest record in its idx (`offset + 4 + length`, saturating), taken +while the view parses each live idx. The idx is writer-controlled, so after a +member body is downloaded a pack far over `small_pack_bytes` is left unmerged and +reported. Report byte counts on a carrier are those idx sizes (log entries, +checkpoints, idx files and a pack with no readable idx count as 0, and the +report says so). + +**Leased deletes.** `Bucket` gains `list_tags` (keys with their tags, one +listing) and `delete_leased` (delete each `(key, tag)` only if the key still +holds that tag; the default has no lease and deletes unconditionally). The Git +carrier pushes the deletions with one `--force-with-lease=refs/sc/K:` and +one `:refs/sc/K` refspec per key (never `+`, never `--atomic`, at most 128 refs +and about 24 KiB of arguments per push). A key that was re-created since the +listing is at another oid and survives; a key still at its lease after a refusal +(a hook) is an error. The sweep deletes each key leased to the oid of the one +listing whose age it judged; nothing lists twice. + +**Authoritative manifest check.** Before every destructive batch (the log and +checkpoint deletes, and each batch of packs: idx files first, then bodies) the +compactor reads the manifest with `get("manifest", None)` and compares the tag. +A conditional read with the cached tag is not enough: the Git carrier answers +one from its 2 s freshness window with no network call, and a stale view is the +one thing the pack deletes cannot tolerate. This applies to every backend. If the +manifest moved before the first batch the plan is stale and is redone; later, +the rest is kept and the report says so. Order per batch: age record write (if +changed), manifest check, deletes. + +**What remains.** The race named under "Residual limit" is narrowed, not closed. +A pusher that finds an aged orphan pack already present (`put_new` returns +false and leaves the oid unchanged, because the bytes are identical) and commits +its manifest after the compactor's check but before the delete lands can still +lose the pack: no lease on the manifest exists to close that. A push also keeps +its pack only if it commits within the grace window of the compactor first +seeing the pack. Both are the contract this ADR already states for every +backend. A crash between the age record write and the deletes leaves every key +in place with its age recorded, so the next run decides again and finishes the +job; a crash between an idx delete and its pack delete leaves a lone `.pack`, +which ages out on its own. + +**Operating it.** Run compaction from one host: runs that share the record need +clocks within far less than the grace window. On a Git host every pusher holds +delete (a push to `refs/sc/*` is a push), unlike the advice above to keep delete +rights away from pushers; branch protection does not cover `refs/sc/*`. +Deleting a ref is not erasure: the host may keep the objects fetchable by id, +and forks and mirrors keep their own copies; sealed content stays ciphertext, so +rotation (`sc rewrap --rotate`) is the cutover. Tested over a bare repository +and sc-forge; not tested against GitHub. + +As built: `compact/sightings.rs` (the record), `compact.rs` (`list_for_sweep`, +`delete_keys`, `manifest_unmoved`), `WalView.pack_sizes`, `Bucket::list_tags` +and `delete_leased` (`GitBucket` overrides both). Tests: +`compact/tests/git.rs`, `crates/objio/src/git.rs`, `crates/cli/tests/git_bucket.rs`; +proof: `demo/run_git_carrier_demo.sh`. diff --git a/docs/adr/0060-git-carried-bucket-remotes.md b/docs/adr/0060-git-carried-bucket-remotes.md index d70cabff..3165f9c8 100644 --- a/docs/adr/0060-git-carried-bucket-remotes.md +++ b/docs/adr/0060-git-carried-bucket-remotes.md @@ -4,7 +4,8 @@ - **Date:** 2026-11-21 - **Phase:** 39 - **Builds on:** ADR-0046 (bucket WAL remotes), ADR-0044 (private branches), - ADR-0058 (sc-forge, addendum for `refs/sc/*`) + ADR-0058 (sc-forge, addendum for `refs/sc/*`), ADR-0057 (compaction; its + addendum lifts the v1 compaction refusal) - **Amends:** ADR-0020 (dependency rule: `objio` may spawn the system `git` binary), ADR-0044 (reason 4 for rejecting opaque sealed refs through the Git bridge, reversed only under a per-remote opt-in) @@ -86,6 +87,10 @@ So a hook or permission refusal never looks like a lost race, which `BucketTransport` would report as a non-fast-forward and the user would retry forever. A tag that is not a full nonzero hex object id returns `None`. Delete pushes `:refs/sc/K` and treats an already absent ref as success. +`delete_leased` (used by compaction) pushes `:refs/sc/K` with +`--force-with-lease=refs/sc/K:` for every key of a chunk, never `+` and +never `--atomic`: a key re-created since the caller's listing holds another oid +and survives; a key still at its lease after a refused push is an error. ### Reads: freshness, create-only caching, family batching @@ -95,8 +100,12 @@ under the temp directory (mode 0700), removed on `Drop`. - A `get` whose cached tag equals the tag this instance observed or wrote for that key less than 2 s ago returns `Unchanged` with no network call. `BucketTransport::has_object` refreshes per object, which would otherwise cost - one `ls-remote` each. Staleness is bounded and only causes a failed server-side - CAS or a redundant upload. + one `ls-remote` each. For a pusher, staleness is bounded by the window and + normally costs a failed server-side CAS or a redundant upload. It is not + harmless for a delete: a view up to 2 s old can miss a push that has just + named a pack, so compaction never trusts it. It reads the manifest with no + cache (`get("manifest", None)`) and compares tags before every destructive + batch (ADR-0057 addendum). - A key is create-only by its layout alone: `log/`, `checkpoints/`, or a `.pack` or `.idx` under `packs/`. Never by anything the writer controls, such as a commit message. This instance serves such a key from the local cache with no @@ -181,13 +190,34 @@ path to durably publishing embargo ciphertext on a public host, does apply, and is why it is gated and why the notice and the refusal message say what stays behind. +### Compaction + +`sc remote compact` works on an `sc+git::` remote (ADR-0057 addendum has the +algorithm; v1 refused it). A Git repository gives the compactor no trustworthy +age for a stored pack: committer time is writer-controlled and predates +publication. So on a carrier the compactor keeps its own record of when it first +saw each pack and checkpoint (`refs/sc/compact/sightings`: ` +` per line, generation-aware by oid), sizes packs from their idx instead of +downloading them (`list_meta` still downloads every value and compaction never +calls it here), deletes with leases (`delete_leased`), and reads the manifest +with no cache before every destructive batch. The first run after an object +appears never deletes it (with a zero grace, `--allow-short-grace`, a first +sighting is immediately old). Consequences to know: + +- Run compaction from one host. Runs that share the record need clocks within + far less than the grace window; a first-seen time more than five minutes ahead + of the compactor's clock is refused. +- Every pusher holds delete, unlike the ADR-0057 advice to withhold it: on a Git + host a push to `refs/sc/*` is a push. The record adds no authority (anyone who + can write it can already delete the store); it can only make something + unreferenced look older, never a live object deletable. +- A push that stalls longer than the grace window between uploading a pack and + committing it can lose the pack, as on any backend; byte-identical reuse of an + aged orphan pack at the moment its delete lands remains a narrowed race. +- Deleting a ref is not erasure (see Consequences). Untested on GitHub. + ### Deferred in v1 -- `sc remote compact` on an `sc+git::` remote is refused. A Git repository gives - the compactor no trustworthy age for a stored pack: committer time is - writer-controlled and predates publication, so it cannot drive pruning. - `list_meta` exists for the trait contract and reports the observation time as - `modified`. - `sc serve --store sc+git::...` is refused. A daemon cannot answer ssh or credential prompts, and it would sit between clients and the client-side private-branch gate. @@ -231,9 +261,9 @@ every transport path. follow-ons (ROADMAP: P34 follow-on 6, a fetch-time freshness attestation; ADR-0059 follow-on 2). - **Deleting a ref is not erasure.** A host may keep the objects fetchable by - id, forks and mirrors keep their own copies, and compaction (when it exists - for Git) only unreferences values. Sealed content stays ciphertext, so - rotation is the cutover (`sc rewrap --rotate`), as for any transport. + id, forks and mirrors keep their own copies, and compaction only unreferences + values. Sealed content stays ciphertext, so rotation is the cutover + (`sc rewrap --rotate`), as for any transport. - **Public content is plaintext in packs** on the host, as on any bucket remote. - **Sealed metadata is readable by every reader of the repository:** private branch names, recipient ids, sealed-object counts and sizes, and the public @@ -242,7 +272,7 @@ every transport path. - A killed process (SIGKILL, or SIGINT, which skips `Drop`) can leave `sc-git-bucket--` behind in the temp directory. It holds fetched store objects. There is no reaper. A normal exit and an error returned to `main` remove it. -- v1 scaling limit: every `ls-remote` and push downloads the whole ref advertisement (the pattern given to `ls-remote` filters on the client; a packet trace showed the server sends every ref, protocol v2 `ref-prefix` or not), and a cold read fetches every `log/` and `checkpoints/` entry. With compaction refused, refs grow by at least three per push (a log entry, a pack and its idx), so a long-lived store gets slower to open until compaction or a ref-filtering host exists. +- Scaling: every `ls-remote` and push downloads the whole ref advertisement (the pattern given to `ls-remote` filters on the client; a packet trace showed the server sends every ref, protocol v2 `ref-prefix` or not), and a cold read fetches every `log/` and `checkpoints/` entry behind the last checkpoint. Refs grow by at least three per push (a log entry, a pack and its idx); `sc remote compact` (above) is what bounds that, folding the small packs and deleting what nothing names, so a long-lived store should be compacted on a schedule. Between runs the advertisement still grows with the pushes since the last one, until a ref-filtering host exists. - Each cold read or push costs git round trips. The freshness window and family batching bound them, but a Git host is slower than S3. If push over `sc+git::` stays slow on large histories, the next step is a batched `has_objects`. @@ -272,7 +302,9 @@ every transport path. `crates/objio/src/git.rs` (`GitBucket`, the contract suite and race tests); `crates/repo/src/bucket_transport.rs` (`BucketScheme::Git`, `is_bucket_url`, -`check_private_push_allowed`, `refuse_git_store`, `refuse_git_compact`); +`check_private_push_allowed`, `refuse_git_store`); +`crates/repo/src/bucket_transport/compact/` (the age record and the carrier +sweep, ADR-0057 addendum); `crates/repo/src/remote.rs` (`allow_private_branches`); `crates/cli` (flags, notice, URL routing); `crates/forge/src/policy.rs` (`refs/sc/`). Tests: `crates/objio/src/git.rs`, `crates/repo/src/bucket_transport/git_remote_tests.rs`, From 9bde1d01724a277821e431e1dfa071b1193e4ed0 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 15:08:36 +0200 Subject: [PATCH 4/7] test(repo): pin that the age record is written before the first delete The crash test now uploads a pack that only the crashing run sees, so the run has something new to record, and asserts the record holds it at that run's time when the crash comes at each delete call. Also: the sweep comments and the exhaustion note say that log and checkpoint deletes wait for the uncached manifest check too (a busy remote can defer cleanup to a rerun), ADR-0057 and CLAUDE.md say the same, and two doc sentences are tightened. --- CLAUDE.md | 7 +++-- ROADMAP.md | 2 +- crates/repo/src/bucket_transport/compact.rs | 13 +++++--- .../src/bucket_transport/compact/tests/git.rs | 31 +++++++++++++++---- docs/adr/0057-bucket-compaction.md | 11 ++++--- 5 files changed, 46 insertions(+), 18 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c62c485f..9e9e7c2a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -262,9 +262,10 @@ transport-adjacent. The rest, imperatively: not hold. It advances the manifest only through the normal CAS (one no-op entry plus a checkpoint) and never deletes what the current chain names. Keep `--prune-expire` well above any push's duration (default 24h): a writer stalled longer than that between uploading a pack and its - commit can still lose the pack (the leases follow-on closes it). On an - `sc+git::` remote the manifest is read uncached before every delete batch - (every backend) and every pusher holds delete. + commit can still lose the pack (the leases follow-on closes it). On every + backend the manifest is read uncached before every delete batch (a busy remote + can defer cleanup to a rerun); on an `sc+git::` remote every pusher also holds + delete. `--dry-run` writes and deletes nothing but still downloads and decodes every candidate pack, so on S3 its download egress and read requests match a real run (the uploads, CAS and deletes it skips are the rest). diff --git a/ROADMAP.md b/ROADMAP.md index 7a5b89f3..5f50f879 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1041,7 +1041,7 @@ P28 review follow-ons, the port de-flake, the clippy allow-list) are recorded in access is the cross-cutting signed-ref effort already tracked under P34 follow-on (6) and ADR-0059 follow-on (2), now reachable by more people; a client-side "highest manifest seq seen" floor was reviewed and judged a false - control (a fresh clone has no floor); (5b) a batched `has_objects`: the 2 s + control; (5b) a batched `has_objects`: the 2 s freshness window already limits the per-object manifest probe to one network call per 2 s; (8) SHA-256 Git repositories (v1 pins its cache to SHA-1): GitHub does not support them and sc-forge is SHA-1, so the value is low and the cost diff --git a/crates/repo/src/bucket_transport/compact.rs b/crates/repo/src/bucket_transport/compact.rs index 2595a5c5..e56bd4bf 100644 --- a/crates/repo/src/bucket_transport/compact.rs +++ b/crates/repo/src/bucket_transport/compact.rs @@ -527,9 +527,12 @@ impl BucketTransport { } } - // Log entries and checkpoints behind the cutoff are safe whatever - // the manifest does meanwhile: the cutoff only ever grows, so they - // stay unreachable. Counters add up across replans. + // Log entries and checkpoints behind the cutoff would stay + // unreachable whatever the manifest does meanwhile (the cutoff only + // ever grows), but nothing is deleted on a picture the manifest has + // moved past, so they too wait for the uncached manifest check below + // (a busy remote can defer their cleanup to a rerun). Counters add up + // across replans. rep.cutoff = rep.cutoff.max(cutoff); // Git carrier: pack sizes for the report, from the idx (the listing // carries none); a pack with no readable idx counts as 0 bytes. @@ -614,7 +617,9 @@ impl BucketTransport { return Ok(()); } rep.notes.push( - "the manifest kept moving during the sweep: unreferenced packs were kept, rerun".into(), + "the manifest kept moving during the sweep: what it had not yet deleted (log entries, \ + checkpoints, unreferenced packs) was kept, rerun" + .into(), ); Ok(()) } diff --git a/crates/repo/src/bucket_transport/compact/tests/git.rs b/crates/repo/src/bucket_transport/compact/tests/git.rs index 1b228fa0..7ad8bb5c 100644 --- a/crates/repo/src/bucket_transport/compact/tests/git.rs +++ b/crates/repo/src/bucket_transport/compact/tests/git.rs @@ -831,6 +831,10 @@ fn a_failure_between_the_record_and_each_delete_call_leaves_a_readable_store_a_r // idx files of the aged packs, then their bodies. Fail before each one. for fail_at in 0..3usize { let hub = template.copy(&format!("git-crash-{fail_at}")); + // A pack that appears only now, so the crashing run has something new + // to record: the record must already hold it when the crash comes. + let fresh = orphan(&hub, "crash-orphan"); + let fresh_keys = [format!("packs/{fresh}.idx"), format!("packs/{fresh}.pack")]; let calls = Arc::new(AtomicUsize::new(0)); let hook: Hook = Box::new(move |op, _, after| { if op == "delete_leased" && !after && calls.fetch_add(1, Ordering::SeqCst) == fail_at { @@ -844,26 +848,41 @@ fn a_failure_between_the_record_and_each_delete_call_leaves_a_readable_store_a_r .unwrap_err() .to_string(); assert!(err.contains("injected crash"), "{err}"); + // The record was written before the first delete was tried: it already + // holds the new pack, first seen at this run's time (and not before). + let rec = hub.record().unwrap(); + for k in &fresh_keys { + assert_eq!(rec.first_seen(k), Some(secs(start + 2 * DAY)), "{k}"); + } // What the crash left is a bucket a cold start reads. assert_readable(&hub, &refs); match fail_at { 0 => { assert_eq!(hub.count("refs/sc/log/"), 13); - assert_eq!(hub.count("refs/sc/packs/"), 2 * 13); + assert_eq!(hub.count("refs/sc/packs/"), 2 * 14); } 1 => { assert_eq!(hub.count("refs/sc/log/"), 0); - assert_eq!(hub.count("refs/sc/packs/"), 2 * 13); + assert_eq!(hub.count("refs/sc/packs/"), 2 * 14); } _ => { // idx files gone, bodies left: plain debris, aged on its own next run - assert_eq!(hub.count(".idx"), 1); - assert_eq!(hub.count(".pack"), 13); + assert_eq!(hub.count(".idx"), 2); + assert_eq!(hub.count(".pack"), 14); } } - // A healthy rerun finishes the job. + // A healthy rerun finishes the job. The new pack is not old enough yet, + // so it survives that run, and everything else unreferenced is gone. hub.carrier().compact(&at(start + 2 * DAY)).unwrap(); - tight(&hub); + for k in &fresh_keys { + assert!( + hub.bucket().exists(k).unwrap(), + "{k} was deleted before it aged" + ); + } + assert_eq!(hub.refs().len(), 5 + fresh_keys.len(), "{:?}", hub.refs()); + assert_eq!(hub.count("refs/sc/log/"), 0); + assert_eq!(hub.count("refs/sc/checkpoints/"), 1); } } diff --git a/docs/adr/0057-bucket-compaction.md b/docs/adr/0057-bucket-compaction.md index 79c41e8e..fac1162e 100644 --- a/docs/adr/0057-bucket-compaction.md +++ b/docs/adr/0057-bucket-compaction.md @@ -340,10 +340,13 @@ checkpoint deletes, and each batch of packs: idx files first, then bodies) the compactor reads the manifest with `get("manifest", None)` and compares the tag. A conditional read with the cached tag is not enough: the Git carrier answers one from its 2 s freshness window with no network call, and a stale view is the -one thing the pack deletes cannot tolerate. This applies to every backend. If the -manifest moved before the first batch the plan is stale and is redone; later, -the rest is kept and the report says so. Order per batch: age record write (if -changed), manifest check, deletes. +one thing the pack deletes cannot tolerate. This applies to every backend, and +to the log and checkpoint deletes too, which the cutoff alone would have made +safe: nothing is deleted on a picture the manifest has moved past, so a remote +that is pushed to continuously can defer cleanup to a rerun (the report says so). +If the manifest moved before the first batch the plan is stale and is redone; +later, the rest is kept. Order per batch: age record write (if changed), +manifest check, deletes. **What remains.** The race named under "Residual limit" is narrowed, not closed. A pusher that finds an aged orphan pack already present (`put_new` returns From 7f3a805201db25754ebe100e4d5086ea4e9517d2 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 16:10:03 +0200 Subject: [PATCH 5/7] fix(repo): review fixes for compaction on sc+git:: Each behavior fix has a test that fails without it (checked by reverting it). - The manifest check is pinned: the late-push test now runs over a bucket that answers a conditional manifest read Unchanged from its freshness window, so a check that reuses the cached tag deletes the pack and fails the test. - A hash is deleted only when every present object of it has aged: a test ages a lone .pack, uploads a fresh .idx, and expects both to survive. - The sweep clock is read per attempt, so a key first listed after a replan is not stamped with the time the sweep began. - A retained log entry or checkpoint whose tag differs from its listed tag keeps every candidate pack (a carrier serves create-only keys from its cache). - Byte sums saturate (a hostile idx can declare u64::MAX, which panicked a debug build), including the merge planner's group sum. - The note for exhausted attempts names the age record when that is what lost; the carrier byte-count note appears whenever a carrier run deletes anything. Docs and comments: report counts are attempts, from_bucket takes the unleased path (Git carriers must use open), the stale list_meta comment, the late-push test comment, THREAT-MODEL (an understating idx is downloaded whole), ADR-0057, and the decoy-branch push failure recorded as a known issue in ROADMAP. --- ROADMAP.md | 7 +- crates/objio/src/git.rs | 7 +- crates/repo/src/bucket_transport.rs | 6 + crates/repo/src/bucket_transport/compact.rs | 140 ++++--- .../src/bucket_transport/compact/tests/git.rs | 387 ++++++++++++++++-- docs/THREAT-MODEL.md | 7 +- docs/adr/0057-bucket-compaction.md | 18 +- 7 files changed, 471 insertions(+), 101 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 5f50f879..2d8b7a6d 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1047,7 +1047,12 @@ P28 review follow-ons, the port de-flake, the clippy allow-list) are recorded in does not support them and sc-forge is SHA-1, so the value is low and the cost moderate (the cache's object format would have to be detected before the first fetch); (S2, from review) the ssh user name that `remote list` prints is not a - credential, and `http(s)` userinfo is refused at validation. + credential, and `http(s)` userinfo is refused at validation. Known issue, + recorded and not fixed: a push to `refs/sc/K` fails with "dst refspec matches + more than one" when a decoy branch such as `refs/heads/refs/sc/K` exists on the + host (found while building `serve --store`). Whoever can push to the + repository can plant such a decoy, so it is a writer-only denial of service on + that key; reads compare exact full ref names and are not affected. - **Serve-side persistent pack cache (P36c follow-on).** A bucket-backed serve instance re-downloads packs per connection; a content-addressed on-disk cache in the serve home would make warm instances cheap without diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index 0d8ad8e6..da95a309 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -1114,9 +1114,10 @@ impl Bucket for GitBucket { .collect()) } - // shortcut: downloads every listed value to read its size; only compaction - // (refused on git remotes in v1) and tests call it. Upgrade to a filtered - // fetch if listing large stores becomes slow. + // shortcut: downloads every listed value to read its size; only tests and + // callers that opened a carrier as a plain bucket call it (compaction on a + // carrier lists with `list_tags` and sizes packs from their idx). Upgrade to + // a filtered fetch if listing large stores becomes slow. fn list_meta(&self, prefix: &str) -> Result> { check_prefix(prefix)?; let listed = self.remote_list(prefix)?; diff --git a/crates/repo/src/bucket_transport.rs b/crates/repo/src/bucket_transport.rs index 0f4d4371..db9a22b7 100644 --- a/crates/repo/src/bucket_transport.rs +++ b/crates/repo/src/bucket_transport.rs @@ -338,6 +338,12 @@ impl BucketTransport { /// Fails only if the initial `refresh` (a conditional GET of the /// manifest plus a walk of its parent chain) errors — an entirely empty /// bucket is a valid, successfully-opened remote with no refs yet. + /// + /// A Git carrier must be opened with [`BucketTransport::open`] and an + /// `sc+git::` URL, not through here: this constructor cannot tell the + /// bucket is one, so `compact` takes the plain-backend path (bucket + /// modification times and unleased deletes), which a Git host cannot make + /// safe. It stays public for tests and for callers with their own backend. pub fn from_bucket(bucket: Box) -> Result { Self::build(bucket, false) } diff --git a/crates/repo/src/bucket_transport/compact.rs b/crates/repo/src/bucket_transport/compact.rs index e56bd4bf..d040d7d3 100644 --- a/crates/repo/src/bucket_transport/compact.rs +++ b/crates/repo/src/bucket_transport/compact.rs @@ -79,10 +79,16 @@ pub struct CompactReport { pub checkpoint_seq: u64, /// The sweep cutoff `K` (0 when no checkpoint is old enough yet). pub cutoff: u64, + /// The deletion counts below are what the run attempted (what a dry run + /// would attempt): a key another writer re-created after the listing + /// survives its leased delete, and one another compactor already removed is + /// already gone, and both are still counted. pub logs_deleted: usize, pub checkpoints_deleted: usize, /// Pack hashes removed (each is a `.pack` and a `.idx`). pub packs_deleted: usize, + /// Bytes behind the deletions above, saturating (an `sc+git::` store + /// reports pack bodies as their idx declares them, which a writer controls). pub bytes_deleted: u64, /// Unreferenced packs kept because they are younger than `prune_expire`. pub packs_held_back: usize, @@ -109,18 +115,21 @@ fn hash_of(key: &str) -> Option<&str> { .or_else(|| name.strip_suffix(".idx")) } +/// Sum that saturates: a size a writer controls (an idx's declared length) can +/// be `u64::MAX`, and an overflow would panic a debug build on every run. +fn total(sizes: impl IntoIterator) -> u64 { + sizes.into_iter().fold(0, u64::saturating_add) +} + /// Bytes of the `.pack` and `.idx` objects behind these packs; a pack in /// `declared` (a Git carrier, which lists no sizes) counts that many instead. fn pack_bytes(packs: &[(&String, &Vec)], declared: &BTreeMap) -> u64 { - packs - .iter() - .map(|(h, ms)| { - declared - .get(*h) - .copied() - .unwrap_or_else(|| ms.iter().map(|m| m.size).sum()) - }) - .sum() + total(packs.iter().map(|(h, ms)| { + declared + .get(*h) + .copied() + .unwrap_or_else(|| total(ms.iter().map(|m| m.size))) + })) } /// What one sweep attempt sees in the bucket. @@ -152,12 +161,12 @@ fn plan_groups( let mut groups: Vec> = Vec::new(); let (mut cur, mut sum): (Vec, u64) = (Vec::new(), 0); for (size, hash) in small { - if !cur.is_empty() && sum + size > opts.group_bytes { + if !cur.is_empty() && sum.saturating_add(size) > opts.group_bytes { groups.push(std::mem::take(&mut cur)); sum = 0; } cur.push(hash.clone()); - sum += size; + sum = sum.saturating_add(size); } groups.push(cur); groups.retain(|g| g.len() >= 2); @@ -195,9 +204,14 @@ impl BucketTransport { .map_or(0, |v| v.manifest.checkpoint_seq); self.consolidate(opts, &mut rep)?; } - // Read the clock after phase 1 so a zero grace sees its own checkpoint. - let now = opts.now.unwrap_or_else(SystemTime::now); - self.sweep(opts, now, &mut rep)?; + self.sweep(opts, &mut rep)?; + if self.git_carrier && rep.logs_deleted + rep.checkpoints_deleted + rep.packs_deleted > 0 { + rep.notes.push( + "Git carrier: byte counts are the pack bodies their idx declares; log entries, \ + checkpoints and idx files count as 0, as does a pack with no readable idx" + .into(), + ); + } Ok(rep) } @@ -451,13 +465,19 @@ impl BucketTransport { } /// Phase 2: delete what is behind the cutoff and what nothing names. - fn sweep(&self, opts: &CompactOptions, now: SystemTime, rep: &mut CompactReport) -> Result<()> { - // `modified <= old_before` means "at least prune_expire old". - let old_before = now - .checked_sub(opts.prune_expire) - .unwrap_or(SystemTime::UNIX_EPOCH); - let old = |m: &ObjectMeta| m.modified <= old_before; + fn sweep(&self, opts: &CompactOptions, rep: &mut CompactReport) -> Result<()> { + // What stopped the last attempt, for the note if they all stop. + let mut lost_record = false; 'attempt: for _ in 0..MAX_ATTEMPTS { + // The clock is read per attempt, after phase 1 (so a zero grace sees + // its own checkpoint) and again on every replan: a key first listed + // in a later attempt is not stamped with the time the sweep began. + let now = opts.now.unwrap_or_else(SystemTime::now); + // `modified <= old_before` means "at least prune_expire old". + let old_before = now + .checked_sub(opts.prune_expire) + .unwrap_or(SystemTime::UNIX_EPOCH); + let old = |m: &ObjectMeta| m.modified <= old_before; self.refresh()?; let (tag, ck_seq, live) = match self.view.borrow().as_ref() { Some(v) => ( @@ -475,8 +495,10 @@ impl BucketTransport { tags, }) = self.list_for_sweep(opts, now, rep)? else { + lost_record = true; continue; }; + lost_record = false; let cutoff = ckpts .iter() .filter(|(s, m)| *s <= ck_seq && old(m)) @@ -514,7 +536,7 @@ impl BucketTransport { if !aged.is_empty() { // Retained structure may still name them: every kept // checkpoint and every kept log entry (on-chain or not). - match self.retained_packs(&ckpts, &logs, cutoff) { + match self.retained_packs(&ckpts, &logs, cutoff, &tags) { Ok(named) => { del_packs = aged .into_iter() @@ -523,7 +545,7 @@ impl BucketTransport { } Err(e) => rep .notes - .push(format!("packs kept, a retained record is unreadable: {e}")), + .push(format!("packs kept, a retained record cannot be used: {e}")), } } @@ -537,20 +559,20 @@ impl BucketTransport { // Git carrier: pack sizes for the report, from the idx (the listing // carries none); a pack with no readable idx counts as 0 bytes. let declared = if self.git_carrier { - self.declared_sizes(&del_packs, rep) + self.declared_sizes(&del_packs) } else { BTreeMap::new() }; if opts.dry_run { rep.logs_deleted += del_logs.len(); rep.checkpoints_deleted += del_ckpts.len(); - rep.bytes_deleted += del_logs - .iter() - .chain(del_ckpts.iter()) - .map(|m| m.size) - .sum::(); + rep.bytes_deleted = rep.bytes_deleted.saturating_add(total( + del_logs.iter().chain(del_ckpts.iter()).map(|m| m.size), + )); rep.packs_deleted += del_packs.len(); - rep.bytes_deleted += pack_bytes(&del_packs, &declared); + rep.bytes_deleted = rep + .bytes_deleted + .saturating_add(pack_bytes(&del_packs, &declared)); if rep.packs_created > 0 { rep.notes.push( "dry run: the sweep figures describe the bucket as it is now; the merge \ @@ -571,11 +593,9 @@ impl BucketTransport { } rep.logs_deleted += del_logs.len(); rep.checkpoints_deleted += del_ckpts.len(); - rep.bytes_deleted += del_logs - .iter() - .chain(del_ckpts.iter()) - .map(|m| m.size) - .sum::(); + rep.bytes_deleted = rep.bytes_deleted.saturating_add(total( + del_logs.iter().chain(del_ckpts.iter()).map(|m| m.size), + )); let keys: Vec<&str> = del_logs .iter() .chain(del_ckpts.iter()) @@ -602,7 +622,9 @@ impl BucketTransport { return Ok(()); } rep.packs_deleted += batch.len(); - rep.bytes_deleted += pack_bytes(batch, &declared); + rep.bytes_deleted = rep + .bytes_deleted + .saturating_add(pack_bytes(batch, &declared)); // idx first: a half-deleted pack is then plain debris. for suffix in [".idx", ".pack"] { let keys: Vec<&str> = batch @@ -616,11 +638,18 @@ impl BucketTransport { } return Ok(()); } - rep.notes.push( - "the manifest kept moving during the sweep: what it had not yet deleted (log entries, \ - checkpoints, unreferenced packs) was kept, rerun" - .into(), - ); + rep.notes.push(if lost_record { + format!( + "the age record refs/sc/{} kept losing its compare-and-swap to another \ + compactor ({MAX_ATTEMPTS} attempts): nothing was deleted, rerun (and run \ + compaction from one host)", + sightings::KEY + ) + } else { + "the manifest kept moving during the sweep: what it had not yet deleted (log \ + entries, checkpoints, unreferenced packs) was kept, rerun" + .into() + }); Ok(()) } @@ -660,11 +689,7 @@ impl BucketTransport { /// Pack body sizes of `packs` as their idx declares them (a Git carrier's /// listing carries no sizes), for the report only. A pack whose idx cannot be /// read is left out and counts as 0 bytes; the report says so. - fn declared_sizes( - &self, - packs: &[(&String, &Vec)], - rep: &mut CompactReport, - ) -> BTreeMap { + fn declared_sizes(&self, packs: &[(&String, &Vec)]) -> BTreeMap { let mut sizes = BTreeMap::new(); for (hash, _) in packs { if let Ok(Fetched::New { bytes, .. }) = self.bucket.get(&idx_key(hash), None) { @@ -673,11 +698,6 @@ impl BucketTransport { } } } - let note = "Git carrier: byte counts are the pack bodies their idx declares; log entries, \ - checkpoints and idx files count as 0, as does a pack with no readable idx"; - if !packs.is_empty() && !rep.notes.iter().any(|n| n == note) { - rep.notes.push(note.into()); - } sizes } @@ -805,19 +825,32 @@ impl BucketTransport { /// Pack hashes named by structure the sweep keeps: checkpoints with /// `seq >= cutoff` and log entries with `seq > cutoff` (everything when /// `cutoff` is 0). Any record that fails to decode aborts: deleting packs - /// on a partial picture is never right. + /// on a partial picture is never right. On a Git carrier a record whose tag + /// differs from the one in this attempt's listing (`tags`) aborts too: a + /// carrier serves a create-only key it has already read from its cache, so + /// the bytes would be a value the listing no longer shows, and a writer that + /// deleted and re-created the entry could be hiding what it names. The packs + /// are kept, and a rerun in a new process reads the entry afresh. fn retained_packs( &self, ckpts: &[(u64, ObjectMeta)], logs: &[(u64, ObjectMeta)], cutoff: u64, + tags: &BTreeMap, ) -> Result> { + let listed = |key: &str, got: &str| match tags.get(key) { + Some(want) if want != got => Err(Error::Wal(format!( + "{key} changed since the listing it was judged against" + ))), + _ => Ok(()), + }; let mut named = BTreeSet::new(); for (seq, m) in ckpts { if cutoff == 0 || *seq >= cutoff { - let Fetched::New { bytes, .. } = self.bucket.get(&m.key, None)? else { + let Fetched::New { bytes, tag } = self.bucket.get(&m.key, None)? else { continue; // deleted since the listing: no longer retained }; + listed(&m.key, &tag)?; named.extend( Checkpoint::decode(&capped("checkpoint", bytes)?) .map_err(|e| Error::Wal(format!("{}: {e}", m.key)))? @@ -827,9 +860,10 @@ impl BucketTransport { } for (seq, m) in logs { if cutoff == 0 || *seq > cutoff { - let Fetched::New { bytes, .. } = self.bucket.get(&m.key, None)? else { + let Fetched::New { bytes, tag } = self.bucket.get(&m.key, None)? else { continue; }; + listed(&m.key, &tag)?; named.extend( LogEntry::decode(&capped("log entry", bytes)?) .map_err(|e| Error::Wal(format!("{}: {e}", m.key)))? diff --git a/crates/repo/src/bucket_transport/compact/tests/git.rs b/crates/repo/src/bucket_transport/compact/tests/git.rs index 7ad8bb5c..af5ef801 100644 --- a/crates/repo/src/bucket_transport/compact/tests/git.rs +++ b/crates/repo/src/bucket_transport/compact/tests/git.rs @@ -122,14 +122,12 @@ impl Hub { /// A compactor whose bucket calls pass through `hook(op, key, after)`. fn hooked(&self, hook: Hook) -> BucketTransport { - BucketTransport::build( - Box::new(Wrapped { - inner: Box::new(GitBucket::open(&self.url()).unwrap()), - hook, - }), - true, - ) - .unwrap() + BucketTransport::build(Box::new(self.wrapped(hook)), true).unwrap() + } + + /// The bucket behind [`Hub::hooked`], for a test that configures it further. + fn wrapped(&self, hook: Hook) -> Wrapped { + Wrapped::new(Box::new(GitBucket::open(&self.url()).unwrap()), hook) } /// `refs/sc/*` names on the hub, sorted. @@ -226,12 +224,44 @@ type Hook = Box scl_objio::Result<()> + Send>; /// Forwards every operation, including `list_tags` and `delete_leased`, through /// a hook. `list_meta` is refused: it downloads every value on a Git carrier, so /// compaction must never ask for it. +/// +/// It can also play the host's caches, both of which a real `GitBucket` has: +/// [`Wrapped::stale_window`] answers a conditional `manifest` read with +/// `Unchanged` while the tag matches the last one served (the 2 s freshness +/// window), and [`Wrapped::pin`] keeps serving the value a create-only key had +/// when it was read (the carrier answers such a key from its cache). struct Wrapped { inner: Box, hook: Hook, + stale_window: bool, + last_manifest: Mutex>, + pinned: Mutex, String)>>, } impl Wrapped { + fn new(inner: Box, hook: Hook) -> Wrapped { + Wrapped { + inner, + hook, + stale_window: false, + last_manifest: Mutex::new(None), + pinned: Mutex::new(BTreeMap::new()), + } + } + + /// Answer a conditional manifest read from the last tag served, never asking + /// the bucket: the freshness window of `GitBucket`, made deterministic. + fn stale_window(mut self) -> Wrapped { + self.stale_window = true; + self + } + + /// Serve `key` as `(bytes, tag)` whatever the bucket now holds. + fn pin(self, key: &str, bytes: Vec, tag: String) -> Wrapped { + self.pinned.lock().unwrap().insert(key.into(), (bytes, tag)); + self + } + fn around( &self, op: &str, @@ -247,7 +277,26 @@ impl Wrapped { impl Bucket for Wrapped { fn get(&self, key: &str, tag: Option<&str>) -> scl_objio::Result { - self.around("get", key, || self.inner.get(key, tag)) + self.around("get", key, || { + if let Some((bytes, pinned)) = self.pinned.lock().unwrap().get(key) { + return Ok(Fetched::New { + bytes: bytes.clone(), + tag: pinned.clone(), + }); + } + if key != "manifest" || !self.stale_window { + return self.inner.get(key, tag); + } + let mut last = self.last_manifest.lock().unwrap(); + if tag.is_some() && tag == last.as_deref() { + return Ok(Fetched::Unchanged); + } + let got = self.inner.get(key, tag)?; + if let Fetched::New { tag, .. } = &got { + *last = Some(tag.clone()); + } + Ok(got) + }) } fn put_new(&self, key: &str, bytes: &[u8]) -> scl_objio::Result { self.around("put_new", key, || self.inner.put_new(key, bytes)) @@ -736,8 +785,14 @@ fn a_key_re_created_between_the_listing_and_the_delete_survives_its_lease() { assert_readable(&hub, &refs); } -/// The scenario of a pusher that reuses an aged orphan pack by identical bytes -/// and lands before the compactor looks at the manifest again. +/// A pusher reuses an aged orphan pack by identical bytes and its commit lands +/// after the compactor listed, but before the compactor's manifest check. This +/// proves the check, and that it reads the manifest uncached: the bucket here +/// answers a conditional manifest read `Unchanged` from its freshness window +/// (as `GitBucket` does for 2 s), so a check that reused the cached tag would +/// call the manifest unmoved and delete the pack. What it does not prove, and +/// nothing can without a lease on the manifest, is the window between the check +/// and the delete: a commit landing there is the residual race ADR-0057 names. #[test] fn a_push_landing_after_the_listing_is_seen_by_the_manifest_check_and_its_pack_kept() { need_git!(); @@ -766,7 +821,8 @@ fn a_push_landing_after_the_listing_is_seen_by_the_manifest_check_and_its_pack_k } Ok(()) }); - let rep = hub.hooked(hook).compact(&at(start + 2 * DAY)).unwrap(); + let t = BucketTransport::build(Box::new(hub.wrapped(hook).stale_window()), true).unwrap(); + let rep = t.compact(&at(start + 2 * DAY)).unwrap(); for k in &keys { assert!(hub.bucket().exists(k).unwrap(), "{k} was deleted: {rep:?}"); } @@ -936,15 +992,9 @@ fn a_failure_at_every_operation_boundary_on_a_git_carrier_leaves_a_readable_buck } Ok(()) }); - let outcome = BucketTransport::build( - Box::new(Wrapped { - inner: work.bucket(), - hook, - }), - true, - ) - // A zero grace sees every first sighting as already old: one run deletes. - .and_then(|t| t.compact(&eager())); + let outcome = BucketTransport::build(Box::new(Wrapped::new(work.bucket(), hook)), true) + // A zero grace sees every first sighting as already old: one run deletes. + .and_then(|t| t.compact(&eager())); // Whatever happened, a fresh reader still sees everything. assert_readable(&work, &refs); @@ -983,14 +1033,7 @@ fn a_manifest_that_moves_during_leased_pack_deletion_stops_the_deletes() { } Ok(()) }); - let t = BucketTransport::build( - Box::new(Wrapped { - inner: mem.bucket(), - hook, - }), - true, - ) - .unwrap(); + let t = BucketTransport::build(Box::new(Wrapped::new(mem.bucket(), hook)), true).unwrap(); let rep = t.compact(&eager()).unwrap(); assert!( rep.notes @@ -1036,6 +1079,22 @@ fn the_double_leases_like_git_and_a_replayed_compaction_changes_nothing() { assert_readable(&mem, &refs); } +/// `idx` with every entry rewritten to start at `offset` and be `length` long: +/// an idx is the writer's word about its pack. +fn lying_idx(idx: &[u8], offset: u64, length: u64) -> Vec { + let entries = scl_core::pack::parse_index(idx).unwrap(); + let mut lie = Vec::new(); + lie.extend_from_slice(b"SCIX"); + lie.extend_from_slice(&1u32.to_le_bytes()); + lie.extend_from_slice(&(entries.len() as u64).to_le_bytes()); + for e in &entries { + lie.extend_from_slice(e.id.as_bytes()); + lie.extend_from_slice(&offset.to_le_bytes()); + lie.extend_from_slice(&length.to_le_bytes()); + } + lie +} + /// A push's objects and the tip it moves a branch to. type Push = (Vec<(ObjectId, Vec)>, ObjectId); @@ -1125,17 +1184,7 @@ fn packs_are_sized_from_their_idx_and_one_far_over_its_idx_is_left_unmerged() { if i != 2 { return idx; } - let entries = scl_core::pack::parse_index(&idx).unwrap(); - let mut lie = Vec::new(); - lie.extend_from_slice(b"SCIX"); - lie.extend_from_slice(&1u32.to_le_bytes()); - lie.extend_from_slice(&(entries.len() as u64).to_le_bytes()); - for e in &entries { - lie.extend_from_slice(e.id.as_bytes()); - lie.extend_from_slice(&8u64.to_le_bytes()); - lie.extend_from_slice(&1u64.to_le_bytes()); - } - lie + lying_idx(&idx, 8, 1) }, ); let opts = CompactOptions { @@ -1183,3 +1232,261 @@ fn the_idx_length_is_the_end_of_the_furthest_record_and_saturates() { assert_eq!(declared_pack_len(&[huge(8, u64::MAX)]), u64::MAX); assert_eq!(declared_pack_len(&[huge(u64::MAX - 3, 0)]), u64::MAX); } + +// ---- review fixes: ages per member, per-attempt clock, retained tags, sums --- + +/// A hook that never interferes. +fn quiet() -> Hook { + Box::new(|_, _, _| Ok(())) +} + +/// Put the pack of `commit(label)` straight into `b` as an orphan; `idx_of` may +/// rewrite its idx. Returns the pack hash. +fn put_orphan(b: &dyn Bucket, label: &str, idx_of: impl Fn(Vec) -> Vec) -> String { + let (_, objects) = commit(label); + let (pack, idx) = scl_core::pack::build_pack(&objects).unwrap(); + let hash = hex::encode(blake3::hash(&pack).as_bytes()); + b.put_new(&pack_key(&hash), &pack).unwrap(); + b.put_new(&idx_key(&hash), &idx_of(idx)).unwrap(); + hash +} + +/// The age record in `s`. +fn record_in(s: &impl Store) -> Sightings { + let Fetched::New { bytes, .. } = s.bucket().get(sightings::KEY, None).unwrap() else { + panic!("no record") + }; + Sightings::decode(&bytes, u64::MAX / 2).unwrap() +} + +#[test] +fn a_pack_is_deleted_only_when_every_object_of_it_has_aged() { + let mem = Arc::new(GitLike::default()); + let refs = hand_built(&mem, 12, 10); + let start = t0(); + // The body alone (what an idx-first crash leaves), sighted by run 1. + let (_, objects) = commit("lone"); + let (pack, idx) = scl_core::pack::build_pack(&objects).unwrap(); + let hash = hex::encode(blake3::hash(&pack).as_bytes()); + let b = mem.bucket(); + b.put_new(&pack_key(&hash), &pack).unwrap(); + like_carrier(&mem).compact(&at(start)).unwrap(); + // A fresh idx for the same hash appears afterwards. + b.put_new(&idx_key(&hash), &idx).unwrap(); + let both = |mem: &Arc| { + let b = mem.bucket(); + ( + b.exists(&pack_key(&hash)).unwrap(), + b.exists(&idx_key(&hash)).unwrap(), + ) + }; + // Two days on the body is old and the idx is not: neither goes. + let r2 = like_carrier(&mem).compact(&at(start + 2 * DAY)).unwrap(); + assert_eq!(both(&mem), (true, true), "{r2:?}"); + assert_eq!(r2.packs_held_back, 1, "{r2:?}"); + // Once the idx has aged too, the pair goes. + like_carrier(&mem).compact(&at(start + 4 * DAY)).unwrap(); + assert_eq!(both(&mem), (false, false)); + assert_readable(&mem, &refs); +} + +#[test] +fn a_key_first_listed_in_a_replan_is_stamped_with_that_attempts_clock() { + let mem = Arc::new(GitLike::default()); + hand_built(&mem, 12, 10); + let late = mem.clone(); + let state = Arc::new(AtomicUsize::new(0)); + let hook_at = Arc::new(AtomicU64::new(0)); + let (st, at_secs) = (state.clone(), hook_at.clone()); + let hook: Hook = Box::new(move |op, key, after| { + match (op, after, st.load(Ordering::SeqCst)) { + ("list_tags", true, 0) => st.store(1, Ordering::SeqCst), + // The first manifest read after the listing: a push lands, and a + // while passes before the compactor looks again. + ("get", false, 1) if key == "manifest" => { + st.store(2, Ordering::SeqCst); + push_commits(&open(&late), "late", 1); + std::thread::sleep(Duration::from_millis(1100)); + at_secs.store(secs(SystemTime::now()), Ordering::SeqCst); + } + _ => {} + } + Ok(()) + }); + // The wall clock (no injected `now`), zero grace so the first attempt reaches + // its manifest check and replans. + BucketTransport::build(Box::new(Wrapped::new(mem.bucket(), hook)), true) + .unwrap() + .compact(&eager()) + .unwrap(); + assert_eq!(state.load(Ordering::SeqCst), 2, "the push never landed"); + // The late push's pack exists only from the second attempt on. + let (_, objects) = commit("late-0"); + let (pack, _) = scl_core::pack::build_pack(&objects).unwrap(); + let hash = hex::encode(blake3::hash(&pack).as_bytes()); + let stamped = record_in(&mem) + .first_seen(&pack_key(&hash)) + .expect("the late pack is tracked"); + assert!( + stamped >= hook_at.load(Ordering::SeqCst), + "first seen at {stamped}, but the replan began at {}", + hook_at.load(Ordering::SeqCst) + ); +} + +#[test] +fn a_retained_entry_that_changed_since_the_listing_keeps_the_packs_it_might_name() { + let mem = Arc::new(GitLike::default()); + let refs = hand_built(&mem, 12, 10); + let start = t0(); + let b = mem.bucket(); + let hidden = put_orphan(b.as_ref(), "hidden", |idx| idx); + let keys = [ + format!("packs/{hidden}.idx"), + format!("packs/{hidden}.pack"), + ]; + like_carrier(&mem).compact(&at(start)).unwrap(); // sights the orphan + let head = manifest(&mem).head_seq; + let entry = |packs: Vec| { + LogEntry { + seq: head + 1, + parent_seq: head, + packs, + updates: vec![], + } + .encode() + }; + let slot = log_key(head + 1); + assert!(b.put_new(&slot, &entry(vec![])).unwrap()); + let Fetched::New { bytes, tag } = b.get(&slot, None).unwrap() else { + panic!() + }; + // A writer deletes that entry and creates it again naming the orphan. A + // carrier that already read the first one keeps serving it from its cache. + b.delete(&slot).unwrap(); + assert!(b.put_new(&slot, &entry(vec![hidden.clone()])).unwrap()); + let stale = Wrapped::new(mem.bucket(), quiet()).pin(&slot, bytes, tag); + let rep = BucketTransport::build(Box::new(stale), true) + .unwrap() + .compact(&at(start + 2 * DAY)) + .unwrap(); + for k in &keys { + assert!(b.exists(k).unwrap(), "{k} was deleted: {rep:?}"); + } + assert!( + rep.notes + .iter() + .any(|n| n.contains("packs kept") && n.contains("changed since the listing")), + "{rep:?}" + ); + // A fresh process reads the entry as it is and keeps the pack it names. + like_carrier(&mem).compact(&at(start + 2 * DAY)).unwrap(); + for k in &keys { + assert!(b.exists(k).unwrap(), "{k} was deleted by the rerun"); + } + assert_readable(&mem, &refs); +} + +#[test] +fn byte_counts_saturate_when_an_idx_declares_an_absurd_length() { + let mem = Arc::new(GitLike::default()); + let refs = hand_built(&mem, 12, 10); + let start = t0(); + let b = mem.bucket(); + // Two orphans whose idx puts every record at the end of the address space. + let lies: Vec = ["lie-1", "lie-2"] + .iter() + .map(|l| put_orphan(b.as_ref(), l, |idx| lying_idx(&idx, u64::MAX, 1))) + .collect(); + like_carrier(&mem).compact(&at(start)).unwrap(); + // A sum of two such sizes overflows a u64 (a panic in a debug build). + let dry = like_carrier(&mem) + .compact(&CompactOptions { + dry_run: true, + ..at(start + 2 * DAY) + }) + .unwrap(); + assert_eq!( + (dry.packs_deleted, dry.bytes_deleted), + (14, u64::MAX), + "{dry:?}" + ); + let real = like_carrier(&mem).compact(&at(start + 2 * DAY)).unwrap(); + assert_eq!( + (real.packs_deleted, real.bytes_deleted), + (14, u64::MAX), + "{real:?}" + ); + for h in &lies { + assert!(!b.exists(&pack_key(h)).unwrap()); + } + assert_readable(&mem, &refs); +} + +#[test] +fn an_age_record_that_keeps_losing_its_cas_is_named_in_the_note() { + let mem = Arc::new(GitLike::default()); + hand_built(&mem, 12, 10); + let rival = mem.bucket(); + let n = Arc::new(AtomicUsize::new(0)); + // Every time the compactor is about to write the record, another compactor + // gets in first with a record of its own. + let hook: Hook = Box::new(move |op, key, after| { + if op == "put_if_tag" && key == sightings::KEY && !after { + let i = n.fetch_add(1, Ordering::SeqCst); + let cur = match rival.get(sightings::KEY, None)? { + Fetched::New { tag, .. } => Some(tag), + _ => None, + }; + let line = format!("1 {} packs/rival{i}.pack\n", "c".repeat(40)); + rival.put_if_tag(sightings::KEY, line.as_bytes(), cur.as_deref())?; + } + Ok(()) + }); + let rep = BucketTransport::build(Box::new(Wrapped::new(mem.bucket(), hook)), true) + .unwrap() + .compact(&eager()) + .unwrap(); + assert!( + rep.notes + .iter() + .any(|n| n.contains("age record") && n.contains("compare-and-swap")), + "{rep:?}" + ); + assert!(!rep.notes.iter().any(|n| n.contains("manifest kept moving"))); + assert_eq!( + (rep.logs_deleted, rep.checkpoints_deleted, rep.packs_deleted), + (0, 0, 0), + "{rep:?}" + ); +} + +#[test] +fn the_byte_note_is_there_whenever_a_carrier_run_deletes_anything() { + let mem = Arc::new(GitLike::default()); + hand_built(&mem, 12, 10); + let start = t0(); + // No merging, so the second run deletes log entries only (no pack). + let opts = |now| CompactOptions { + small_pack_bytes: 0, + ..at(now) + }; + let first = like_carrier(&mem).compact(&opts(start)).unwrap(); + assert!( + !first.notes.iter().any(|n| n.contains("byte counts")), + "nothing was deleted, nothing was counted: {first:?}" + ); + let second = like_carrier(&mem).compact(&opts(start + 2 * DAY)).unwrap(); + assert_eq!( + (second.logs_deleted, second.packs_deleted), + (10, 0), + "{second:?}" + ); + assert!( + second + .notes + .iter() + .any(|n| n.contains("byte counts are the pack bodies their idx declares")), + "{second:?}" + ); +} diff --git a/docs/THREAT-MODEL.md b/docs/THREAT-MODEL.md index a3edb860..90379452 100644 --- a/docs/THREAT-MODEL.md +++ b/docs/THREAT-MODEL.md @@ -781,7 +781,12 @@ The bucket model above applies unchanged, with the Git repository as the bucket re-created since survives, and the manifest is read uncached before every destructive batch. Left open: a pusher reusing an aged orphan pack byte for byte can still lose it if the delete lands after the manifest check and before its commit (no lease on the manifest), and a push - that stalls past the grace window loses its pack, as on any backend. **Every pusher holds + that stalls past the grace window loses its pack, as on any backend. A writer can also make the + compactor download a pack it did not need to: sizes for the merge plan come from the pack's idx, + which the writer controls, so a pack whose idx understates it is downloaded whole (up to the + 2 GiB value cap, in RAM) before the real length is checked and the pack is left unmerged. + A retained log entry or checkpoint whose value changed since the listing (a writer deleted and + re-created it) keeps every candidate pack for that run. **Every pusher holds delete** (a push to `refs/sc/*` is a push); unlike S3, delete rights cannot be withheld from them. Run compaction from one host (clocks). - **Not built in v1.** `sc serve --store sc+git::` is refused (a daemon cannot answer prompts diff --git a/docs/adr/0057-bucket-compaction.md b/docs/adr/0057-bucket-compaction.md index fac1162e..3e49541d 100644 --- a/docs/adr/0057-bucket-compaction.md +++ b/docs/adr/0057-bucket-compaction.md @@ -320,10 +320,22 @@ bound. the compactor never calls it there. A live pack's size for the merge plan is the end of its furthest record in its idx (`offset + 4 + length`, saturating), taken while the view parses each live idx. The idx is writer-controlled, so after a -member body is downloaded a pack far over `small_pack_bytes` is left unmerged and -reported. Report byte counts on a carrier are those idx sizes (log entries, +member body is downloaded a pack far over `small_pack_bytes` (twice it) is left +unmerged and reported; the body is already in RAM by then (up to the 2 GiB value +cap), so an understating idx costs the compactor a download. Report byte counts +on a carrier are those idx sizes, summed with saturation (log entries, checkpoints, idx files and a pack with no readable idx count as 0, and the -report says so). +report says so whenever the run deletes anything). Deletion counts are what the +run attempted: a key that survives its lease, or that another compactor already +removed, is still counted. + +**Clock and retained records.** The clock is read per sweep attempt, so a key +first listed after a replan is stamped with that attempt's time, not the time the +sweep began. A retained log entry or checkpoint is compared, by tag, with the +tag its own listing showed: a carrier serves a create-only key it has already +read from its cache, so a mismatch (a writer deleted and re-created the entry) +means the bytes read are not the value the listing shows, and every candidate +pack is kept for that run (a rerun, in a new process, reads it afresh). **Leased deletes.** `Bucket` gains `list_tags` (keys with their tags, one listing) and `delete_leased` (delete each `(key, tag)` only if the key still From 4033f3da11b8002284b023bbce4740aa14d529ca Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 16:25:15 +0200 Subject: [PATCH 6/7] fix(repo): stamp sightings after the listing, check the manifest before the body deletes Each fix has a test that fails without it (checked by reverting it). - The sweep clock on a carrier is read after the tagged listing returns, and that one instant is both the first-seen stamp and the age cutoff. Before, it was read at the start of the attempt: a process that stalled before listing stamped a pack uploaded during the stall with an older time, and the next run could delete it while its push was still staging. - A batch of pack deletes is two calls (idx files, then bodies). The uncached manifest check now runs before each: a writer can reuse a body that is still there (put_new leaves its oid unchanged), create the idx again and commit between the calls, and the body's lease still matched. A stop in that window keeps the bodies and says so. Byte counts are added per call so a stop reports only what was attempted. - The default Bucket::delete_leased is an error instead of an unconditional delete, so a lease is never silently dropped. Only the Git carrier overrides it; compaction deletes from the other backends with plain delete (tested). The lease contract checks move to leased_delete_suite, run on GitBucket only. --- ARCHITECTURE.md | 2 +- CLAUDE.md | 4 +- crates/objio/src/git.rs | 6 +- crates/objio/src/lib.rs | 48 +++++-- crates/repo/src/bucket_transport/compact.rs | 122 ++++++++++++------ .../src/bucket_transport/compact/tests.rs | 17 +++ .../src/bucket_transport/compact/tests/git.rs | 95 +++++++++++++- docs/THREAT-MODEL.md | 2 +- docs/adr/0057-bucket-compaction.md | 29 +++-- docs/adr/0060-git-carried-bucket-remotes.md | 2 +- 10 files changed, 253 insertions(+), 74 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index fe95d213..d0585364 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1387,7 +1387,7 @@ decision uses it. Pack sizes for the merge plan come from each live idx (`offset length`), never from `list_meta`, which would download every value. Deletes are leased (`Bucket::delete_leased`: one `git push` of `:refs/sc/K` with a `--force-with-lease=refs/sc/K:` per key, the oid the one listing showed), and the -manifest is read uncached before every destructive batch. The first run after an object +manifest is read uncached before every delete call. The first run after an object appears never deletes it; run compaction from one host. A private branch goes to an `sc+git::` remote only under the per-remote opt-in diff --git a/CLAUDE.md b/CLAUDE.md index 9e9e7c2a..6bc8beb4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -234,7 +234,7 @@ transport-adjacent. The rest, imperatively: `refs/sc/compact/sightings` (` `, generation-aware, strict decode: a bad record blocks the run, a time over five minutes ahead of the compactor's clock is refused), leased deletes (`Bucket::delete_leased`), and - an uncached manifest read before every destructive batch. The first run + an uncached manifest read before every delete call. The first run deletes nothing it did not already see (zero grace: a first sighting is immediately old); run it from one host (clocks). Every git call goes through `objio::GitBucket` with a scrubbed environment; CAS is `--force-with-lease`, @@ -263,7 +263,7 @@ transport-adjacent. The rest, imperatively: entry plus a checkpoint) and never deletes what the current chain names. Keep `--prune-expire` well above any push's duration (default 24h): a writer stalled longer than that between uploading a pack and its commit can still lose the pack (the leases follow-on closes it). On every - backend the manifest is read uncached before every delete batch (a busy remote + backend the manifest is read uncached before every delete call (a busy remote can defer cleanup to a rerun); on an `sc+git::` remote every pusher also holds delete. `--dry-run` writes and deletes nothing but still downloads and decodes every diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index da95a309..3b74d39f 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -1256,7 +1256,7 @@ impl Bucket for GitBucket { #[cfg(test)] mod tests { use super::*; - use crate::contract_suite; + use crate::{contract_suite, leased_delete_suite}; use std::path::Path; use std::process::Command; use std::sync::atomic::{AtomicU64, Ordering}; @@ -1371,7 +1371,9 @@ mod tests { need_git!(); let t = Tmp::new("contract"); let (_, url) = hub(&t); - contract_suite(&open(&url)); + let b = open(&url); + contract_suite(&b); + leased_delete_suite(&b); } #[test] diff --git a/crates/objio/src/lib.rs b/crates/objio/src/lib.rs index 39145547..51dd78e2 100644 --- a/crates/objio/src/lib.rs +++ b/crates/objio/src/lib.rs @@ -90,14 +90,14 @@ pub trait Bucket: Send { /// that now holds another value (or is already gone) is left alone and is /// not an error; a key that still holds its expected tag but could not be /// removed is an error. Several keys go in one call so a backend can batch - /// them. This default has no lease to check, so it deletes each key - /// unconditionally (what the backends without conditional deletes do); a - /// backend that can lease must override it (the Git carrier does). - fn delete_leased(&self, items: &[(String, String)]) -> Result<()> { - for (key, _) in items { - self.delete(key)?; - } - Ok(()) + /// them. A backend that cannot check a lease keeps this default, an error: + /// a lease is never silently dropped by deleting unconditionally. Only the + /// Git carrier overrides it, and compaction asks only a Git carrier for it + /// (the other backends are deleted from with [`Bucket::delete`]). + fn delete_leased(&self, _items: &[(String, String)]) -> Result<()> { + Err(Error::Backend( + "leased delete is not supported by this backend".into(), + )) } } @@ -190,10 +190,13 @@ pub fn contract_suite(b: &dyn Bucket) { assert_eq!(b.list_meta("packs/").unwrap().len(), 2); // a deleted key can be created again assert!(b.put_new("packs/ab12.pack", b"again").unwrap()); - // delete_leased: removes keys holding their expected tag, is idempotent, - // touches only its keys. Whether a stale tag survives is up to the backend - // (the default cannot lease), so that is checked in each lease-capable - // backend's own tests. +} + +/// Conformance checks for a backend that overrides [`Bucket::delete_leased`] +/// (only the Git carrier does): it removes keys holding their expected tag, is +/// idempotent, and touches only its keys. Whether a stale tag survives is +/// checked in that backend's own tests. Panics on violation (test helper). +pub fn leased_delete_suite(b: &dyn Bucket) { assert!(b.put_new("lease/a", b"1").unwrap()); assert!(b.put_new("lease/b", b"2").unwrap()); assert!(b.put_new("lease/keep", b"3").unwrap()); @@ -223,4 +226,25 @@ mod tests { std::fs::remove_dir_all(&root).unwrap(); assert!(!root.exists()); } + + /// A backend that cannot check a lease must refuse, not delete without one. + #[test] + fn a_backend_without_leases_refuses_a_leased_delete_and_deletes_nothing() { + let root = std::env::temp_dir().join(format!("scl-objio-nolease-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let b = DirBucket::open(&root).unwrap(); + assert!(b.put_new("k", b"v").unwrap()); + let Fetched::New { tag, .. } = b.get("k", None).unwrap() else { + panic!("k should exist") + }; + let err = b.delete_leased(&[("k".into(), tag)]).unwrap_err(); + assert!( + matches!(&err, Error::Backend(m) if m.contains("not supported")), + "{err}" + ); + assert!(b.exists("k").unwrap(), "a refused lease must not delete"); + assert!(b.list_tags("").is_err(), "no tagged listing either"); + std::fs::remove_dir_all(&root).unwrap(); + assert!(!root.exists()); + } } diff --git a/crates/repo/src/bucket_transport/compact.rs b/crates/repo/src/bucket_transport/compact.rs index d040d7d3..40813e22 100644 --- a/crates/repo/src/bucket_transport/compact.rs +++ b/crates/repo/src/bucket_transport/compact.rs @@ -121,19 +121,38 @@ fn total(sizes: impl IntoIterator) -> u64 { sizes.into_iter().fold(0, u64::saturating_add) } -/// Bytes of the `.pack` and `.idx` objects behind these packs; a pack in -/// `declared` (a Git carrier, which lists no sizes) counts that many instead. -fn pack_bytes(packs: &[(&String, &Vec)], declared: &BTreeMap) -> u64 { +/// Bytes of these packs' objects whose key ends with `suffix` (`.idx` or +/// `.pack`); a pack in `declared` (a Git carrier, which lists no sizes) counts +/// its declared body length under `.pack` and nothing under `.idx`. +fn pack_bytes( + packs: &[(&String, &Vec)], + declared: &BTreeMap, + suffix: &str, +) -> u64 { total(packs.iter().map(|(h, ms)| { - declared - .get(*h) - .copied() - .unwrap_or_else(|| total(ms.iter().map(|m| m.size))) + match declared.get(*h) { + Some(d) if suffix == ".pack" => *d, + Some(_) => 0, + None => total( + ms.iter() + .filter(|m| m.key.ends_with(suffix)) + .map(|m| m.size), + ), + } })) } /// What one sweep attempt sees in the bucket. struct Listing { + /// The instant the attempt's ages are judged at (`opts.now` overrides the + /// clock). On a Git carrier it is read AFTER the tagged listing returns and + /// serves both as the first-seen stamp of every new key and as the age + /// cutoff: a key that appeared while this process stalled before listing is + /// stamped no earlier than the listing that showed it, never with an older + /// time from before the stall. Elsewhere ages come from the bucket's own + /// modification times and the instant is read before listing, the + /// conservative side. + now: SystemTime, ckpts: Vec<(u64, ObjectMeta)>, logs: Vec<(u64, ObjectMeta)>, /// `.pack` and `.idx` objects by pack hash. @@ -469,15 +488,6 @@ impl BucketTransport { // What stopped the last attempt, for the note if they all stop. let mut lost_record = false; 'attempt: for _ in 0..MAX_ATTEMPTS { - // The clock is read per attempt, after phase 1 (so a zero grace sees - // its own checkpoint) and again on every replan: a key first listed - // in a later attempt is not stamped with the time the sweep began. - let now = opts.now.unwrap_or_else(SystemTime::now); - // `modified <= old_before` means "at least prune_expire old". - let old_before = now - .checked_sub(opts.prune_expire) - .unwrap_or(SystemTime::UNIX_EPOCH); - let old = |m: &ObjectMeta| m.modified <= old_before; self.refresh()?; let (tag, ck_seq, live) = match self.view.borrow().as_ref() { Some(v) => ( @@ -489,16 +499,23 @@ impl BucketTransport { }; // A lost race on the age record means someone else updated it: replan. let Some(Listing { + now, ckpts, logs, by_hash, tags, - }) = self.list_for_sweep(opts, now, rep)? + }) = self.list_for_sweep(opts, rep)? else { lost_record = true; continue; }; lost_record = false; + // `modified <= old_before` means "at least prune_expire old". The + // instant is the listing's own (see `Listing::now`). + let old_before = now + .checked_sub(opts.prune_expire) + .unwrap_or(SystemTime::UNIX_EPOCH); + let old = |m: &ObjectMeta| m.modified <= old_before; let cutoff = ckpts .iter() .filter(|(s, m)| *s <= ck_seq && old(m)) @@ -570,9 +587,11 @@ impl BucketTransport { del_logs.iter().chain(del_ckpts.iter()).map(|m| m.size), )); rep.packs_deleted += del_packs.len(); - rep.bytes_deleted = rep - .bytes_deleted - .saturating_add(pack_bytes(&del_packs, &declared)); + for suffix in [".idx", ".pack"] { + rep.bytes_deleted = rep + .bytes_deleted + .saturating_add(pack_bytes(&del_packs, &declared, suffix)); + } if rep.packs_created > 0 { rep.notes.push( "dry run: the sweep figures describe the bucket as it is now; the merge \ @@ -605,27 +624,15 @@ impl BucketTransport { } // A pack nothing names can still be named by a push that lands // while we plan, and the delete loop can run long, so the manifest - // is looked at before every batch: if it moved before the first, the - // plan is stale (replan); later, what is left is kept. A push that - // lands meanwhile may have re-used one of these old packs byte for - // byte. + // is looked at before every delete call, and a batch is two calls + // (idx files, then bodies): a writer can reuse a body that is still + // there (its `put_new` leaves the oid unchanged), recreate the idx + // we just deleted and commit between the two, and the body's lease + // would still match. If the manifest moved before the first call the + // plan is stale (replan); later, what is left is kept. Idx first: a + // half-deleted pack is then plain debris. for (i, batch) in del_packs.chunks(PACK_DELETE_BATCH).enumerate() { - if !self.manifest_unmoved(&tag)? { - if i == 0 { - continue 'attempt; - } - rep.notes.push(format!( - "the manifest moved during pack deletion: {} unreferenced pack(s) kept, \ - rerun", - del_packs.len() - i * PACK_DELETE_BATCH - )); - return Ok(()); - } - rep.packs_deleted += batch.len(); - rep.bytes_deleted = rep - .bytes_deleted - .saturating_add(pack_bytes(batch, &declared)); - // idx first: a half-deleted pack is then plain debris. + let mut sent = false; // a delete call of this batch has gone out for suffix in [".idx", ".pack"] { let keys: Vec<&str> = batch .iter() @@ -633,8 +640,29 @@ impl BucketTransport { .map(|m| m.key.as_str()) .filter(|k| k.ends_with(suffix)) .collect(); + if keys.is_empty() { + continue; // lone bodies after an idx-first crash have no idx + } + if !self.manifest_unmoved(&tag)? { + if i == 0 && !sent { + continue 'attempt; + } + rep.notes.push(format!( + "the manifest moved during pack deletion: {} unreferenced pack(s) \ + kept, rerun", + del_packs.len() - i * PACK_DELETE_BATCH + )); + return Ok(()); + } self.delete_keys(&keys, &tags)?; + sent = true; + // Counted once its delete call went out, so a stop above + // reports only what was attempted. + rep.bytes_deleted = rep + .bytes_deleted + .saturating_add(pack_bytes(batch, &declared, suffix)); } + rep.packs_deleted += batch.len(); } return Ok(()); } @@ -727,10 +755,12 @@ impl BucketTransport { fn list_for_sweep( &self, opts: &CompactOptions, - now: SystemTime, rep: &mut CompactReport, ) -> Result> { if !self.git_carrier { + // Ages here are the bucket's own modification times, so the clock + // only sets the cutoff: read before listing, the conservative side. + let now = opts.now.unwrap_or_else(SystemTime::now); let metas = |prefix: &str| -> Result> { Ok(self .bucket @@ -746,14 +776,21 @@ impl BucketTransport { } } return Ok(Some(Listing { + now, ckpts: metas("checkpoints/")?, logs: metas("log/")?, by_hash, tags: BTreeMap::new(), })); } - let now_secs = unix_secs(now)?; let listed = self.bucket.list_tags("")?; + // The clock is read only now, after the listing returned (and again on + // every replan): any key in `listed` was on the host before this instant, + // so its first-seen stamp cannot predate its own listing even if this + // process stalled before it. A stall after the listing only makes a stamp + // later, which holds an object back longer. + let now = opts.now.unwrap_or_else(SystemTime::now); + let now_secs = unix_secs(now)?; let (before, record_tag) = self.read_sightings(now_secs)?; let seen = before.observe(&listed, now_secs); let added = seen.added_since(&before); @@ -815,6 +852,7 @@ impl BucketTransport { tags.insert(key, oid); } Ok(Some(Listing { + now, ckpts, logs, by_hash, diff --git a/crates/repo/src/bucket_transport/compact/tests.rs b/crates/repo/src/bucket_transport/compact/tests.rs index a587459b..f22c4943 100644 --- a/crates/repo/src/bucket_transport/compact/tests.rs +++ b/crates/repo/src/bucket_transport/compact/tests.rs @@ -262,6 +262,23 @@ fn the_in_memory_double_honors_the_bucket_contract() { scl_objio::contract_suite(&Shared(Arc::new(MemBucket::default()))); } +#[test] +fn plain_backends_refuse_a_leased_delete_and_compaction_never_asks_for_one() { + // The double keeps the trait default, like `DirBucket` and `S3Bucket`. + let mem = Arc::new(MemBucket::default()); + let err = Shared(mem.clone()) + .delete_leased(&[("packs/a.pack".into(), "x".into())]) + .unwrap_err(); + assert!(err.to_string().contains("not supported"), "{err}"); + // Compaction there deletes through plain `delete`: one leased call, or one + // tagged listing, would fail this run. + let t = open(&mem); + let refs = push_commits(&t, "w", 70); + let rep = t.compact(&eager()).unwrap(); + assert!(rep.packs_deleted > 0 && rep.logs_deleted > 0, "{rep:?}"); + assert_readable(&mem, &refs); +} + #[test] fn merges_small_packs_and_sweeps_everything_behind_the_checkpoint() { let root = scratch("full"); diff --git a/crates/repo/src/bucket_transport/compact/tests/git.rs b/crates/repo/src/bucket_transport/compact/tests/git.rs index af5ef801..67ce8294 100644 --- a/crates/repo/src/bucket_transport/compact/tests/git.rs +++ b/crates/repo/src/bucket_transport/compact/tests/git.rs @@ -1014,8 +1014,9 @@ fn a_failure_at_every_operation_boundary_on_a_git_carrier_leaves_a_readable_buck } /// The leased analogue of the pack-deletion stop on the plain backends: a push -/// lands while the first batch of packs is going, and the next batch's manifest -/// check keeps the rest. +/// lands once the first batch of packs is gone, and the next batch's manifest +/// check keeps the rest. (A push landing between a batch's idx and body deletes +/// is `a_reuse_between_the_idx_delete_and_the_body_delete_keeps_the_body`.) #[test] fn a_manifest_that_moves_during_leased_pack_deletion_stops_the_deletes() { let mem = Arc::new(GitLike::default()); @@ -1023,10 +1024,10 @@ fn a_manifest_that_moves_during_leased_pack_deletion_stops_the_deletes() { let fired = Arc::new(AtomicBool::new(false)); let late = mem.clone(); let hook: Hook = Box::new(move |op, key, after| { - // After the idx files of the first batch went: someone else pushes. + // After the bodies of the first batch went: someone else pushes. if op == "delete_leased" && after - && key.starts_with("packs/") + && key.ends_with(".pack") && !fired.swap(true, Ordering::SeqCst) { push_commits(&open(&late), "late", 1); @@ -1259,6 +1260,92 @@ fn record_in(s: &impl Store) -> Sightings { Sightings::decode(&bytes, u64::MAX / 2).unwrap() } +#[test] +fn a_key_first_listed_in_a_run_is_stamped_no_earlier_than_its_listing() { + let mem = Arc::new(GitLike::default()); + hand_built(&mem, 12, 10); + let pusher = mem.clone(); + let stalled = Arc::new(AtomicBool::new(false)); + let listing_from = Arc::new(AtomicU64::new(0)); + let (once, from) = (stalled.clone(), listing_from.clone()); + let hook: Hook = Box::new(move |op, _, after| { + // The compactor stalls before its listing; a push uploads a pack + // meanwhile, and a while passes before the listing is taken. + if op == "list_tags" && !after && !once.swap(true, Ordering::SeqCst) { + put_orphan(pusher.bucket().as_ref(), "stall-0", |idx| idx); + std::thread::sleep(Duration::from_millis(1100)); + from.store(secs(SystemTime::now()), Ordering::SeqCst); + } + Ok(()) + }); + // The wall clock, no injected `now`. + BucketTransport::build(Box::new(Wrapped::new(mem.bucket(), hook)), true) + .unwrap() + .compact(&CompactOptions::default()) + .unwrap(); + assert!( + stalled.load(Ordering::SeqCst), + "the listing was never taken" + ); + let (_, objects) = commit("stall-0"); + let (pack, _) = scl_core::pack::build_pack(&objects).unwrap(); + let hash = hex::encode(blake3::hash(&pack).as_bytes()); + let stamped = record_in(&mem) + .first_seen(&pack_key(&hash)) + .expect("the pack is tracked"); + assert!( + stamped >= listing_from.load(Ordering::SeqCst), + "stamped {stamped}, before the listing that first showed it at {}", + listing_from.load(Ordering::SeqCst) + ); +} + +#[test] +fn a_reuse_between_the_idx_delete_and_the_body_delete_keeps_the_body() { + let mem = Arc::new(GitLike::default()); + let mut refs = hand_built(&mem, 12, 10); + let start = t0(); + let lone = orphan(&mem, "reuse-0"); + let keys = [format!("packs/{lone}.idx"), format!("packs/{lone}.pack")]; + like_carrier(&mem).compact(&at(start)).unwrap(); // sights the orphan + let late = mem.clone(); + let fired = Arc::new(AtomicBool::new(false)); + let hook: Hook = Box::new(move |op, key, after| { + // Right after the idx files went: a push reuses the body that is still + // there (its `put_new` leaves the body's oid unchanged), creates the idx + // again and commits. The body's lease would still match its old oid. + if op == "delete_leased" + && after + && key.ends_with(".idx") + && !fired.swap(true, Ordering::SeqCst) + { + push_commits(&open(&late), "reuse", 1); + } + Ok(()) + }); + let rep = BucketTransport::build(Box::new(Wrapped::new(mem.bucket(), hook)), true) + .unwrap() + .compact(&at(start + 2 * DAY)) + .unwrap(); + for k in &keys { + assert!( + mem.bucket().exists(k).unwrap(), + "{k} was deleted from under a live chain: {rep:?}" + ); + } + assert!( + rep.notes + .iter() + .any(|n| n.contains("moved during pack deletion")), + "{rep:?}" + ); + refs.push(("reuse-0".to_string(), commit("reuse-0").0)); + assert_readable(&mem, &refs); + // A rerun, seeing the pack live, leaves it and finishes the rest. + like_carrier(&mem).compact(&at(start + 2 * DAY)).unwrap(); + assert_readable(&mem, &refs); +} + #[test] fn a_pack_is_deleted_only_when_every_object_of_it_has_aged() { let mem = Arc::new(GitLike::default()); diff --git a/docs/THREAT-MODEL.md b/docs/THREAT-MODEL.md index 90379452..31e73c33 100644 --- a/docs/THREAT-MODEL.md +++ b/docs/THREAT-MODEL.md @@ -778,7 +778,7 @@ The bucket model above applies unchanged, with the Git repository as the bucket object deletable. A malformed record, one naming a key outside `packs/` and `checkpoints/`, or one dated more than five minutes ahead of the compactor's clock, blocks the run (it is never read as "everything is old"). Deletes are leased to the oid the listing showed, so a key - re-created since survives, and the manifest is read uncached before every destructive batch. + re-created since survives, and the manifest is read uncached before every delete call. Left open: a pusher reusing an aged orphan pack byte for byte can still lose it if the delete lands after the manifest check and before its commit (no lease on the manifest), and a push that stalls past the grace window loses its pack, as on any backend. A writer can also make the diff --git a/docs/adr/0057-bucket-compaction.md b/docs/adr/0057-bucket-compaction.md index 3e49541d..3ee8944f 100644 --- a/docs/adr/0057-bucket-compaction.md +++ b/docs/adr/0057-bucket-compaction.md @@ -329,9 +329,13 @@ report says so whenever the run deletes anything). Deletion counts are what the run attempted: a key that survives its lease, or that another compactor already removed, is still counted. -**Clock and retained records.** The clock is read per sweep attempt, so a key -first listed after a replan is stamped with that attempt's time, not the time the -sweep began. A retained log entry or checkpoint is compared, by tag, with the +**Clock and retained records.** On a carrier the clock is read per sweep attempt +and only AFTER the tagged listing returns: that one instant is both the first-seen +stamp of every new key and the age cutoff. A process that stalls before listing +therefore cannot stamp a pack that was uploaded during the stall with a time from +before it (the next run could otherwise delete it while its push is still +staging), and a key first listed after a replan gets that attempt's time. A +stall after the listing only makes stamps later. A retained log entry or checkpoint is compared, by tag, with the tag its own listing showed: a carrier serves a create-only key it has already read from its cache, so a mismatch (a writer deleted and re-created the entry) means the bytes read are not the value the listing shows, and every candidate @@ -339,7 +343,9 @@ pack is kept for that run (a rerun, in a new process, reads it afresh). **Leased deletes.** `Bucket` gains `list_tags` (keys with their tags, one listing) and `delete_leased` (delete each `(key, tag)` only if the key still -holds that tag; the default has no lease and deletes unconditionally). The Git +holds that tag; the default is an error, so a lease is never silently dropped by +deleting without one, and only the Git carrier overrides it: compaction deletes +from the other backends with plain `delete`). The Git carrier pushes the deletions with one `--force-with-lease=refs/sc/K:` and one `:refs/sc/K` refspec per key (never `+`, never `--atomic`, at most 128 refs and about 24 KiB of arguments per push). A key that was re-created since the @@ -347,18 +353,23 @@ listing is at another oid and survives; a key still at its lease after a refusal (a hook) is an error. The sweep deletes each key leased to the oid of the one listing whose age it judged; nothing lists twice. -**Authoritative manifest check.** Before every destructive batch (the log and -checkpoint deletes, and each batch of packs: idx files first, then bodies) the +**Authoritative manifest check.** Before every delete call (the log and +checkpoint deletes, and for each batch of packs the idx files and then, as a +separate call with its own check, the bodies) the compactor reads the manifest with `get("manifest", None)` and compares the tag. +The second check matters: a writer can reuse a body that is still there (its +`put_new` leaves the oid unchanged), create the idx we just deleted again and +commit between the two calls, and the body's lease would still match. A conditional read with the cached tag is not enough: the Git carrier answers one from its 2 s freshness window with no network call, and a stale view is the one thing the pack deletes cannot tolerate. This applies to every backend, and to the log and checkpoint deletes too, which the cutoff alone would have made safe: nothing is deleted on a picture the manifest has moved past, so a remote that is pushed to continuously can defer cleanup to a rerun (the report says so). -If the manifest moved before the first batch the plan is stale and is redone; -later, the rest is kept. Order per batch: age record write (if changed), -manifest check, deletes. +If the manifest moved before anything was deleted the plan is stale and is +redone; later (including between a batch's idx and body deletes), the rest is +kept and the report says so. Order per call: age record write (if changed), +manifest check, delete. **What remains.** The race named under "Residual limit" is narrowed, not closed. A pusher that finds an aged orphan pack already present (`put_new` returns diff --git a/docs/adr/0060-git-carried-bucket-remotes.md b/docs/adr/0060-git-carried-bucket-remotes.md index 3165f9c8..683c3176 100644 --- a/docs/adr/0060-git-carried-bucket-remotes.md +++ b/docs/adr/0060-git-carried-bucket-remotes.md @@ -200,7 +200,7 @@ saw each pack and checkpoint (`refs/sc/compact/sightings`: ` ` per line, generation-aware by oid), sizes packs from their idx instead of downloading them (`list_meta` still downloads every value and compaction never calls it here), deletes with leases (`delete_leased`), and reads the manifest -with no cache before every destructive batch. The first run after an object +with no cache before every delete call. The first run after an object appears never deletes it (with a zero grace, `--allow-short-grace`, a first sighting is immediately old). Consequences to know: From 21590614be584a67e1349e89469c2425b0423b36 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 16:30:04 +0200 Subject: [PATCH 7/7] fix(compact): charge a carrier body only when its .pack key is listed A lone aged .idx (body already gone) made the report count the declared length of a body that no longer exists. --- crates/repo/src/bucket_transport/compact.rs | 6 ++++-- .../src/bucket_transport/compact/tests.rs | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/crates/repo/src/bucket_transport/compact.rs b/crates/repo/src/bucket_transport/compact.rs index 40813e22..e3cdb4c7 100644 --- a/crates/repo/src/bucket_transport/compact.rs +++ b/crates/repo/src/bucket_transport/compact.rs @@ -123,15 +123,17 @@ fn total(sizes: impl IntoIterator) -> u64 { /// Bytes of these packs' objects whose key ends with `suffix` (`.idx` or /// `.pack`); a pack in `declared` (a Git carrier, which lists no sizes) counts -/// its declared body length under `.pack` and nothing under `.idx`. +/// its declared body length under `.pack`, only when its `.pack` key is listed +/// (a lone `.idx` deletes no body), and nothing under `.idx`. fn pack_bytes( packs: &[(&String, &Vec)], declared: &BTreeMap, suffix: &str, ) -> u64 { total(packs.iter().map(|(h, ms)| { + let has_body = || ms.iter().any(|m| m.key.ends_with(".pack")); match declared.get(*h) { - Some(d) if suffix == ".pack" => *d, + Some(d) if suffix == ".pack" && has_body() => *d, Some(_) => 0, None => total( ms.iter() diff --git a/crates/repo/src/bucket_transport/compact/tests.rs b/crates/repo/src/bucket_transport/compact/tests.rs index f22c4943..3600ffa6 100644 --- a/crates/repo/src/bucket_transport/compact/tests.rs +++ b/crates/repo/src/bucket_transport/compact/tests.rs @@ -1051,3 +1051,23 @@ fn candidates_that_are_each_large_once_decoded_are_not_merged_one_by_one() { } mod git; + +/// A Git carrier lists no sizes, so a pack's body is charged at its idx's +/// declared length, but only when the `.pack` key itself is listed: a lone +/// aged `.idx` (its body already gone) deletes no body bytes. +#[test] +fn declared_body_bytes_count_only_when_the_pack_key_is_listed() { + let meta = |key: &str| ObjectMeta { + key: key.into(), + size: 0, + modified: SystemTime::UNIX_EPOCH, + }; + let (lone, whole) = ("aa".to_string(), "bb".to_string()); + let lone_ms = vec![meta("packs/aa.idx")]; + let whole_ms = vec![meta("packs/bb.idx"), meta("packs/bb.pack")]; + let declared: BTreeMap = [(lone.clone(), 700), (whole.clone(), 50)].into(); + let packs = [(&lone, &lone_ms), (&whole, &whole_ms)]; + assert_eq!(pack_bytes(&packs, &declared, ".pack"), 50); + assert_eq!(pack_bytes(&packs, &declared, ".idx"), 0); + assert_eq!(pack_bytes(&packs[..1], &declared, ".pack"), 0); +}