From 9507666d684a1c5d3e9f517f9a3851bde50578f0 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 14:57:48 +0200 Subject: [PATCH 1/9] feat(objio): read a mutable key with one filtered fetch and watch every fetch The manifest read was `ls-remote` followed by `fetch`. It is now one `git fetch --depth=1 +refs/sc/K:refs/sc/K` under protocol v2, so the host filters its ref advertisement to that ref (verified with GIT_TRACE_PACKET against a local bare repository and against sc-forge over smart HTTP). A failed fetch falls back to an exact `ls-remote` to tell an absent ref from an error. Git resolves a refspec source that matches nothing exactly by its abbreviation rules, so a branch or tag spelled `refs/sc/K` after its own prefix could stand in for an absent key; the fetch carries five negative refspecs that keep the exact-name semantics of the old read. `git fetch` has no input-size limit, so every fetch now runs under a watchdog: `fetch.unpackLimit=1` puts the download into one `tmp_pack_*` file, the growth of `objects/pack` is polled every 50 ms, and git is killed over the budget (the key's value cap plus 0.1% and 1 MiB, 2 GiB for a family or batch) or after 120 s without progress. A fetch that ends between two polls is caught by a check after it exits. A breach removes the files the fetch added, every local ref and the lock files a killed git cannot release, and the error names the cap. The bound is disk per fetch, not RAM inside index-pack, not the total cache. Pipe readers are detached threads, so a helper that outlives a killed git cannot block the call. Serve mode (unix) runs git in its own process group and kills the group before reaping the leader; the CLI keeps Ctrl-C and ssh prompts by killing git alone. GitBucket::open_in puts the cache under a given directory and enables serve mode. GitBucket::reap_stale removes caches of dead owners only, so several `sc serve --stdio` processes can share one home. Tests: one filtered fetch per manifest read with the packet trace, decoys at all five abbreviations, oversized planted value in both modes and both check paths with a working instance afterwards, a runaway fetch killed with its descendants and locks, the no-progress deadline, a helper holding the pipes, the reaper, and a forge end-to-end check of the filtered advertisement. --- crates/forge/tests/e2e_git.rs | 56 +++ crates/objio/src/git.rs | 891 ++++++++++++++++++++++++++++++++-- 2 files changed, 908 insertions(+), 39 deletions(-) diff --git a/crates/forge/tests/e2e_git.rs b/crates/forge/tests/e2e_git.rs index b2c2c2a7..e59a8e2d 100644 --- a/crates/forge/tests/e2e_git.rs +++ b/crates/forge/tests/e2e_git.rs @@ -1521,6 +1521,62 @@ fn refs_sc_create_cas_update_fetch_and_delete_round_trip_through_the_real_receiv /// One receive-pack request by hand (`old` and `new` are what the client believes), so the /// server's own compare-and-swap is exercised: the real client checks the advertisement first. +/// How `sc+git::` reads one mutable key (`GitBucket::get`, ADR-0060): a single +/// `git fetch` of exactly `refs/sc/` under protocol v2. The host must answer +/// the ref listing with that ref alone: the branch and the other keys never cross +/// the wire, which the old `ls-remote` plus `fetch` pair did not achieve. +#[test] +fn a_v2_fetch_of_one_sc_ref_gets_a_filtered_advertisement() { + let (_s, c, url, auth, store) = store_world("sc-filtered"); + seed_main(&c, &url, &auth); + let m = "refs/sc/manifest"; + let v1 = bucket_value(&c, &store, &["one\n"]); + let log = bucket_value(&c, &store, &["entry\n"]); + assert!(cas_push(&c, &store, &auth, &url, m, "", &v1).0); + let other = "refs/sc/log/00000000000000000001"; + assert!(cas_push(&c, &store, &auth, &url, other, "", &log).0); + + let fresh = c.dir.join("fresh"); + c.ok(&c.dir, None, &["init", "-q", fresh.to_str().unwrap()]); + let trace = c.dir.join("trace.log"); + let (ok, text) = c.git_env( + &fresh, + Some(&auth), + &[("GIT_TRACE_PACKET", trace.to_str().unwrap())], + &[ + "-c", + "protocol.version=2", + "-c", + "fetch.unpackLimit=1", + "fetch", + "--no-tags", + "--no-write-fetch-head", + "--depth=1", + "--end-of-options", + &url, + &format!("+{m}:{m}"), + // what `GitBucket` adds so abbreviations cannot stand in for the key + &format!("^refs/{m}"), + &format!("^refs/heads/{m}"), + ], + ); + assert!(ok, "fetch: {text}"); + assert_eq!(c.ok(&fresh, None, &["rev-parse", m]).trim(), v1); + + let text = std::fs::read_to_string(&trace).unwrap(); + assert!( + text.contains(&format!("ref-prefix {m}")), + "the client must ask for the ref by prefix: {text}" + ); + // The reply to the ls-refs request: every ` ` the host listed. + let listed: Vec<&str> = text + .lines() + .filter(|l| l.contains("fetch<") && l.contains(" refs/")) + .filter_map(|l| l.split_whitespace().last()) + .collect(); + assert_eq!(listed, [m], "the host must not list other refs: {text}"); +} + fn raw_update(s: &Srv, auth: &str, old: &str, new: &str, refname: &str) -> String { let pkt = |t: &str| format!("{:04x}{t}", t.len() + 4); let mut body = pkt(&format!( diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index 3b74d39f..f1e7013a 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -21,10 +21,11 @@ use crate::{validate_key, Bucket, Error, Fetched, ObjectMeta, Result}; use std::collections::HashMap; +use std::ffi::OsString; use std::io::{Read, Write}; -use std::path::PathBuf; -use std::process::{Command, Stdio}; -use std::sync::{Mutex, MutexGuard}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::{mpsc, Mutex, MutexGuard}; use std::time::{Duration, Instant, SystemTime}; /// Largest chunk blob a value is split into (GitHub's hard per-blob limit is @@ -55,10 +56,24 @@ const FETCH_BATCH: usize = 256; /// 32 KiB, so the budget stays well under it). const DELETE_BATCH: usize = 128; const DELETE_ARG_BUDGET: usize = 24 << 10; +/// How long a fetch may neither grow `objects/pack` nor exit before the +/// watchdog kills it (a served store depends on its git child, so a silent +/// host must not park a session forever). +const IDLE: Duration = Duration::from_secs(120); +/// Download budget of a fetch of a whole key family or a batch of keys. +const FAMILY_BUDGET: u64 = MAX_VALUE; /// `-c` overrides on every git call: transport allowlist, no hooks, no signing, -/// no background maintenance, bounded stalls. +/// no background maintenance, bounded stalls. `protocol.version=2` makes a +/// fetch send `ref-prefix`, so the host filters its ref advertisement down to +/// what is asked for; `fetch.unpackLimit=1` keeps every incoming object in one +/// `objects/pack/tmp_pack_*` file (never loose objects), which is what the +/// fetch watchdog measures; `fetch.uriProtocols=` stops a host from redirecting +/// the download to packfile URIs the watchdog could not see. const GIT_CONFIG: &[&str] = &[ + "protocol.version=2", + "fetch.unpackLimit=1", + "fetch.uriProtocols=", "protocol.allow=never", "protocol.file.allow=always", "protocol.ssh.allow=always", @@ -192,6 +207,34 @@ enum Pushed { Refused(String), } +/// Fetch watchdog limits: what one fetch may cost before it is killed. +#[derive(Clone, Copy)] +struct Tuning { + /// Replaces the computed download budget of every fetch (tests only). + budget: Option, + /// A fetch that neither grows `objects/pack` nor exits for this long is killed. + idle: Duration, + /// Fixed polling interval; `None` polls after 2 ms and backs off to 50 ms + /// (tests fix it to make a fetch finish between two polls). + poll: Option, +} + +impl Default for Tuning { + fn default() -> Self { + Tuning { + budget: None, + idle: IDLE, + poll: None, + } + } +} + +/// One watched fetch: its download budget and the name of the cap behind it. +struct Watch { + budget: u64, + what: String, +} + /// A bucket stored as `refs/sc/*` in a Git repository reachable at `url`. /// Intent: let a plain Git host carry a native sc store (P39) without a new /// server. See the module docs for the value encoding and write protocol. @@ -202,6 +245,13 @@ pub struct GitBucket { /// Chunk size used for writes (tests shrink it). chunk: usize, state: Mutex>, + tune: Tuning, + /// Serve mode: every fetch leads its own process group, so the watchdog + /// can kill `ssh` and `git-remote-*` along with git. Off for the CLI, where + /// the terminal's Ctrl-C must still reach git and ssh must still be able to + /// prompt on the controlling terminal. + #[cfg(unix)] + group: bool, } fn is_oid(s: &str) -> bool { @@ -290,6 +340,94 @@ fn check_len(key: &str, len: u64) -> Result<()> { Ok(()) } +/// Download budget of a fetch of exactly one key: its value cap plus the pack +/// overhead (0.1% and 1 MiB; deflate can grow incompressible data a little). +fn single_budget(key: &str) -> u64 { + let cap = value_cap(key); + cap + cap / 1024 + (1 << 20) +} + +/// Negative refspecs that make `+refs/sc/:refs/sc/` exact. For a +/// refspec without a glob git also accepts a remote ref that merely abbreviates +/// to the name (its rev-parse rules: `refs/heads/refs/sc/`, a tag, a +/// remote-tracking ref, ...), which would let a branch or tag stand in for an +/// absent key. `ls-remote` plus an exact-name compare never had that gap, so +/// the single-key fetch excludes every such spelling. +fn exact_only(full: &str) -> Vec { + ["refs/", "refs/tags/", "refs/heads/", "refs/remotes/"] + .iter() + .map(|p| format!("^{p}{full}")) + .chain([format!("^refs/remotes/{full}/HEAD")]) + .collect() +} + +/// Name and size of every regular file directly in `dir` (empty when it cannot +/// be read). +fn dir_files(dir: &Path) -> HashMap { + std::fs::read_dir(dir) + .into_iter() + .flatten() + .flatten() + .filter_map(|e| { + let m = e.metadata().ok()?; + m.is_file().then(|| (e.file_name(), m.len())) + }) + .collect() +} + +/// Total size of the regular files directly in `dir`. +fn dir_bytes(dir: &Path) -> u64 { + std::fs::read_dir(dir) + .into_iter() + .flatten() + .flatten() + .filter_map(|e| e.metadata().ok().filter(|m| m.is_file()).map(|m| m.len())) + .sum() +} + +/// Remove the `*.lock` files under `dir` (the cache root and its `refs/`, not +/// `objects/`): only a killed git can have left one, because nothing else uses +/// the cache while its fetch is being undone. +fn remove_stale_locks(dir: &Path, depth: u32) { + for e in std::fs::read_dir(dir).into_iter().flatten().flatten() { + let (path, kind) = (e.path(), e.file_type()); + match kind { + Ok(t) if t.is_dir() && depth < 8 && e.file_name() != "objects" => { + remove_stale_locks(&path, depth + 1); + } + Ok(t) if t.is_file() && path.extension().is_some_and(|x| x == "lock") => { + let _ = std::fs::remove_file(&path); + } + _ => {} + } + } +} + +/// Read up to `cap` bytes from `r` and discard the rest, so the writer never +/// blocks on a full pipe. The flag says more than `cap` bytes came. +fn drain(r: &mut impl Read, cap: usize) -> (Vec, bool) { + let mut buf = Vec::new(); + let _ = r.by_ref().take(cap as u64 + 1).read_to_end(&mut buf); + let over = buf.len() > cap; + buf.truncate(cap); + let _ = std::io::copy(r, &mut std::io::sink()); + (buf, over) +} + +fn over_budget(watch: &Watch) -> Error { + Error::Backend(format!( + "git fetch downloaded more than {} bytes ({}); it was stopped and its download \ + discarded (hostile or corrupt remote)", + watch.budget, watch.what + )) +} + +fn stalled(idle: Duration) -> Error { + Error::Backend(format!( + "git fetch made no progress for {idle:?}; it was stopped (unreachable or hostile remote)" + )) +} + /// How a refused write turned out, judged by re-reading the exact ref. #[derive(Debug, PartialEq, Eq)] enum Settled { @@ -426,14 +564,47 @@ impl GitBucket { Self::open_with(url, git, CHUNK) } + /// [`GitBucket::open`] for a long-running server (`sc serve --store`): the + /// cache directory is made under `parent` (the serve home's `.sc/tmp/`, created + /// if missing) instead of the system temp directory, and every fetch + /// leads its own process group so the watchdog, when it kills one, takes `ssh` or + /// `git-remote-https` with it. A killed server can leave the cache behind; + /// [`GitBucket::reap_stale`] removes such leftovers at the next start. + pub fn open_in(url: &str, parent: &Path) -> Result { + let git = std::env::var("SC_GIT").unwrap_or_else(|_| "git".to_string()); + Self::open_full(url, git, CHUNK, Some(parent), true) + } + fn open_with(url: &str, git: String, chunk: usize) -> Result { + Self::open_full(url, git, chunk, None, false) + } + + fn open_full( + url: &str, + git: String, + chunk: usize, + parent: Option<&Path>, + group: bool, + ) -> Result { Self::validate_url(url)?; + #[cfg(not(unix))] + let _ = group; // process groups are unix-only; the watchdog kills the child alone + let cache = match parent { + Some(p) => { + std::fs::create_dir_all(p)?; + new_cache_dir_in(p)? + } + None => new_cache_dir_in(&std::env::temp_dir())?, + }; let b = GitBucket { url: url.to_string(), git, - cache: new_cache_dir()?, + cache, chunk: chunk.max(1), state: Mutex::new(HashMap::new()), + tune: Tuning::default(), + #[cfg(unix)] + group, }; // An early `?` drops `b`, which removes the cache dir. The cache is // pinned to sha1 whatever the user's `init.defaultObjectFormat`: values @@ -458,14 +629,9 @@ impl GitBucket { self.state.lock().unwrap_or_else(|p| p.into_inner()) } - /// Run git in the cache repository with the hardened environment. - fn run( - &self, - args: &[&str], - stdin: Option<&[u8]>, - env: &[(&str, &str)], - max_out: usize, - ) -> Result { + /// The git command line in the cache repository with the hardened + /// environment, stdout and stderr piped. + fn command(&self, args: &[&str], env: &[(&str, &str)], piped_stdin: bool) -> Command { let mut cmd = Command::new(&self.git); for c in GIT_CONFIG { cmd.arg("-c").arg(c); @@ -482,19 +648,35 @@ impl GitBucket { for (k, v) in env { cmd.env(k, v); } - cmd.stdin(if stdin.is_some() { + cmd.stdin(if piped_stdin { Stdio::piped() } else { Stdio::null() }) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let mut child = cmd.spawn().map_err(|e| { + cmd + } + + fn spawn(&self, cmd: &mut Command) -> Result { + cmd.spawn().map_err(|e| { Error::Backend(format!( "cannot run git ({}): {e} (install git or set SC_GIT)", self.git )) - })?; + }) + } + + /// Run git in the cache repository with the hardened environment. + fn run( + &self, + args: &[&str], + stdin: Option<&[u8]>, + env: &[(&str, &str)], + max_out: usize, + ) -> Result { + let mut cmd = self.command(args, env, stdin.is_some()); + let mut child = self.spawn(&mut cmd)?; let sin = child.stdin.take(); let mut sout = child.stdout.take().expect("piped"); let mut serr = child.stderr.take().expect("piped"); @@ -548,6 +730,153 @@ impl GitBucket { } } + /// Run `git fetch` under the download watchdog. Instead of blocking on the + /// pipes like [`Self::run`], this polls the growth of `objects/pack` (every + /// download lands in one `tmp_pack_*` file, see [`GIT_CONFIG`]) while + /// detached threads drain the pipes, and kills git when the growth passes + /// `watch.budget` or when nothing grows and nothing exits for `idle`. A + /// check after a normal exit catches a fetch that finished between two + /// polls. A breach removes what the fetch added and returns an error + /// naming the cap. + /// + /// The bound is disk per fetch. It is not RAM inside `index-pack` and not + /// the total size of the cache, which grows by what each fetch keeps. + fn run_watched(&self, args: &[&str], max_out: usize, watch: &Watch) -> Result { + let pack_dir = self.cache.join("objects").join("pack"); + let before = dir_files(&pack_dir); + let base: u64 = before.values().sum(); + let mut cmd = self.command(args, &[], false); + #[cfg(unix)] + if self.group { + std::os::unix::process::CommandExt::process_group(&mut cmd, 0); + } + let mut child = self.spawn(&mut cmd)?; + let mut sout = child.stdout.take().expect("piped"); + let mut serr = child.stderr.take().expect("piped"); + // Detached readers: a descendant that outlives a killed git and keeps a + // pipe open must never be able to block this call. + let (otx, orx) = mpsc::channel(); + let (etx, erx) = mpsc::channel(); + std::thread::spawn(move || { + let _ = otx.send(drain(&mut sout, max_out)); + }); + std::thread::spawn(move || { + let _ = etx.send(drain(&mut serr, SMALL_CAP).0); + }); + + let (mut last, mut moved) = (0u64, Instant::now()); + let mut pause = self.tune.poll.unwrap_or(Duration::from_millis(2)); + let mut breach = None; + let status = loop { + match child.try_wait() { + Ok(Some(st)) => break Some(st), + Ok(None) => {} + Err(e) => { + self.stop(&mut child); + let _ = child.wait(); + return Err(e.into()); + } + } + let grown = dir_bytes(&pack_dir).saturating_sub(base); + if grown > watch.budget { + breach = Some(over_budget(watch)); + break None; + } + if grown != last { + (last, moved) = (grown, Instant::now()); + } else if moved.elapsed() >= self.tune.idle { + breach = Some(stalled(self.tune.idle)); + break None; + } + std::thread::sleep(pause); + if self.tune.poll.is_none() { + pause = (pause * 2).min(Duration::from_millis(50)); + } + }; + let Some(status) = status else { + // Signal before reaping, while the group id is still ours. + self.stop(&mut child); + let _ = child.wait(); + self.discard_download(&pack_dir, &before); + return Err(breach.expect("only a breach leaves the loop without a status")); + }; + if dir_bytes(&pack_dir).saturating_sub(base) > watch.budget { + self.discard_download(&pack_dir, &before); + return Err(over_budget(watch)); + } + let held = |what: &str| { + Error::Backend(format!( + "git exited but left its {what} open for {:?} (a helper outlived it)", + self.tune.idle + )) + }; + let (stdout, over) = orx + .recv_timeout(self.tune.idle) + .map_err(|_| held("stdout"))?; + let stderr = erx + .recv_timeout(self.tune.idle) + .map_err(|_| held("stderr"))?; + if over { + return Err(Error::Backend(format!( + "git output exceeded {max_out} bytes (hostile or corrupt remote)" + ))); + } + Ok(Out { + success: status.success(), + code: status.code(), + stdout, + stderr: self.scrub(&String::from_utf8_lossy(&stderr)), + }) + } + + /// Kill a watched git. Called before the child is reaped, so its group id + /// cannot have been reused. In serve mode the child leads its own process + /// group and the whole group is signalled, so `ssh` and `git-remote-*` + /// die with it; the system `kill` keeps this std-only, with no `unsafe`. + fn stop(&self, child: &mut Child) { + #[cfg(unix)] + if self.group { + let _ = Command::new("kill") + .args(["-s", "KILL", "--"]) + .arg(format!("-{}", child.id())) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + let _ = child.kill(); + } + + /// Undo a fetch that went over budget: remove every file it added to + /// `objects/pack` and every ref under `refs/sc/`. A fetch that finished + /// before the check had already pointed refs at what is deleted here; the + /// cache is only a copy of the host, so the next read fetches again. + fn discard_download(&self, pack_dir: &Path, before: &HashMap) { + // A killed git cannot release its locks (`shallow.lock`, ref locks), and + // a lock that stays would fail every later fetch into this cache. + remove_stale_locks(&self.cache, 0); + // Names first, while the objects still exist: a ref to a missing + // object is skipped by `for-each-ref`. + let refs: Vec = self + .local_refs(NS) + .map(|v| v.into_iter().map(|(k, _, _)| format!("{NS}{k}")).collect()) + .unwrap_or_default(); + for e in std::fs::read_dir(pack_dir).into_iter().flatten().flatten() { + if !before.contains_key(&e.file_name()) { + let _ = std::fs::remove_file(e.path()); + } + } + if !refs.is_empty() { + let cmds: String = refs.iter().map(|r| format!("delete {r}\n")).collect(); + let _ = self.run( + &["update-ref", "--stdin"], + Some(cmds.as_bytes()), + &[], + SMALL_CAP, + ); + } + } + /// Error text safe to print: URL userinfo redacted, control bytes (ESC /// from a hostile hook message) dropped, length bounded. fn scrub(&self, text: &str) -> String { @@ -635,7 +964,32 @@ impl GitBucket { /// downloaded. `family` = a glob fetch of a whole key family, which also /// prunes stale local refs. A shallow glob fetch that matches no ref exits 1 /// with no error text; [`Self::get_family`] tells that apart by listing. + /// The download budget is [`FAMILY_BUDGET`]; a fetch of exactly one key + /// goes through [`Self::fetch_one`] and gets that key's own, tighter one. fn fetch(&self, refspecs: &[String], family: bool) -> Result { + let what = "the 2 GiB budget of a family or batched fetch".to_string(); + self.fetch_watched(refspecs, family, FAMILY_BUDGET, what) + } + + /// Fetch exactly `refs/sc/` in one call, which both reads the ref and + /// updates the cache (a v2 host answers with just that ref, see + /// [`GIT_CONFIG`]). Exact by name: see [`exact_only`]. A failure does not + /// say whether the ref is absent; the caller asks `ls-remote` for that. + fn fetch_one(&self, key: &str) -> Result { + let full = format!("{NS}{key}"); + let mut specs = vec![format!("+{full}:{full}")]; + specs.extend(exact_only(&full)); + let what = format!("the value cap of {key:?}"); + self.fetch_watched(&specs, false, single_budget(key), what) + } + + fn fetch_watched( + &self, + refspecs: &[String], + family: bool, + budget: u64, + what: String, + ) -> Result { let mut a = vec![ "fetch", "--no-tags", @@ -648,7 +1002,11 @@ impl GitBucket { } a.extend(["--end-of-options", &self.url]); a.extend(refspecs.iter().map(String::as_str)); - self.run(&a, None, &[], SMALL_CAP) + let watch = Watch { + budget: self.tune.budget.unwrap_or(budget), + what, + }; + self.run_watched(&a, SMALL_CAP, &watch) } /// Push `oid` to `refs/sc/` (or delete it when `oid` is `None`) under @@ -971,15 +1329,74 @@ impl Drop for GitBucket { } } -/// Fresh private temp directory `sc-git-bucket--` (mode 0700). -fn new_cache_dir() -> Result { +/// Prefix of every cache directory name: `sc-git-bucket--`. +const CACHE_PREFIX: &str = "sc-git-bucket-"; + +impl GitBucket { + /// Remove cache directories a killed process left under `parent`, and return + /// how many were removed. Only a `sc-git-bucket--*` directory whose + /// owner process is provably gone is touched: several `sc serve --stdio` + /// processes can share one `parent`, and a live one's cache must survive. + /// A name that does not parse, this process's own directories, and a pid + /// that cannot be shown dead are all kept. Unix only; on other platforms it + /// removes nothing. + pub fn reap_stale(parent: &Path) -> usize { + #[cfg(unix)] + { + let mut removed = 0; + for e in std::fs::read_dir(parent).into_iter().flatten().flatten() { + let name = e.file_name().to_string_lossy().into_owned(); + let Some(rest) = name.strip_prefix(CACHE_PREFIX) else { + continue; + }; + let Some(pid) = rest.split('-').next().and_then(|p| p.parse::().ok()) else { + continue; + }; + // `file_type` does not follow a symlink, so only a real directory goes. + let is_dir = e.file_type().is_ok_and(|t| t.is_dir()); + if is_dir + && pid != std::process::id() + && pid_is_gone(pid) + && std::fs::remove_dir_all(e.path()).is_ok() + { + removed += 1; + } + } + removed + } + #[cfg(not(unix))] + { + let _ = parent; + 0 + } + } +} + +/// Is `pid` provably not running? `kill -0` says "No such process" for a pid +/// that is gone; any other answer (alive, not permitted, unknown wording) is +/// not proof, so the caller keeps what it owns. +#[cfg(unix)] +fn pid_is_gone(pid: u32) -> bool { + Command::new("kill") + .args(["-0", &pid.to_string()]) + .env("LC_ALL", "C") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .output() + .is_ok_and(|o| { + !o.status.success() && String::from_utf8_lossy(&o.stderr).contains("No such process") + }) +} + +/// Fresh private directory `sc-git-bucket--` (mode 0700) under `parent`. +fn new_cache_dir_in(parent: &Path) -> Result { for attempt in 0u64..16 { let name = format!( - "sc-git-bucket-{}-{}", + "{CACHE_PREFIX}{}-{}", std::process::id(), &nonce_seed(attempt)[..16] ); - let p = std::env::temp_dir().join(name); + let p = parent.join(name); #[cfg_attr(not(unix), allow(unused_mut))] // `mut` only for the unix mode let mut b = std::fs::DirBuilder::new(); #[cfg(unix)] @@ -1035,26 +1452,14 @@ impl Bucket for GitBucket { if let Some(fam) = family_of(key) { return self.get_family(&mut st, key, cached_tag, fam); } - let Some(remote) = self.remote_ref(key)? else { - st.remove(key); - return Ok(Fetched::Absent); - }; - if cached_tag == Some(remote.as_str()) { - // Refresh only a cache entry for this very value; an entry for an - // older value would otherwise look fresh. A cold hit stays uncached. - match st.get_mut(key) { - Some(k) if k.oid == remote => k.at = Instant::now(), - _ => { - st.remove(key); - } - } - return Ok(Fetched::Unchanged); - } + // One fetch both reads the ref and tells whether it changed: a v2 host + // filters its advertisement to this ref, where `ls-remote` followed by + // `fetch` cost two calls (and the whole advertisement on a v0 host). let full = format!("{NS}{key}"); - let out = self.fetch(&[format!("+{full}:{full}")], false)?; + let out = self.fetch_one(key)?; if !out.success { - // The ref may have vanished between ls-remote and fetch; only the - // exact full refname decides (patterns tail-match). + // The fetch fails for an absent ref and for a real error alike; + // only the exact full refname decides (patterns tail-match). return match self.remote_ref(key)? { None => { st.remove(key); @@ -1074,6 +1479,17 @@ impl Bucket for GitBucket { if ty != "commit" { return Err(corrupt("ref does not point at a commit")); } + if cached_tag == Some(oid.as_str()) { + // Refresh only a cache entry for this very value; an entry for an + // older value would otherwise look fresh. A cold hit stays uncached. + match st.get_mut(key) { + Some(k) if k.oid == oid => k.at = Instant::now(), + _ => { + st.remove(key); + } + } + return Ok(Fetched::Unchanged); + } self.finish_read(&mut st, key, &oid, cached_tag) } @@ -2374,4 +2790,401 @@ mod tests { } assert_eq!(leases, N, "one lease per ref"); } + + /// Shell snippet for a shim that logs the git subcommand of every call. + #[cfg(unix)] + fn count_snippet(log: &Path) -> String { + format!( + "s=; k=0; for x in \"$@\"; do if [ $k = 1 ]; then k=0; continue; fi; if [ \"$x\" = -c ]; then k=1; continue; fi; s=$x; break; done; echo \"$s\" >> {}", + log.display() + ) + } + + /// Network subcommands logged by [`count_snippet`], in order. + #[cfg(unix)] + fn network_calls(log: &Path) -> Vec { + std::fs::read_to_string(log) + .unwrap_or_default() + .lines() + .filter(|l| matches!(*l, "fetch" | "ls-remote" | "push")) + .map(String::from) + .collect() + } + + /// Wait for `pid` to be gone (a zombie awaiting its reaper counts as gone). + #[cfg(unix)] + fn wait_gone(pid: u32) { + for _ in 0..80 { + let out = Command::new("ps") + .args(["-o", "stat=", "-p", &pid.to_string()]) + .output() + .unwrap(); + let stat = String::from_utf8_lossy(&out.stdout).trim().to_string(); + if stat.is_empty() || stat.starts_with('Z') { + return; + } + std::thread::sleep(Duration::from_millis(50)); + } + panic!("process {pid} survived"); + } + + #[cfg(unix)] + fn kill_pid(pid: u32) { + let _ = Command::new("kill") + .args(["-s", "KILL", &pid.to_string()]) + .status(); + } + + /// The error text of a read that must fail. + fn get_err(b: &GitBucket, key: &str) -> String { + match b.get(key, None) { + Err(e) => e.to_string(), + Ok(_) => panic!("reading {key:?} must fail"), + } + } + + /// Plant a value commit holding one incompressible chunk of `n` bytes. + fn plant_big(hubdir: &Path, key: &str, n: usize) { + let mut x = 0x9E37_79B9_7F4A_7C15u64; + let data: Vec = (0..n) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + (x >> 24) as u8 + }) + .collect(); + let f = hubdir.join("big.bin"); + std::fs::write(&f, &data).unwrap(); + let blob = git(hubdir, &["hash-object", "-w", f.to_str().unwrap()], ""); + std::fs::remove_file(&f).unwrap(); + let tree = git(hubdir, &["mktree"], &format!("100644 blob {blob}\tc0000\n")); + let c = git(hubdir, &["commit-tree", &tree, "-m", "planted"], ""); + git(hubdir, &["update-ref", &format!("refs/sc/{key}"), &c], ""); + } + + /// A mutable-key read is one `git fetch`, and the host's reply lists that + /// ref alone: protocol v2 sends `ref-prefix`, so the branch and the other + /// keys on the host never cross the wire (they used to, in a separate + /// `ls-remote`). + #[cfg(unix)] + #[test] + fn a_manifest_read_is_one_fetch_with_a_filtered_advertisement() { + need_git!(); + let t = Tmp::new("filtered"); + let (hubdir, url) = hub(&t); + let w = open(&url); + let m = w.put_if_tag("manifest", b"v1", None).unwrap().unwrap(); + w.put_new("log/1", b"entry").unwrap(); + w.put_new("other", b"o").unwrap(); + let oid = git(&hubdir, &["rev-parse", "refs/sc/other"], ""); + git(&hubdir, &["update-ref", "refs/heads/main", &oid], ""); + + let (calls, trace) = (t.0.join("calls.log"), t.0.join("trace.log")); + // GIT_TRACE_PACKET is stripped from the environment `run` passes, so + // the shim sets it itself. + let sh = shim( + &t, + "trace.sh", + &format!( + "{}\nexport GIT_TRACE_PACKET={}", + count_snippet(&calls), + trace.display() + ), + ); + let r = open_git(&url, &sh); + let Fetched::New { bytes, tag } = r.get("manifest", None).unwrap() else { + panic!("cold read") + }; + assert_eq!((bytes.as_slice(), tag.as_str()), (&b"v1"[..], m.as_str())); + assert_eq!(network_calls(&calls), ["fetch"], "one call, no ls-remote"); + + let text = std::fs::read_to_string(&trace).unwrap(); + assert!( + text.contains("ref-prefix refs/sc/manifest"), + "the fetch must ask for the ref by prefix" + ); + let advertised: Vec<&str> = text + .lines() + .filter(|l| l.contains("upload-pack>") && l.contains(" refs/")) + .filter_map(|l| l.split_whitespace().last()) + .collect(); + assert_eq!( + advertised, + ["refs/sc/manifest"], + "reply must be filtered: {text}" + ); + + // A poll after the freshness window is again one call, and unchanged. + std::thread::sleep(FRESH + Duration::from_millis(200)); + assert!(matches!( + r.get("manifest", Some(&m)).unwrap(), + Fetched::Unchanged + )); + assert_eq!(network_calls(&calls), ["fetch", "fetch"]); + } + + /// Git resolves a refspec source that is absent by its abbreviation rules, + /// so a branch, tag or remote-tracking ref spelled `refs/sc/` after its + /// own prefix would stand in for the key. The exact-name reads of v1 never + /// allowed that; the single-key fetch still must not. + #[test] + fn abbreviated_ref_names_never_stand_in_for_an_absent_key() { + need_git!(); + let t = Tmp::new("abbrev"); + let (hubdir, url) = hub(&t); + plant(&hubdir, "decoy-src", false, &[("100644", "blob", "c0000")]); + let c = git(&hubdir, &["rev-parse", "refs/sc/decoy-src"], ""); + let decoys = [ + "refs/refs/sc/manifest", + "refs/tags/refs/sc/manifest", + "refs/heads/refs/sc/manifest", + "refs/remotes/refs/sc/manifest", + "refs/remotes/refs/sc/manifest/HEAD", + ]; + for decoy in decoys { + git(&hubdir, &["update-ref", decoy, &c], ""); + let r = open(&url); + assert!( + matches!(r.get("manifest", None).unwrap(), Fetched::Absent), + "{decoy} stood in for the manifest" + ); + assert!(!r.exists("manifest").unwrap()); + git(&hubdir, &["update-ref", "-d", decoy], ""); + } + // With every decoy present, the real key is the one read. (A push to a + // name some other ref abbreviates to is ambiguous, so the key goes in first.) + assert!(open(&url) + .put_if_tag("manifest", b"real", None) + .unwrap() + .is_some()); + for decoy in &decoys[..4] { + git(&hubdir, &["update-ref", decoy, &c], ""); + } + let Fetched::New { bytes, .. } = open(&url).get("manifest", None).unwrap() else { + panic!() + }; + assert_eq!(bytes, b"real"); + } + + /// A value over its budget is discarded whether the watchdog catches it in + /// flight or only after git exited between two polls (`poll` fixed longer + /// than the whole fetch), in serve mode (the group is killed) and in CLI + /// mode (git alone), and the same instance keeps reading good keys. A git + /// killed in flight cannot release its `shallow.lock`; the next read must + /// not trip on it. After a CLI-mode kill a surviving `index-pack` may still + /// finish its own file, so only the other cases assert an empty pack dir. + #[test] + fn an_oversized_value_is_discarded_and_the_instance_keeps_working() { + need_git!(); + let t = Tmp::new("oversize"); + let (hubdir, url) = hub(&t); + assert!(open(&url).put_new("good", b"fine").unwrap()); + plant_big(&hubdir, "manifest", 3 << 20); + let late = Some(Duration::from_millis(400)); + for (group, poll) in [(true, None), (true, late), (false, None), (false, late)] { + let mut b = GitBucket::open_full(&url, "git".into(), CHUNK, None, group).unwrap(); + b.tune.budget = Some(256 << 10); + b.tune.poll = poll; + let started = Instant::now(); + let e = get_err(&b, "manifest"); + assert!( + e.contains("262144 bytes") && e.contains("manifest"), + "the error must name the cap: {e}" + ); + if poll.is_some() { + assert!( + started.elapsed() >= Duration::from_millis(400), + "final check path" + ); + } + if group || poll.is_some() { + assert!( + dir_files(&b.cache.join("objects").join("pack")).is_empty(), + "the download must be removed" + ); + } + assert!(b.local_refs(NS).unwrap().is_empty(), "no ref may survive"); + let Fetched::New { bytes, .. } = b.get("good", None).unwrap() else { + panic!("the instance must still read other keys") + }; + assert_eq!(bytes, b"fine"); + // the hostile key is refused again, not cached as absent or known + assert!(b.get("manifest", None).is_err()); + } + } + + /// The fetch is killed in flight, with every descendant, and leaves nothing: + /// the shim stands in for a git that keeps growing `tmp_pack_*` and has a + /// helper process (as `ssh` or `git-remote-https` would). + #[cfg(unix)] + #[test] + fn a_runaway_fetch_is_killed_with_its_descendants() { + need_git!(); + let t = Tmp::new("runaway"); + let (_, url) = hub(&t); + let pids = t.0.join("pids"); + // Like a git killed mid-fetch, it holds locks it can never release. + let body = format!( + "case \" $* \" in *\" fetch \"*) sleep 600 & echo $! > {}; touch \"$GIT_DIR/shallow.lock\"; mkdir -p \"$GIT_DIR/refs/sc\"; touch \"$GIT_DIR/refs/sc/manifest.lock\"; while :; do head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/tmp_pack_x\"; sleep 0.01; done;; esac", + pids.display() + ); + let sh = shim(&t, "runaway.sh", &body); + let parent = t.0.join("scratch"); + let mut b = GitBucket::open_full(&url, sh, CHUNK, Some(&parent), true).unwrap(); + assert!(b.cache.starts_with(&parent)); + b.tune.budget = Some(1 << 20); + let started = Instant::now(); + let e = get_err(&b, "manifest"); + assert!(e.contains("1048576 bytes"), "{e}"); + assert!(started.elapsed() < Duration::from_secs(20)); + let helper: u32 = std::fs::read_to_string(&pids) + .unwrap() + .trim() + .parse() + .unwrap(); + wait_gone(helper); + let tmp_pack = b.cache.join("objects").join("pack").join("tmp_pack_x"); + assert!(!tmp_pack.exists(), "partial download must be removed"); + for lock in ["shallow.lock", "refs/sc/manifest.lock"] { + assert!(!b.cache.join(lock).exists(), "{lock} must be removed"); + } + // The loop that wrote it died too: nothing comes back. + std::thread::sleep(Duration::from_millis(300)); + assert!(!tmp_pack.exists(), "the fetch kept running"); + } + + /// No growth and no exit for `idle` is a stuck host. Both modes return + /// promptly; in serve mode the group dies with it. + #[cfg(unix)] + #[test] + fn a_silent_fetch_hits_the_no_progress_deadline() { + need_git!(); + for group in [false, true] { + let t = Tmp::new("silent"); + let (_, url) = hub(&t); + let pids = t.0.join("pids"); + let body = format!( + "case \" $* \" in *\" fetch \"*) sleep 600 & echo $! > {}; wait;; esac", + pids.display() + ); + let sh = shim(&t, "silent.sh", &body); + let mut b = GitBucket::open_full(&url, sh, CHUNK, None, group).unwrap(); + b.tune.idle = Duration::from_millis(800); + let started = Instant::now(); + let e = get_err(&b, "manifest"); + let took = started.elapsed(); + let helper: u32 = std::fs::read_to_string(&pids) + .unwrap() + .trim() + .parse() + .unwrap(); + assert!(e.contains("no progress"), "{e}"); + assert!(took >= Duration::from_millis(800), "{took:?}"); + assert!( + took < Duration::from_secs(10), + "a descendant holding the pipes must not park the call: {took:?}" + ); + if group { + wait_gone(helper); + } else { + // Only the child was killed: the orphan still holds the pipes, + // which is exactly what must not block the caller. + kill_pid(helper); + wait_gone(helper); + } + } + } + + /// A git that exits cleanly while a helper keeps its pipes open is an error + /// within `idle`, not a hang. + #[cfg(unix)] + #[test] + fn a_helper_holding_the_pipes_after_exit_is_an_error() { + need_git!(); + let t = Tmp::new("held"); + let (_, url) = hub(&t); + let pids = t.0.join("pids"); + let body = format!( + "case \" $* \" in *\" fetch \"*) sleep 600 & echo $! > {}; exit 0;; esac", + pids.display() + ); + let sh = shim(&t, "held.sh", &body); + let mut b = open_git(&url, &sh); + b.tune.idle = Duration::from_millis(800); + let started = Instant::now(); + let e = get_err(&b, "manifest"); + assert!(e.contains("left its"), "{e}"); + assert!(started.elapsed() < Duration::from_secs(10)); + let helper: u32 = std::fs::read_to_string(&pids) + .unwrap() + .trim() + .parse() + .unwrap(); + kill_pid(helper); + wait_gone(helper); + } + + /// Serve mode (own cache parent, own process group) is the same store. + #[cfg(unix)] + #[test] + fn a_served_bucket_passes_the_contract_and_cleans_up() { + need_git!(); + let t = Tmp::new("served"); + let (_, url) = hub(&t); + let parent = t.0.join("home").join(".sc").join("tmp"); + let b = GitBucket::open_in(&url, &parent).unwrap(); + let cache = b.cache.clone(); + assert_eq!(cache.parent(), Some(parent.as_path())); + contract_suite(&b); + drop(b); + assert!(!cache.exists()); + assert_eq!(std::fs::read_dir(&parent).unwrap().count(), 0); + } + + /// The startup sweep removes only what a dead process left. A live foreign + /// process (another `sc serve --stdio` on the same home), this process, an + /// unparsable name and unrelated entries stay. + #[cfg(unix)] + #[test] + fn reap_stale_removes_only_caches_of_dead_processes() { + need_git!(); + let t = Tmp::new("reap"); + let (_, url) = hub(&t); + let parent = t.0.join("tmp"); + std::fs::create_dir_all(&parent).unwrap(); + let mk = |name: &str| { + let p = parent.join(name); + std::fs::create_dir_all(p.join("objects")).unwrap(); + p + }; + let mut gone = Command::new("true").spawn().unwrap(); + let dead = gone.id(); + gone.wait().unwrap(); + let mut live = Command::new("sleep").arg("30").spawn().unwrap(); + let stale = mk(&format!("{CACHE_PREFIX}{dead}-aaaa")); + let foreign = mk(&format!("{CACHE_PREFIX}{}-bbbb", live.id())); + let junk = mk(&format!("{CACHE_PREFIX}notapid-cccc")); + let other = mk("serve-bucket-1-0"); + // two live buckets of this process, and a symlink named like a stale cache + let (a, b) = ( + GitBucket::open_in(&url, &parent).unwrap(), + GitBucket::open_in(&url, &parent).unwrap(), + ); + let link = parent.join(format!("{CACHE_PREFIX}{dead}-link")); + std::os::unix::fs::symlink(&other, &link).unwrap(); + + assert_eq!(GitBucket::reap_stale(&parent), 1); + assert!(!stale.exists()); + for kept in [&foreign, &junk, &other, &link, &a.cache, &b.cache] { + assert!(kept.exists(), "{kept:?} must survive"); + } + assert!(a.put_new("k", b"v").unwrap() && b.exists("k").unwrap()); + live.kill().unwrap(); + live.wait().unwrap(); + // once its owner is dead the foreign cache goes too + assert_eq!(GitBucket::reap_stale(&parent), 1); + assert!(!foreign.exists()); + assert_eq!(GitBucket::reap_stale(&parent.join("missing")), 0); + } } From 722d78b86a5bcd708c6fc6c212c7caa1791d6d88 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 14:57:50 +0200 Subject: [PATCH 2/9] feat(serve): serve --store sc+git:: behind a server-side private-branch gate `sc serve --store sc+git::` now works over --stdio and --http. The daemon needs git to authenticate without prompting. The private-branch opt-in moves to the operator: `--allow-private-branches` (refused for any other store). Without it the server refuses, as InvalidArgument with no wire change, every object of a sealed-branch kind (Sealed, Manifest): - PutPack stops at the first such object before upload_pack, in any record order, so a mixed pack adds no refs/sc/*. - PutObject refuses before staging, so a later UpdateRef cannot flush it. - UpdateRef refuses a manifest tip even when its objects are already on the chain, so an existing private branch cannot be advanced or re-pointed. The flag lives on BucketTransport (set by open_for_serve) and is plumbed through WirePolicy and ServeLimits; false means gated. The serve cache lives under the serve home's .sc/tmp/ and `sc serve` reaps dead processes' caches at startup. Tests drive the hostile orders against a real bare repository and inspect it afterwards (no refs, no objects, no sealed object in any stored pack), over pipes and through a real HTTP listener with real clients, and through the binary. Disabling each of the three checks in turn fails a test. --- crates/cli/src/main.rs | 48 +- crates/cli/tests/git_bucket.rs | 179 ++++++- crates/repo/src/bucket_transport.rs | 91 +++- .../src/bucket_transport/git_remote_tests.rs | 36 +- .../src/bucket_transport/git_serve_tests.rs | 443 ++++++++++++++++++ crates/repo/src/http_transport.rs | 7 + crates/repo/src/lib.rs | 4 +- crates/repo/src/transport.rs | 9 +- crates/repo/src/wire.rs | 40 +- 9 files changed, 778 insertions(+), 79 deletions(-) create mode 100644 crates/repo/src/bucket_transport/git_serve_tests.rs diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 48a36efe..73ad7d03 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -413,13 +413,24 @@ enum Cmd { /// --tls-cert (enforced in run_serve, not by clap). #[arg(long)] tls_key: Option, - /// Serve a bucket WAL remote (`sc+wal://…` or `sc+s3://…`; `sc+git::` - /// is not supported yet) instead of this repo's own object store (P36c). `` remains the serve - /// home: its `.sc/` still provides access tokens, the TLS identity, - /// and scratch space — but all served content lives in the bucket, - /// making this instance disposable. + /// Serve a bucket WAL remote (`sc+wal://…`, `sc+s3://…` or + /// `sc+git::`) instead of this repo's own object store (P36c). + /// `` remains the serve home: its `.sc/` still provides access + /// tokens, the TLS identity, and scratch space — but all served content + /// lives in the bucket, making this instance disposable. A `sc+git::` + /// store needs git to authenticate without prompting (ssh agent, a key + /// without a passphrase, or a credential helper): a daemon cannot answer + /// a prompt. #[arg(long)] store: Option, + /// Accept private branches into a `sc+git::` store (`--store` only). + /// Without it the server refuses every private-branch object and ref + /// before anything reaches the Git host: the host shows the branch + /// name, recipient ids and sizes to everyone who can read the + /// repository and keeps the ciphertext after any later revoke. The + /// opt-in is the operator's, never a client's. (ADR-0060) + #[arg(long)] + allow_private_branches: bool, /// Repo root to serve (the directory containing `.sc/`). Required /// unless a `token`/`fingerprint` subcommand is used. path: Option, @@ -1278,6 +1289,7 @@ fn real_main() -> Result<()> { tls_cert, tls_key, store, + allow_private_branches, path, } => match sub { Some(ServeSub::Token { op }) => run_serve_token(op), @@ -1308,6 +1320,7 @@ fn real_main() -> Result<()> { tls_cert, tls_key, store, + allow_private_branches, path, ) } @@ -4245,7 +4258,10 @@ fn runs_default_budget(conn_bytes_given: bool, max_pack_size: Option) -> bo /// (P36c) redirects served content to a bucket WAL remote; `path` remains /// the serve home (tokens/TLS identity/scratch) either way. A malformed /// `--store` URL is validated up front, before any bind — same fail-fast -/// idiom as `run_remote`'s `BucketUrl::parse` check. +/// idiom as `run_remote`'s `BucketUrl::parse` check. A `sc+git::` store +/// (ADR-0060) refuses private-branch objects unless `--allow-private-branches` +/// is given, which only that store accepts, and reaps Git caches a killed +/// earlier server left in the serve home. fn run_serve( stdio: bool, http: Option, @@ -4259,6 +4275,7 @@ fn run_serve( tls_cert: Option, tls_key: Option, store: Option, + allow_private_branches: bool, path: PathBuf, ) -> Result<()> { let max_pack = max_pack_size.unwrap_or(scl_repo::wire::DEFAULT_MAX_PACK_SIZE); @@ -4271,9 +4288,24 @@ fn run_serve( }; scl_repo::wire::validate_max_conn_bytes(conn_bytes, max_pack, scl_core::max_object_size())?; if let Some(url) = &store { - scl_repo::refuse_git_store(url)?; scl_repo::BucketUrl::parse(url)?; // fail fast on malformed URLs, before any bind } + let git_store = store.as_deref().is_some_and(scl_repo::is_git_bucket_url); + if allow_private_branches && !git_store { + anyhow::bail!("--allow-private-branches applies only to --store sc+git::"); + } + if git_store { + // A killed earlier server can have left its Git cache in the serve home. + scl_repo::wire::reap_stale_serve_scratch(&path); + // Once per daemon start; a `--stdio` session would repeat it to every ssh client. + if !allow_private_branches && http.is_some() { + eprintln!( + "note: serving a Git repository; private-branch objects are refused \ + (--allow-private-branches accepts them). git must authenticate without \ + prompting (ssh agent, a key without a passphrase, or a credential helper)" + ); + } + } match (stdio, http) { (true, None) => { if read_only || allow_public { @@ -4313,6 +4345,7 @@ fn run_serve( max_pack_size: max_pack, ro_drain_cap: scl_repo::wire::RO_DRAIN_CAP, max_conn_bytes: conn_bytes, + allow_private_branches, }; let mut stdin = std::io::stdin().lock(); let mut stdout = std::io::stdout().lock(); @@ -4340,6 +4373,7 @@ fn run_serve( limits.max_pack_size = m; } limits.max_conn_bytes = conn_bytes; + limits.allow_private_branches = allow_private_branches; if default_budget && !read_only && !scl_repo::http_transport::addr_is_loopback(&addr) { eprintln!( "warning: writable listener on {addr} runs with the default --max-conn-bytes \ diff --git a/crates/cli/tests/git_bucket.rs b/crates/cli/tests/git_bucket.rs index 1c3c656e..9f3a1ab2 100644 --- a/crates/cli/tests/git_bucket.rs +++ b/crates/cli/tests/git_bucket.rs @@ -141,28 +141,181 @@ fn sc_git_is_never_a_git_bridge_remote_and_the_opt_in_is_sc_git_only() { std::fs::remove_dir_all(&a).unwrap(); } -/// Deferred in v1: `sc serve --store` refuses an `sc+git::` store with a clear -/// message, before any I/O. +/// `sc serve --allow-private-branches` is the operator's opt-in for a `sc+git::` +/// store and means nothing for any other, so it is refused there instead of +/// being ignored. #[test] -fn serve_store_refuses_sc_git() { - let a = tmp("deferred"); - init_repo(&a); - let home = tmp("deferred-home"); +fn serve_allow_private_branches_needs_a_git_store() { + let home = tmp("serve-flag"); init_repo(&home); - let e = err_text(&sc( + let h = home.to_str().unwrap(); + for store in [&["--store", "sc+wal:///nonexistent/bucket"][..], &[][..]] { + let mut args = vec!["serve", "--stdio", "--allow-private-branches"]; + args.extend_from_slice(store); + args.push(h); + let e = err_text(&sc(&home, &args)); + assert!(e.contains("applies only to --store sc+git::"), "{e}"); + } + std::fs::remove_dir_all(&home).unwrap(); +} + +/// Spawn `sc serve --http 127.0.0.1:0 --store `; returns +/// the child and its announced `host:port` (as `http_remote.rs` does). +fn spawn_serve(home: &Path, store: &str, extra: &[&str]) -> (std::process::Child, String) { + use std::io::BufRead; + let mut child = Command::new(env!("CARGO_BIN_EXE_sc")) + .args(["serve", "--http", "127.0.0.1:0", "--store", store]) + .args(extra) + .arg(home) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn sc serve"); + let mut line = String::new(); + std::io::BufReader::new(child.stdout.take().unwrap()) + .read_line(&mut line) + .expect("read the announce line"); + let addr = line + .trim() + .strip_prefix("listening on ") + .unwrap_or_else(|| panic!("unexpected serve startup line: {line:?}")) + .to_string(); + (child, addr) +} + +/// A client returns before the server has dropped its session, and a killed +/// server leaves its cache behind, so wait (bounded) for `dir` to empty before +/// the server is stopped. +fn wait_until_empty(dir: &Path) { + for _ in 0..100 { + let left = std::fs::read_dir(dir).map_or(0, Iterator::count); + if left == 0 { + return; + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + panic!("a session left something in {dir:?}"); +} + +/// A bare Git repository, `sc serve --http --store sc+git::` in front, and +/// real clients: a public push and a clone work, a private branch is refused +/// before anything reaches the repository, and only a server restarted with +/// `--allow-private-branches` takes it. +#[test] +fn serve_store_sc_git_gates_private_branches_until_the_operator_opts_in() { + let root = tmp("serve-git"); + let keys = tmp("serve-git-keys"); + let hub = root.join("hub.git"); + assert!(Command::new("git") + .args(["init", "--bare", "-q", hub.to_str().unwrap()]) + .status() + .unwrap() + .success()); + let store = format!("sc+git::file://{}", hub.display()); + let home = root.join("home"); + init_repo(&home); + let (id, pk) = keygen(&keys, "alice"); + let idp = id.to_str().unwrap(); + let a = root.join("a"); + init_repo(&a); + std::fs::write( + a.join(".sc/recipients.toml"), + format!("[recipients]\nalice = \"{pk}\"\n"), + ) + .unwrap(); + std::fs::write(a.join("README.md"), "public\n").unwrap(); + ok(sc(&a, &["commit", "-m", "base"])); + + let (mut gated, addr) = spawn_serve(&home, &store, &[]); + let origin = format!("sc+http://{addr}/x"); + ok(sc(&a, &["remote", "add", "origin", &origin])); + ok(sc(&a, &["push", "origin"])); // public content needs no opt-in + let public_refs = refs_on_carrier(&hub); + assert!(public_refs.contains("refs/sc/manifest"), "{public_refs}"); + + ok(sc( &a, &[ + "branch", + "hotfix", + "--private", + "--to", + "alice", + "--identity", + idp, + ], + )); + ok(sc(&a, &["switch", "hotfix", "--identity", idp])); + std::fs::write(a.join("embargo.txt"), "embargoed\n").unwrap(); + ok(sc(&a, &["commit", "-m", "embargoed", "--identity", idp])); + let e = err_text(&sc(&a, &["push", "origin"])); + assert!(e.contains("--allow-private-branches"), "{e}"); + assert!(e.contains("Nothing was uploaded"), "{e}"); + assert_eq!( + refs_on_carrier(&hub), + public_refs, + "refused: host unchanged" + ); + wait_until_empty(&home.join(".sc").join("tmp")); + gated.kill().unwrap(); + gated.wait().unwrap(); + + let (mut open, addr) = spawn_serve(&home, &store, &["--allow-private-branches"]); + let opted = format!("sc+http://{addr}/x"); + ok(sc(&a, &["remote", "add", "opted", &opted])); + ok(sc(&a, &["push", "opted"])); + assert_ne!(refs_on_carrier(&hub), public_refs); + let keyless = root.join("keyless"); + ok(sc(&root, &["clone", &opted, keyless.to_str().unwrap()])); + let listing = ok(sc(&keyless, &["branch", "list"])); + assert!(listing.contains("hotfix (private, no access)"), "{listing}"); + // No Git cache outlives its sessions in the serve home. + wait_until_empty(&home.join(".sc").join("tmp")); + open.kill().unwrap(); + open.wait().unwrap(); + for d in [&root, &keys] { + std::fs::remove_dir_all(d).unwrap(); + } +} + +/// A starting `sc serve --store sc+git::` removes the Git caches a killed +/// earlier server left in the serve home, and only those whose owner is gone: +/// another server on the same home may be live. Unix only, like the sweep. +#[cfg(unix)] +#[test] +fn serve_start_reaps_caches_of_dead_servers_only() { + let root = tmp("serve-reap"); + let home = root.join("home"); + init_repo(&home); + let scratch = home.join(".sc").join("tmp"); + std::fs::create_dir_all(&scratch).unwrap(); + let mut gone = Command::new("true").spawn().unwrap(); + let dead = gone.id(); + gone.wait().unwrap(); + let mut live = Command::new("sleep").arg("60").spawn().unwrap(); + let stale = scratch.join(format!("sc-git-bucket-{dead}-aaaa")); + let foreign = scratch.join(format!("sc-git-bucket-{}-bbbb", live.id())); + for d in [&stale, &foreign] { + std::fs::create_dir_all(d.join("objects")).unwrap(); + } + // No stdin: the session ends at once, after the startup sweep. + let out = Command::new(env!("CARGO_BIN_EXE_sc")) + .args([ "serve", "--stdio", "--store", "sc+git::file:///nonexistent/carrier.git", home.to_str().unwrap(), - ], - )); - assert!(e.contains("does not support sc+git::"), "{e}"); - for d in [&a, &home] { - std::fs::remove_dir_all(d).unwrap(); - } + ]) + .stdin(std::process::Stdio::null()) + .output() + .unwrap(); + let _ = out; + live.kill().unwrap(); + live.wait().unwrap(); + assert!(!stale.exists(), "the dead server's cache must be reaped"); + assert!(foreign.exists(), "a live server's cache must stay"); + std::fs::remove_dir_all(&root).unwrap(); } /// The private-branch gate through the binary, with a config written by hand diff --git a/crates/repo/src/bucket_transport.rs b/crates/repo/src/bucket_transport.rs index db9a22b7..d11b0511 100644 --- a/crates/repo/src/bucket_transport.rs +++ b/crates/repo/src/bucket_transport.rs @@ -17,6 +17,8 @@ use std::io::Write; mod compact; #[cfg(test)] mod git_remote_tests; +#[cfg(test)] +mod git_serve_tests; pub use compact::{CompactOptions, CompactReport}; /// How many times a read restarts from a fresh manifest after an object the @@ -102,6 +104,12 @@ pub struct BucketTransport { /// age, so compaction keeps its own record of when it first saw each object /// (`compact/sightings`) and deletes with leases (ADR-0057 addendum). git_carrier: bool, + /// Set by [`BucketTransport::open_for_serve`] on a `sc+git::` store whose + /// operator did not pass `--allow-private-branches`: objects of the + /// sealed-branch kinds (`transport::is_sealed_branch_object`) are refused at + /// ingest and a ref may not point at a manifest, so no private-branch + /// ciphertext reaches the Git host through this server (ADR-0060). + refuse_sealed: bool, } /// A leaf object a pack holds that no tree or snapshot references, so the @@ -229,20 +237,6 @@ pub fn check_private_push_allowed(url: &str, allowed: bool, branch: &str) -> Res ))) } -/// `sc serve --store sc+git::…` is refused (deferred): a daemon cannot answer -/// ssh or credential prompts, and it would sit between clients and the -/// client-side private-branch gate. Checked on the prefix, before any parse. -pub fn refuse_git_store(url: &str) -> Result<()> { - if is_git_bucket_url(url) { - return Err(Error::InvalidArgument( - "sc serve --store does not support sc+git:: remotes yet (deferred); \ - serve an sc+wal:// or sc+s3:// store, or push to the Git repository with `sc push`" - .into(), - )); - } - Ok(()) -} - /// A parsed `sc+wal://`, `sc+s3:///` or /// `sc+git::` remote URL. /// `sc+wal` treats everything after the scheme as a directory path (so @@ -334,6 +328,42 @@ impl BucketTransport { BucketTransport::build(bucket, parsed.scheme == BucketScheme::Git) } + /// Open the store behind `sc serve --store` (ADR-0060). For `sc+wal://` and + /// `sc+s3://` this is [`BucketTransport::open`]. For `sc+git::` the Git + /// cache lives under `scratch` (the serve home's `.sc/tmp/`), every fetch + /// call runs in its own process group so the fetch watchdog can take a + /// stuck `ssh` or `git-remote-https` down with it, and private-branch + /// objects are refused unless `allow_private_branches` is set: a Git host + /// shows the branch name, recipient ids and sizes to every reader and keeps + /// the ciphertext, so the opt-in belongs to the operator, not to a client. + pub fn open_for_serve( + url: &str, + scratch: &std::path::Path, + allow_private_branches: bool, + ) -> Result { + let parsed = BucketUrl::parse(url)?; + if parsed.scheme != BucketScheme::Git { + return Self::open(url); + } + let bucket = Box::new(scl_objio::GitBucket::open_in(&parsed.bucket, scratch)?); + let mut t = Self::build(bucket, true)?; + t.refuse_sealed = !allow_private_branches; + Ok(t) + } + + /// The refusal for private-branch objects on a gated `sc+git::` store. The + /// text names the server-side flag; it never shows the store URL. + fn sealed_refusal() -> Error { + Error::InvalidArgument( + "this server stores into a Git repository (sc+git::) and was not started with \ + --allow-private-branches, so it refuses private-branch objects: a Git host shows \ + the branch name, recipient ids and sizes to everyone who can read the repository \ + and keeps the ciphertext after any later revoke. Nothing was uploaded. The \ + operator can restart `sc serve` with --allow-private-branches" + .into(), + ) + } + /// Open a transport directly over an already-constructed bucket backend. /// Fails only if the initial `refresh` (a conditional GET of the /// manifest plus a walk of its parent chain) errors — an entirely empty @@ -360,6 +390,7 @@ impl BucketTransport { saw_absent: Cell::new(false), attempt_tag: RefCell::new(None), git_carrier, + refuse_sealed: false, }; t.refresh()?; Ok(t) @@ -951,6 +982,10 @@ impl Transport for BucketTransport { if obj.id() != *id { return Err(Error::CorruptObject(*id)); // decodes, but not canonically } + // Before staging: a refused object is never held, flushed or uploaded. + if self.refuse_sealed && crate::transport::is_sealed_branch_object(&obj) { + return Err(Self::sealed_refusal()); + } self.pending_objects.borrow_mut().note(*id, obj); self.staged.borrow_mut().push((*id, bytes.to_vec())); Ok(()) @@ -965,7 +1000,17 @@ impl Transport for BucketTransport { // pack still content-addresses identically for identical input. let mut objects: Vec<(ObjectId, Vec)> = Vec::new(); let mut received = PendingObjects::default(); - scl_core::pack::parse_pack_reader(src, |id, obj| { + let mut sealed = false; + let parsed = scl_core::pack::parse_pack_reader(src, |id, obj| { + // Stop at the first sealed-branch object on a gated store, before + // anything is uploaded (the callback cannot return our own error + // type, so the flag says why the parse was aborted). + if self.refuse_sealed && crate::transport::is_sealed_branch_object(&obj) { + sealed = true; + return Err(scl_core::Error::PackCorrupt( + "private-branch object refused".into(), + )); + } // The upload below re-encodes each object; bytes that decode but // are not canonical would be stored under an id they do not hash // to, and every later read would fail verification. @@ -978,7 +1023,11 @@ impl Transport for BucketTransport { received.note(id, obj); objects.push((id, bytes)); Ok(()) - })?; + }); + if sealed { + return Err(Self::sealed_refusal()); + } + parsed?; let (hash, ids) = self.upload_pack(&objects)?; self.pending_packs.borrow_mut().push(hash); // Only after the upload: a failed one must not leave ids that no pack holds. @@ -1066,6 +1115,16 @@ impl Transport for BucketTransport { pending: &pending_objects, }; let trusted = crate::ref_guard::check_ref_move(&mut objects, &tips, current, *id)?; + // A manifest tip is a private branch: refused on a gated store + // even when its objects are already on the chain (put by a direct + // writer), so an existing private branch cannot be advanced or + // re-pointed through this server without the opt-in. + if self.refuse_sealed { + use crate::ref_guard::RefObjects as _; + if let Some(crate::ref_guard::Node::Manifest(_)) = objects.node(id)? { + return Err(Self::sealed_refusal()); + } + } checked_against = Some((current, trusted)); } // Claim a log slot for this attempt. `put_new` on `log/` is diff --git a/crates/repo/src/bucket_transport/git_remote_tests.rs b/crates/repo/src/bucket_transport/git_remote_tests.rs index b0feeb0a..5fff837c 100644 --- a/crates/repo/src/bucket_transport/git_remote_tests.rs +++ b/crates/repo/src/bucket_transport/git_remote_tests.rs @@ -1,6 +1,6 @@ //! `sc+git::` remote plumbing: URL parsing and the reserved prefix, the -//! private-branch gate, and the `serve --store` refusal. -//! Nothing here reaches `GitBucket`'s real I/O except the URL-parse test. +//! private-branch gate. The serve-side gate has its own tests in +//! `git_serve_tests.rs`. Nothing here reaches `GitBucket`'s real I/O except the URL-parse test. use super::*; use crate::error::Error; @@ -162,38 +162,6 @@ fn push_of_a_private_branch_to_a_git_carrier_refuses_before_any_io() { std::fs::remove_dir_all(&d).unwrap(); } -#[test] -fn serve_store_refuses_git_carriers_before_parsing() { - let url = "sc+git::file:///srv/r.git"; - assert!(invalid(refuse_git_store(url)).contains("sc serve --store")); - // The reserved prefix is refused too, so a near miss never reaches a parser. - assert!(refuse_git_store("sc+git:/x").is_err()); - assert!(refuse_git_store("sc+wal:///tmp/b").is_ok()); - - // The wire layer enforces it for both `--stdio` and `--http` serving, and - // the client sees why at the handshake. - let home = root("serve-store"); - std::fs::create_dir_all(home.join(".sc")).unwrap(); - let (client_read, mut server_write) = std::io::pipe().unwrap(); - let (mut server_read, client_write) = std::io::pipe().unwrap(); - let home_for_server = home.clone(); - let srv = std::thread::spawn(move || { - crate::wire::serve_bucket_with_policy( - url, - &home_for_server, - &mut server_read, - &mut server_write, - crate::wire::WirePolicy::default(), - ) - }); - let Err(err) = crate::stdio_transport::WireClient::handshake(client_read, client_write) else { - panic!("the session is refused"); - }; - assert!(err.to_string().contains("sc serve --store"), "{err}"); - srv.join().unwrap().unwrap(); - std::fs::remove_dir_all(&home).unwrap(); -} - #[test] fn sc_git_urls_parse_with_the_inner_url_verbatim() { for inner in [ diff --git a/crates/repo/src/bucket_transport/git_serve_tests.rs b/crates/repo/src/bucket_transport/git_serve_tests.rs new file mode 100644 index 00000000..01e4e7fb --- /dev/null +++ b/crates/repo/src/bucket_transport/git_serve_tests.rs @@ -0,0 +1,443 @@ +//! `sc serve --store sc+git::` (ADR-0060). The private-branch gate is a security +//! boundary: whatever order of `PutObject`, `PutPack` and `UpdateRef` a client +//! sends, a refused request must leave no `refs/sc/*` and no sealed object on +//! the Git host. Every test drives a real bare repository and inspects it. + +use crate::error::Error; +use crate::repo::Repo; +use crate::stdio_transport::WireClient; +use crate::transport::Transport; +use crate::wire::{serve_bucket_with_policy, WirePolicy}; +use scl_core::{Object, ObjectId}; +use scl_objio::{Bucket, Fetched}; +use std::io::Cursor; +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn have_git() -> bool { + Command::new("git") + .arg("--version") + .output() + .is_ok_and(|o| o.status.success()) +} + +macro_rules! need_git { + () => { + if !have_git() { + eprintln!("git not available; skipping"); + return; + } + }; +} + +fn root(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("scl-bt-gitserve-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +fn git(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_NOSYSTEM", "1") + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +/// A bare Git repository standing in for the host. +struct Hub { + dir: PathBuf, + bare: PathBuf, + /// The `sc+git::` store URL. + url: String, +} + +impl Hub { + fn new(tag: &str) -> Hub { + let dir = root(&format!("{tag}-hub")); + let bare = dir.join("hub.git"); + git(&dir, &["init", "--bare", "-q", bare.to_str().unwrap()]); + let url = format!("sc+git::file://{}", bare.display()); + Hub { dir, bare, url } + } + + /// Every `refs/sc/*` ref on the host. + fn refs(&self) -> String { + git(&self.bare, &["for-each-ref", "refs/sc/"]) + } + + /// Every object on the host (not only the reachable ones). + fn objects(&self) -> String { + git( + &self.bare, + &["cat-file", "--batch-all-objects", "--batch-check"], + ) + } + + /// No pack stored on the host holds an object of a sealed-branch kind. + fn assert_no_sealed(&self) { + let b = scl_objio::GitBucket::open(&format!("file://{}", self.bare.display())).unwrap(); + for key in b + .list("packs/") + .unwrap() + .into_iter() + .filter(|k| k.ends_with(".pack")) + { + let Fetched::New { bytes, .. } = b.get(&key, None).unwrap() else { + panic!("listed pack {key} is absent") + }; + for (_, obj) in scl_core::pack::parse_pack(&bytes).unwrap() { + assert!( + !matches!(obj, Object::Sealed(_) | Object::Manifest(_)), + "a sealed object reached the Git host in {key}" + ); + } + } + } +} + +impl Drop for Hub { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +/// An initialized directory used only as the serve home. +fn serve_home(tag: &str) -> PathBuf { + let d = root(&format!("{tag}-home")); + drop(Repo::init(&d).unwrap()); + d +} + +fn encoded(repo: &Repo, tip: ObjectId) -> Vec<(ObjectId, Vec)> { + let store_arc = repo.vfs().store(); + let mut store = store_arc.lock().unwrap(); + crate::reachable::reachable_objects(&mut *store, &[tip]) + .unwrap() + .into_iter() + .map(|id| (id, store.get(&id).unwrap().encode())) + .collect() +} + +fn is_sealed_kind(bytes: &[u8]) -> bool { + matches!( + Object::decode(bytes), + Ok(Object::Sealed(_) | Object::Manifest(_)) + ) +} + +fn pack_of(objects: &[(ObjectId, Vec)]) -> Cursor> { + Cursor::new(scl_core::pack::build_pack(objects).unwrap().0) +} + +/// A repository with a public `main` and a private branch `emb`. +struct World { + dir: PathBuf, + repo: Repo, + sk: scl_crypto::SecretKey, + main: ObjectId, + /// Everything reachable from `main`. + public: Vec<(ObjectId, Vec)>, + /// The private branch's manifest tip. + emb: ObjectId, + /// What a push of `emb` sends beyond `public`, sealed-kind objects included. + private: Vec<(ObjectId, Vec)>, +} + +impl World { + fn new(tag: &str) -> World { + let dir = root(&format!("{tag}-world")); + let repo = Repo::init(&dir).unwrap(); + std::fs::write(dir.join("f.txt"), "public\n").unwrap(); + let main = repo.commit("t", "base").unwrap(); + let (sk, pk) = scl_crypto::generate_keypair(); + repo.branch_private("emb", &sk, std::slice::from_ref(&pk), &[]) + .unwrap(); + repo.switch_with_identity("emb", Some(&sk)).unwrap(); + let mut w = World { + public: encoded(&repo, main), + main, + emb: main, + private: Vec::new(), + dir, + repo, + sk, + }; + w.commit_private("embargo.txt", "embargoed\n"); + w + } + + /// Commit one more file on `emb`; sets `emb` and `private` to the new tip's + /// objects that `main` does not already carry. + fn commit_private(&mut self, file: &str, text: &str) { + std::fs::write(self.dir.join(file), text).unwrap(); + self.repo + .commit_private("t", "embargoed", Some(&self.sk)) + .unwrap(); + let (_, tip, _) = self.repo.head_private().unwrap().unwrap(); + self.emb = tip; + let known: std::collections::BTreeSet = + self.public.iter().map(|(id, _)| *id).collect(); + self.private = encoded(&self.repo, tip) + .into_iter() + .filter(|(id, _)| !known.contains(id)) + .collect(); + assert!( + self.private.iter().any(|(_, b)| is_sealed_kind(b)), + "premise: the private branch has sealed objects" + ); + } + + fn everything(&self) -> Vec<(ObjectId, Vec)> { + let mut all = self.public.clone(); + all.extend(self.private.iter().cloned()); + all + } +} + +impl Drop for World { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +type Client = WireClient; +type Server = std::thread::JoinHandle>; + +/// One `--stdio`-style session over pipes, served from `home` into `store`. +fn session(store: &str, home: &Path, policy: WirePolicy) -> (Client, Server) { + let (client_read, mut server_write) = std::io::pipe().unwrap(); + let (mut server_read, client_write) = std::io::pipe().unwrap(); + let (store, home) = (store.to_string(), home.to_path_buf()); + let srv = std::thread::spawn(move || { + serve_bucket_with_policy(&store, &home, &mut server_read, &mut server_write, policy) + }); + ( + WireClient::handshake(client_read, client_write).unwrap(), + srv, + ) +} + +fn finish(client: Client, srv: Server) { + client.bye().unwrap(); + drop(client); + srv.join().unwrap().unwrap(); +} + +/// The refusal names the server-side flag. +fn assert_refused(e: &Error) { + let m = e.to_string(); + assert!(m.contains("--allow-private-branches"), "{m}"); + assert!(m.contains("Nothing was uploaded"), "{m}"); +} + +fn allowing() -> WirePolicy { + WirePolicy { + allow_private_branches: true, + ..Default::default() + } +} + +/// Every order a hostile client can try: a pack mixing public and sealed +/// objects, loose sealed objects, a ref at a manifest, then a clean public +/// push in the same session. Nothing sealed may reach the host, and the +/// session keeps serving. +#[test] +fn a_gated_session_never_lets_a_sealed_object_reach_the_host() { + need_git!(); + let w = World::new("gate"); + let hub = Hub::new("gate"); + let home = serve_home("gate"); + let (client, srv) = session(&hub.url, &home, WirePolicy::default()); + + // A mixed pack: the public part comes first, so a check that only looked at + // the first record would pass it. + assert_refused(&client.put_pack(&mut pack_of(&w.everything())).unwrap_err()); + assert_eq!(hub.refs(), "", "a refused pack adds no ref"); + assert_eq!(hub.objects(), "", "a refused pack adds no object"); + + // Loose objects: refused before they are staged. + for (id, bytes) in w.private.iter().filter(|(_, b)| is_sealed_kind(b)) { + assert_refused(&client.put_object(id, bytes).unwrap_err()); + } + // A ref at a manifest that was never ingested fails too. + assert!(client.update_ref("emb", &w.emb, None).is_err()); + assert_eq!(hub.refs(), "", "refused objects and refs add nothing"); + + // The same session still takes a public push, and only that reaches the host. + client.put_pack(&mut pack_of(&w.public)).unwrap(); + client.update_ref("main", &w.main, None).unwrap(); + assert!(client.update_ref("emb", &w.emb, None).is_err()); + finish(client, srv); + + let t = crate::bucket_transport::BucketTransport::open(&hub.url).unwrap(); + assert_eq!(t.list_refs().unwrap(), vec![("main".to_string(), w.main)]); + hub.assert_no_sealed(); + drop(t); + let tmp = home.join(".sc").join("tmp"); + assert!( + !tmp.exists() || std::fs::read_dir(&tmp).unwrap().next().is_none(), + "the Git cache and the spool must be gone with the session" + ); + let _ = std::fs::remove_dir_all(&home); +} + +/// A private branch that already sits on the host cannot be advanced, or +/// another ref pointed at its manifest, through a gated server; with the +/// operator's flag the same requests work. +#[test] +fn an_existing_private_branch_cannot_be_advanced_or_repointed_without_the_flag() { + need_git!(); + let mut w = World::new("advance"); + let hub = Hub::new("advance"); + // Put both branches on the host directly, as a writer with push access would. + { + let t = crate::bucket_transport::BucketTransport::open(&hub.url).unwrap(); + t.put_pack(&mut pack_of(&w.everything())).unwrap(); + t.update_ref("main", &w.main, None).unwrap(); + t.update_ref("emb", &w.emb, None).unwrap(); + } + let (old_emb, old_private) = (w.emb, w.private.clone()); + w.commit_private("more.txt", "more embargoed\n"); + let (new_emb, new_private) = (w.emb, w.private.clone()); + assert_ne!(old_emb, new_emb); + let before = hub.refs(); + + let home = serve_home("advance"); + let (client, srv) = session(&hub.url, &home, WirePolicy::default()); + // The advance sends new sealed objects and a new manifest. + assert_refused(&client.put_pack(&mut pack_of(&new_private)).unwrap_err()); + assert!(client.update_ref("emb", &new_emb, Some(&old_emb)).is_err()); + // Pointing a ref at a manifest already on the host needs no new object, so + // only the ref check can stop it. + assert_refused(&client.update_ref("emb-copy", &old_emb, None).unwrap_err()); + assert_eq!(hub.refs(), before, "nothing was written"); + finish(client, srv); + + // With the opt-in the very same requests succeed. + let (client, srv) = session(&hub.url, &home, allowing()); + client.put_pack(&mut pack_of(&new_private)).unwrap(); + client.update_ref("emb", &new_emb, Some(&old_emb)).unwrap(); + client.update_ref("emb-copy", &old_emb, None).unwrap(); + finish(client, srv); + let t = crate::bucket_transport::BucketTransport::open(&hub.url).unwrap(); + let refs: std::collections::BTreeMap<_, _> = t.list_refs().unwrap().into_iter().collect(); + assert_eq!(refs.get("emb"), Some(&new_emb)); + assert_eq!(refs.get("emb-copy"), Some(&old_emb)); + drop((t, old_private)); + let _ = std::fs::remove_dir_all(&home); +} + +/// The flag only matters for a `sc+git::` store: the other stores take private +/// branches as before, whatever the policy says. +#[test] +fn the_gate_applies_to_git_stores_only() { + let w = World::new("wal"); + let dir = root("wal-store"); + let store = format!("sc+wal://{}", dir.join("bucket").display()); + let home = serve_home("wal"); + let (client, srv) = session(&store, &home, WirePolicy::default()); + client.put_pack(&mut pack_of(&w.everything())).unwrap(); + client.update_ref("emb", &w.emb, None).unwrap(); + finish(client, srv); + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_dir_all(&home); +} + +/// A listener on 127.0.0.1 serving `store` from `home`; returns its port. +fn spawn_http(home: PathBuf, store: String, allow_private_branches: bool) -> u16 { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + std::thread::spawn(move || { + crate::http_transport::serve_http_listener( + listener, + &home, + false, + false, + crate::http_transport::ServeLimits { + allow_private_branches, + ..Default::default() + }, + None, + Some(store), + ) + .unwrap(); + }); + port +} + +/// The same boundary through `sc serve --http` with a real client: a private +/// branch pushed through a gated server is refused and leaves the host as it +/// was; through a server started with the flag it lands, and advancing it is +/// refused again on the gated one. +#[test] +fn http_push_of_a_private_branch_needs_the_servers_opt_in() { + need_git!(); + let hub = Hub::new("http"); + let (home_a, home_b) = (serve_home("http-a"), serve_home("http-b")); + let port_a = spawn_http(home_a.clone(), hub.url.clone(), false); + let port_b = spawn_http(home_b.clone(), hub.url.clone(), true); + + let mut w = World::new("http"); + // `World::new` leaves `emb` checked out; a public push needs `main`. + w.repo.switch_with_identity("main", Some(&w.sk)).unwrap(); + w.repo + .remote_add("a", &format!("sc+http://127.0.0.1:{port_a}/x")) + .unwrap(); + w.repo + .remote_add("b", &format!("sc+http://127.0.0.1:{port_b}/x")) + .unwrap(); + w.repo.push("a").unwrap(); // public content needs no opt-in + let public_only = hub.refs(); + assert!(!public_only.is_empty()); + + w.repo.switch_with_identity("emb", Some(&w.sk)).unwrap(); + assert_refused(&w.repo.push("a").unwrap_err()); + assert_eq!( + hub.refs(), + public_only, + "the refused push left the host as it was" + ); + hub.assert_no_sealed(); + + w.repo.push("b").unwrap(); // the operator of b opted in + let with_private = hub.refs(); + assert_ne!(with_private, public_only); + + w.commit_private("later.txt", "later\n"); + assert_refused(&w.repo.push("a").unwrap_err()); + assert_eq!( + hub.refs(), + with_private, + "an existing private branch is not advanced" + ); + w.repo.push("b").unwrap(); + assert_ne!(hub.refs(), with_private); + + let t = crate::bucket_transport::BucketTransport::open(&hub.url).unwrap(); + let refs: std::collections::BTreeMap<_, _> = t.list_refs().unwrap().into_iter().collect(); + assert_eq!(refs.get("emb"), Some(&w.emb)); + drop(t); + // A client returns before the server thread has dropped its session, so + // wait (bounded) for the caches to go; they must not outlive the sessions. + for h in [&home_a, &home_b] { + let tmp = h.join(".sc").join("tmp"); + let empty = || !tmp.exists() || std::fs::read_dir(&tmp).unwrap().next().is_none(); + for _ in 0..100 { + if empty() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + assert!(empty(), "a Git cache outlived its session in {h:?}"); + } +} diff --git a/crates/repo/src/http_transport.rs b/crates/repo/src/http_transport.rs index 14be1d12..bbe79477 100644 --- a/crates/repo/src/http_transport.rs +++ b/crates/repo/src/http_transport.rs @@ -852,6 +852,11 @@ pub struct ServeLimits { /// The CLI refuses `0`. The default is /// [`crate::wire::default_max_conn_bytes`] of the default pack cap. pub max_conn_bytes: u64, + /// Forwarded to [`crate::wire::WirePolicy::allow_private_branches`]: not a + /// bound, but the one served-store policy the HTTP path carries beside the + /// limits. Only a `sc+git::` store looks at it; `false` refuses + /// private-branch objects (ADR-0060). + pub allow_private_branches: bool, } impl Default for ServeLimits { fn default() -> Self { @@ -860,6 +865,7 @@ impl Default for ServeLimits { timeout_secs: 300, max_pack_size: crate::wire::DEFAULT_MAX_PACK_SIZE, max_conn_bytes: crate::wire::default_max_conn_bytes(crate::wire::DEFAULT_MAX_PACK_SIZE), + allow_private_branches: false, } } } @@ -1405,6 +1411,7 @@ fn handle_http_connection( max_pack_size: limits.max_pack_size, ro_drain_cap: crate::wire::RO_DRAIN_CAP, max_conn_bytes: limits.max_conn_bytes, + allow_private_branches: limits.allow_private_branches, }; match store { Some(url) => { diff --git a/crates/repo/src/lib.rs b/crates/repo/src/lib.rs index 0c2506cd..1c74cc75 100644 --- a/crates/repo/src/lib.rs +++ b/crates/repo/src/lib.rs @@ -80,8 +80,8 @@ mod ws_refresh; pub use scl_objio::redact; pub use bucket_transport::{ - check_private_push_allowed, is_bucket_url, is_git_bucket_url, refuse_git_store, BucketScheme, - BucketTransport, BucketUrl, CompactOptions, CompactReport, + check_private_push_allowed, is_bucket_url, is_git_bucket_url, BucketScheme, BucketTransport, + BucketUrl, CompactOptions, CompactReport, }; pub use conflicts::{ActiveOp, ConflictKind, ConflictVersions, ResolveSide, Side}; pub use error::{Error, Result}; diff --git a/crates/repo/src/transport.rs b/crates/repo/src/transport.rs index f271535c..e8671a26 100644 --- a/crates/repo/src/transport.rs +++ b/crates/repo/src/transport.rs @@ -131,13 +131,20 @@ fn check_servable_size(id: &ObjectId, len: u64, max: usize) -> Result<()> { Ok(()) } +/// The object kinds only a private branch produces: sealed members and branch +/// manifests (P34). `sc serve --store sc+git::` refuses them at ingest unless the +/// operator opted in (ADR-0060), so this is the one place that names the kinds. +pub(crate) fn is_sealed_branch_object(obj: &Object) -> bool { + matches!(obj, Object::Sealed(_) | Object::Manifest(_)) +} + /// Object kinds a by-id read never serves (ADR-0047): sealed private-branch /// members and branch manifests. Filtered transfers already exclude them /// (`filter_manifest_wants`); this keeps the by-id verb consistent with that /// rule. It is a consistency rule, not a confidentiality boundary: sealed /// objects are ciphertext and an unfiltered pack still serves them. fn refused_by_id(obj: &Object) -> bool { - matches!(obj, Object::Sealed(_) | Object::Manifest(_)) + is_sealed_branch_object(obj) } /// What the unlocked half of a local `update_ref` leaned on, for the locked diff --git a/crates/repo/src/wire.rs b/crates/repo/src/wire.rs index bf51ad8c..9d3f10d5 100644 --- a/crates/repo/src/wire.rs +++ b/crates/repo/src/wire.rs @@ -838,6 +838,12 @@ pub struct WirePolicy { /// means unlimited (embedders and tests; `sc serve` refuses it). Reads /// are not budgeted. pub max_conn_bytes: u64, + /// Consulted only for a `sc+git::` store behind `sc serve --store` + /// (ADR-0060): accept private-branch objects and refs. `false` refuses + /// them (fail closed), because a Git host exposes the branch name, + /// recipient ids and sizes to every reader of the repository. Every other + /// store and a local repo ignore it. + pub allow_private_branches: bool, } impl Default for WirePolicy { @@ -847,6 +853,7 @@ impl Default for WirePolicy { max_pack_size: DEFAULT_MAX_PACK_SIZE, ro_drain_cap: RO_DRAIN_CAP, max_conn_bytes: default_max_conn_bytes(DEFAULT_MAX_PACK_SIZE), + allow_private_branches: false, } } } @@ -1117,15 +1124,33 @@ pub fn serve_with_policy( /// [`serve_bucket_with_policy`] only needs one match to decide whether to /// reply OK or a typed error. `home` is the serve session's HOME directory /// (see [`TempServeDir::create_in`]) — the spool dir is created under its -/// `.sc/tmp/`, not the process-wide temp dir. -fn open_bucket_serve_transport(store_url: &str, home: &std::path::Path) -> Result { - crate::bucket_transport::refuse_git_store(store_url)?; - let transport = BucketTransport::open(store_url)?; +/// `.sc/tmp/`, not the process-wide temp dir. A `sc+git::` store keeps its Git +/// cache under the same `.sc/tmp/` and refuses private-branch objects unless +/// `policy.allow_private_branches` is set. +fn open_bucket_serve_transport( + store_url: &str, + home: &std::path::Path, + policy: &WirePolicy, +) -> Result { + let transport = BucketTransport::open_for_serve( + store_url, + &crate::layout::Layout::at(home).tmp_dir(), + policy.allow_private_branches, + )?; let tmp = TempServeDir::create_in(home)?; Ok(ServeTransport::Bucket { transport, tmp }) } -/// Serve a bucket WAL (`sc+wal://`/`sc+s3://`; `sc+git::` is refused) over the same wire protocol +/// Remove Git cache directories a killed `sc serve --store sc+git::…` process +/// left under the serve home's `.sc/tmp/`, and return how many were removed. Call +/// once at startup. Only a directory whose owner process is provably gone is +/// removed: many `sc serve --stdio` processes can share one home, and a live +/// session's cache must survive. Unix only; elsewhere it removes nothing. +pub fn reap_stale_serve_scratch(home: &std::path::Path) -> usize { + scl_objio::GitBucket::reap_stale(&crate::layout::Layout::at(home).tmp_dir()) +} + +/// Serve a bucket WAL (`sc+wal://`, `sc+s3://` or `sc+git::`) over the same wire protocol /// `serve_with_policy` speaks for a local repo (P36c): same handshake, same /// `PROTOCOL_VERSION`, same read-only gate and pack-spool caps — every verb /// except the pack pair goes through [`Transport`] identically either way. @@ -1154,7 +1179,7 @@ pub fn serve_bucket_with_policy( if !handshake_hello(r, w)? { return Ok(()); } - let transport = match open_bucket_serve_transport(store_url, home) { + let transport = match open_bucket_serve_transport(store_url, home, &policy) { Ok(t) => { write_ok(w, &u32_body(PROTOCOL_VERSION))?; t @@ -2413,6 +2438,7 @@ mod tests { max_pack_size: 4096, ro_drain_cap: RO_DRAIN_CAP, max_conn_bytes: 0, + allow_private_branches: false, }; serve_with_policy(&root, &mut reader, &mut output, policy).unwrap(); @@ -2499,6 +2525,7 @@ mod tests { max_pack_size: 0, ro_drain_cap: 4096, max_conn_bytes: 0, + allow_private_branches: false, }; serve_with_policy(&root, &mut reader, &mut output, policy).unwrap(); @@ -2587,6 +2614,7 @@ mod tests { max_pack_size, ro_drain_cap: 4096, max_conn_bytes: 0, + allow_private_branches: false, }; serve_with_policy(&root, &mut std::io::Cursor::new(input), &mut output, policy) .unwrap(); From 98bfc1b8d75944cac2be704382e13c1988e7b8d1 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 15:40:43 +0200 Subject: [PATCH 3/9] test(objio): a discarded hostile value with a parent leaves the cache usable A depth-1 fetch records a value commit that has a parent as a shallow boundary, and a discarded download deletes that object. The new test checks that the same instance still reads a good key and refuses the hostile one again, in serve and CLI mode and on both check paths. The timing upper bounds of the watchdog tests move to 90 s (they only guard against a hang of 600 s) and wait_gone waits up to 20 s, so a loaded machine does not fail them. --- crates/objio/src/git.rs | 52 ++++++++++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index f1e7013a..b265b211 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -2814,7 +2814,7 @@ mod tests { /// Wait for `pid` to be gone (a zombie awaiting its reaper counts as gone). #[cfg(unix)] fn wait_gone(pid: u32) { - for _ in 0..80 { + for _ in 0..400 { let out = Command::new("ps") .args(["-o", "stat=", "-p", &pid.to_string()]) .output() @@ -2843,8 +2843,10 @@ mod tests { } } - /// Plant a value commit holding one incompressible chunk of `n` bytes. - fn plant_big(hubdir: &Path, key: &str, n: usize) { + /// Plant a value commit holding one incompressible chunk of `n` bytes, with + /// a parent commit when `parent` (a hostile shape: a shallow fetch records + /// it as a shallow boundary). + fn plant_big(hubdir: &Path, key: &str, n: usize, parent: bool) { let mut x = 0x9E37_79B9_7F4A_7C15u64; let data: Vec = (0..n) .map(|_| { @@ -2859,10 +2861,44 @@ mod tests { let blob = git(hubdir, &["hash-object", "-w", f.to_str().unwrap()], ""); std::fs::remove_file(&f).unwrap(); let tree = git(hubdir, &["mktree"], &format!("100644 blob {blob}\tc0000\n")); - let c = git(hubdir, &["commit-tree", &tree, "-m", "planted"], ""); + let mut args = vec!["commit-tree", &tree, "-m", "planted"]; + let base; + if parent { + let empty = git(hubdir, &["mktree"], ""); + base = git(hubdir, &["commit-tree", &empty, "-m", "base"], ""); + args.extend(["-p", &base]); + } + let c = git(hubdir, &args, ""); git(hubdir, &["update-ref", &format!("refs/sc/{key}"), &c], ""); } + /// A hostile value with a parent is recorded in `shallow` by the depth-1 + /// fetch, and a discarded download deletes that very object. The cache must + /// not be left pointing at it: the next reads, of the same hostile key and + /// of a good one, still work, in both modes and on both check paths. + #[test] + fn a_discarded_value_with_a_parent_leaves_the_cache_usable() { + need_git!(); + let t = Tmp::new("shallow"); + let (hubdir, url) = hub(&t); + assert!(open(&url).put_new("good", b"fine").unwrap()); + plant_big(&hubdir, "manifest", 3 << 20, true); + let late = Some(Duration::from_millis(400)); + for (group, poll) in [(true, None), (true, late), (false, late)] { + let mut b = GitBucket::open_full(&url, "git".into(), CHUNK, None, group).unwrap(); + b.tune.budget = Some(256 << 10); + b.tune.poll = poll; + let e = get_err(&b, "manifest"); + assert!(e.contains("262144 bytes"), "{e}"); + let Fetched::New { bytes, .. } = b.get("good", None).unwrap() else { + panic!("the cache must still read a good key") + }; + assert_eq!(bytes, b"fine"); + let again = get_err(&b, "manifest"); + assert!(again.contains("262144 bytes"), "{again}"); + } + } + /// A mutable-key read is one `git fetch`, and the host's reply lists that /// ref alone: protocol v2 sends `ref-prefix`, so the branch and the other /// keys on the host never cross the wire (they used to, in a separate @@ -2980,7 +3016,7 @@ mod tests { let t = Tmp::new("oversize"); let (hubdir, url) = hub(&t); assert!(open(&url).put_new("good", b"fine").unwrap()); - plant_big(&hubdir, "manifest", 3 << 20); + plant_big(&hubdir, "manifest", 3 << 20, false); let late = Some(Duration::from_millis(400)); for (group, poll) in [(true, None), (true, late), (false, None), (false, late)] { let mut b = GitBucket::open_full(&url, "git".into(), CHUNK, None, group).unwrap(); @@ -3037,7 +3073,7 @@ mod tests { let started = Instant::now(); let e = get_err(&b, "manifest"); assert!(e.contains("1048576 bytes"), "{e}"); - assert!(started.elapsed() < Duration::from_secs(20)); + assert!(started.elapsed() < Duration::from_secs(90)); let helper: u32 = std::fs::read_to_string(&pids) .unwrap() .trim() @@ -3082,7 +3118,7 @@ mod tests { assert!(e.contains("no progress"), "{e}"); assert!(took >= Duration::from_millis(800), "{took:?}"); assert!( - took < Duration::from_secs(10), + took < Duration::from_secs(90), "a descendant holding the pipes must not park the call: {took:?}" ); if group { @@ -3115,7 +3151,7 @@ mod tests { let started = Instant::now(); let e = get_err(&b, "manifest"); assert!(e.contains("left its"), "{e}"); - assert!(started.elapsed() < Duration::from_secs(10)); + assert!(started.elapsed() < Duration::from_secs(90)); let helper: u32 = std::fs::read_to_string(&pids) .unwrap() .trim() From 0a8d075b15d6b7e1882bd74c69a48d97da72dfdc Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 16:20:41 +0200 Subject: [PATCH 4/9] fix(objio): reap serve caches by file lock, and harden the fetch watchdog Review fixes for serve --store sc+git::. Reaper liveness is now an advisory file lock, not a pid. Each bucket opened by open_in holds File::lock on /.owner.lock for its life; reap_stale removes a sc-git-bucket-* directory only when it can take that lock (never a symlink, a directory without a lock file, or another name). That drops the dependency on the `kill` binary and the own-pid exemption, and survives pid reuse, pid-1 containers and several pid namespaces on one host. The sweep holds the lock while it removes the directory, and an opener whose lock file vanished starts over. One host only; unreliable over NFS, as for DirBucket. Watchdog: - If the `kill` program for a process group cannot be run, serve mode falls back to killing git alone and warns once on stderr. - After a clean exit with a helper holding the pipes, both pipe receives share one deadline (not twice the idle time) and serve mode stops the group before returning the error. - transfer.bundleURI=false is pinned beside fetch.uriProtocols=. - The family budget error says a legitimate store over 2 GiB needs compaction. The serve gate's refusal now says "No private-branch object was uploaded": public objects of the same push may already be on the host as an orphan pack. Docs: ADR-0060, THREAT-MODEL, CLAUDE.md, ARCHITECTURE.md and the skill state the lock semantics, the `kill` dependency, that only serve mode on unix is disk-bounded against a hostile HTTP host, that refs written by a family fetch are not measured, the 2 GiB family cliff, the unwatched ls-remote and push (ServerAliveInterval over ssh), that a fetch group outlives a killed sc serve, that an ssh user who runs sc serve is an operator, the per-connection cold cache cost times --max-connections, and the 2 ms to 50 ms poll back-off. forge e2e: the new test no longer splits raw_update from its doc comment and fetches with all five negative refspecs. Tests (each fails without its fix): reap_stale_goes_by_the_owner_lock_not_by_pid, a_sweep_never_removes_a_live_bucket_while_others_open (a smoke test of the create/sweep race), the_pipe_wait_after_exit_has_one_deadline, a_helper_holding_the_pipes_after_exit_is_an_error (group variant), a_missing_kill_program_falls_back_to_killing_git_and_warns_once, the_forced_config_pins_the_download_redirect_switches, a_family_fetch_over_budget_names_the_cap_and_compaction, the serve-gate wording in git_serve_tests, and serve_start_reaps_caches_nobody_holds. --- crates/cli/tests/git_bucket.rs | 37 +- crates/forge/tests/e2e_git.rs | 10 +- crates/objio/src/git.rs | 425 ++++++++++++++---- crates/repo/src/bucket_transport.rs | 8 +- .../src/bucket_transport/git_serve_tests.rs | 2 +- crates/repo/src/wire.rs | 8 +- 6 files changed, 381 insertions(+), 109 deletions(-) diff --git a/crates/cli/tests/git_bucket.rs b/crates/cli/tests/git_bucket.rs index 9f3a1ab2..fde1b660 100644 --- a/crates/cli/tests/git_bucket.rs +++ b/crates/cli/tests/git_bucket.rs @@ -250,7 +250,7 @@ fn serve_store_sc_git_gates_private_branches_until_the_operator_opts_in() { ok(sc(&a, &["commit", "-m", "embargoed", "--identity", idp])); let e = err_text(&sc(&a, &["push", "origin"])); assert!(e.contains("--allow-private-branches"), "{e}"); - assert!(e.contains("Nothing was uploaded"), "{e}"); + assert!(e.contains("No private-branch object was uploaded"), "{e}"); assert_eq!( refs_on_carrier(&hub), public_refs, @@ -279,11 +279,13 @@ fn serve_store_sc_git_gates_private_branches_until_the_operator_opts_in() { } /// A starting `sc serve --store sc+git::` removes the Git caches a killed -/// earlier server left in the serve home, and only those whose owner is gone: -/// another server on the same home may be live. Unix only, like the sweep. +/// earlier server left in the serve home, and only those nobody holds: another +/// server on the same home may be live, and liveness is the owner lock, not a +/// pid (the test process holds one lock, in a directory named for a dead pid). +/// Unix only, like the sweep. #[cfg(unix)] #[test] -fn serve_start_reaps_caches_of_dead_servers_only() { +fn serve_start_reaps_caches_nobody_holds() { let root = tmp("serve-reap"); let home = root.join("home"); init_repo(&home); @@ -292,12 +294,25 @@ fn serve_start_reaps_caches_of_dead_servers_only() { let mut gone = Command::new("true").spawn().unwrap(); let dead = gone.id(); gone.wait().unwrap(); - let mut live = Command::new("sleep").arg("60").spawn().unwrap(); - let stale = scratch.join(format!("sc-git-bucket-{dead}-aaaa")); - let foreign = scratch.join(format!("sc-git-bucket-{}-bbbb", live.id())); - for d in [&stale, &foreign] { + // A killed server's cache: the lock file is there, nobody holds it. Its pid + // is that of the running test, which no pid check could tell from a live one. + let stale = scratch.join(format!("sc-git-bucket-{}-aaaa", std::process::id())); + // A live server's cache: the lock is held (by this process, from the point + // of view of the `sc serve` that sweeps), whatever its pid looks like. + let live = scratch.join(format!("sc-git-bucket-{dead}-bbbb")); + // Not a cache this sweep can judge: no lock file. + let unlocked = scratch.join(format!("sc-git-bucket-{dead}-cccc")); + for d in [&stale, &live, &unlocked] { std::fs::create_dir_all(d.join("objects")).unwrap(); } + for d in [&stale, &live] { + std::fs::write(d.join(".owner.lock"), b"").unwrap(); + } + let held = std::fs::OpenOptions::new() + .write(true) + .open(live.join(".owner.lock")) + .unwrap(); + held.lock().unwrap(); // No stdin: the session ends at once, after the startup sweep. let out = Command::new(env!("CARGO_BIN_EXE_sc")) .args([ @@ -311,10 +326,10 @@ fn serve_start_reaps_caches_of_dead_servers_only() { .output() .unwrap(); let _ = out; - live.kill().unwrap(); - live.wait().unwrap(); assert!(!stale.exists(), "the dead server's cache must be reaped"); - assert!(foreign.exists(), "a live server's cache must stay"); + assert!(live.exists(), "a cache whose lock is held must stay"); + assert!(unlocked.exists(), "a directory with no lock file must stay"); + drop(held); std::fs::remove_dir_all(&root).unwrap(); } diff --git a/crates/forge/tests/e2e_git.rs b/crates/forge/tests/e2e_git.rs index e59a8e2d..6ddd860f 100644 --- a/crates/forge/tests/e2e_git.rs +++ b/crates/forge/tests/e2e_git.rs @@ -1519,8 +1519,6 @@ fn refs_sc_create_cas_update_fetch_and_delete_round_trip_through_the_real_receiv } } -/// One receive-pack request by hand (`old` and `new` are what the client believes), so the -/// server's own compare-and-swap is exercised: the real client checks the advertisement first. /// How `sc+git::` reads one mutable key (`GitBucket::get`, ADR-0060): a single /// `git fetch` of exactly `refs/sc/` under protocol v2. The host must answer /// the ref listing with that ref alone: the branch and the other keys never cross @@ -1555,9 +1553,13 @@ fn a_v2_fetch_of_one_sc_ref_gets_a_filtered_advertisement() { "--end-of-options", &url, &format!("+{m}:{m}"), - // what `GitBucket` adds so abbreviations cannot stand in for the key + // The five spellings `GitBucket` excludes so that a branch, tag or + // remote-tracking ref cannot stand in for the key (`exact_only` in objio). &format!("^refs/{m}"), + &format!("^refs/tags/{m}"), &format!("^refs/heads/{m}"), + &format!("^refs/remotes/{m}"), + &format!("^refs/remotes/{m}/HEAD"), ], ); assert!(ok, "fetch: {text}"); @@ -1577,6 +1579,8 @@ fn a_v2_fetch_of_one_sc_ref_gets_a_filtered_advertisement() { assert_eq!(listed, [m], "the host must not list other refs: {text}"); } +/// One receive-pack request by hand (`old` and `new` are what the client believes), so the +/// server's own compare-and-swap is exercised: the real client checks the advertisement first. fn raw_update(s: &Srv, auth: &str, old: &str, new: &str, refname: &str) -> String { let pkt = |t: &str| format!("{:04x}{t}", t.len() + 4); let mut body = pkt(&format!( diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index b265b211..0e0134b3 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -68,12 +68,14 @@ const FAMILY_BUDGET: u64 = MAX_VALUE; /// fetch send `ref-prefix`, so the host filters its ref advertisement down to /// what is asked for; `fetch.unpackLimit=1` keeps every incoming object in one /// `objects/pack/tmp_pack_*` file (never loose objects), which is what the -/// fetch watchdog measures; `fetch.uriProtocols=` stops a host from redirecting -/// the download to packfile URIs the watchdog could not see. +/// fetch watchdog measures; `fetch.uriProtocols=` and `transfer.bundleURI=false` +/// stop a host from redirecting the download to packfile or bundle URLs the +/// watchdog could not see. const GIT_CONFIG: &[&str] = &[ "protocol.version=2", "fetch.unpackLimit=1", "fetch.uriProtocols=", + "transfer.bundleURI=false", "protocol.allow=never", "protocol.file.allow=always", "protocol.ssh.allow=always", @@ -217,6 +219,10 @@ struct Tuning { /// Fixed polling interval; `None` polls after 2 ms and backs off to 50 ms /// (tests fix it to make a fetch finish between two polls). poll: Option, + /// The program that signals a process group in serve mode (tests point it + /// at a missing one). + #[cfg(unix)] + kill: &'static str, } impl Default for Tuning { @@ -225,6 +231,8 @@ impl Default for Tuning { budget: None, idle: IDLE, poll: None, + #[cfg(unix)] + kill: "kill", } } } @@ -246,6 +254,10 @@ pub struct GitBucket { chunk: usize, state: Mutex>, tune: Tuning, + /// Serve mode: the lock on `/.owner.lock`, held for the bucket's whole + /// life. It is how [`GitBucket::reap_stale`] tells a live cache from one a + /// killed process left behind (the OS drops the lock with its holder). + owner: Option, /// Serve mode: every fetch leads its own process group, so the watchdog /// can kill `ssh` and `git-remote-*` along with git. Off for the CLI, where /// the terminal's Ctrl-C must still reach git and ssh must still be able to @@ -395,7 +407,11 @@ fn remove_stale_locks(dir: &Path, depth: u32) { Ok(t) if t.is_dir() && depth < 8 && e.file_name() != "objects" => { remove_stale_locks(&path, depth + 1); } - Ok(t) if t.is_file() && path.extension().is_some_and(|x| x == "lock") => { + Ok(t) + if t.is_file() + && e.file_name() != OWNER_LOCK + && path.extension().is_some_and(|x| x == "lock") => + { let _ = std::fs::remove_file(&path); } _ => {} @@ -403,6 +419,36 @@ fn remove_stale_locks(dir: &Path, depth: u32) { } } +/// Signal every process of group `pgid` with SIGKILL through the system `kill`. +/// False only when that program cannot be run; a non-zero exit (the group is +/// already gone) is fine. +#[cfg(unix)] +fn kill_group(program: &str, pgid: u32) -> bool { + Command::new(program) + .args(["-s", "KILL", "--"]) + .arg(format!("-{pgid}")) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok() +} + +/// Set once a missing `kill` has been reported. +#[cfg(unix)] +static KILL_WARNED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// Print `msg` to stderr unless `flag` says it was already printed; true when it +/// printed. +#[cfg(unix)] +fn warn_once(flag: &std::sync::atomic::AtomicBool, msg: &str) -> bool { + let first = !flag.swap(true, std::sync::atomic::Ordering::Relaxed); + if first { + eprintln!("{msg}"); + } + first +} + /// Read up to `cap` bytes from `r` and discard the rest, so the writer never /// blocks on a full pipe. The flag says more than `cap` bytes came. fn drain(r: &mut impl Read, cap: usize) -> (Vec, bool) { @@ -568,8 +614,10 @@ impl GitBucket { /// cache directory is made under `parent` (the serve home's `.sc/tmp/`, created /// if missing) instead of the system temp directory, and every fetch /// leads its own process group so the watchdog, when it kills one, takes `ssh` or - /// `git-remote-https` with it. A killed server can leave the cache behind; - /// [`GitBucket::reap_stale`] removes such leftovers at the next start. + /// `git-remote-https` with it (which needs a `kill` program on the PATH). The + /// bucket holds a lock on `/.owner.lock` until it is dropped. A killed + /// server can leave the cache behind; [`GitBucket::reap_stale`] removes such + /// leftovers at the next start. pub fn open_in(url: &str, parent: &Path) -> Result { let git = std::env::var("SC_GIT").unwrap_or_else(|_| "git".to_string()); Self::open_full(url, git, CHUNK, Some(parent), true) @@ -589,12 +637,13 @@ impl GitBucket { Self::validate_url(url)?; #[cfg(not(unix))] let _ = group; // process groups are unix-only; the watchdog kills the child alone - let cache = match parent { + let (cache, owner) = match parent { Some(p) => { std::fs::create_dir_all(p)?; - new_cache_dir_in(p)? + let (cache, owner) = new_owned_cache_dir_in(p)?; + (cache, Some(owner)) } - None => new_cache_dir_in(&std::env::temp_dir())?, + None => (new_cache_dir_in(&std::env::temp_dir())?, None), }; let b = GitBucket { url: url.to_string(), @@ -603,6 +652,7 @@ impl GitBucket { chunk: chunk.max(1), state: Mutex::new(HashMap::new()), tune: Tuning::default(), + owner, #[cfg(unix)] group, }; @@ -804,18 +854,13 @@ impl GitBucket { self.discard_download(&pack_dir, &before); return Err(over_budget(watch)); } - let held = |what: &str| { - Error::Backend(format!( - "git exited but left its {what} open for {:?} (a helper outlived it)", - self.tune.idle - )) - }; - let (stdout, over) = orx - .recv_timeout(self.tune.idle) - .map_err(|_| held("stdout"))?; - let stderr = erx - .recv_timeout(self.tune.idle) - .map_err(|_| held("stderr"))?; + // One deadline for both pipes. A helper that outlives git and holds one + // open is still in git's process group in serve mode, so the group is + // signalled before the error returns (the group is non-empty, so its id + // cannot have been reused). + let deadline = Instant::now() + self.tune.idle; + let (stdout, over) = self.receive(&mut child, &orx, deadline, "stdout")?; + let stderr = self.receive(&mut child, &erx, deadline, "stderr")?; if over { return Err(Error::Backend(format!( "git output exceeded {max_out} bytes (hostile or corrupt remote)" @@ -829,20 +874,45 @@ impl GitBucket { }) } + /// Wait for a pipe reader's result until `deadline`. Past it, a helper that + /// outlived git holds the pipe: stop the group (serve mode) and fail. + fn receive( + &self, + child: &mut Child, + rx: &mpsc::Receiver, + deadline: Instant, + what: &str, + ) -> Result { + let left = deadline.saturating_duration_since(Instant::now()); + rx.recv_timeout(left).map_err(|_| { + self.stop(child); + Error::Backend(format!( + "git exited but left its {what} open for {:?} (a helper outlived it)", + self.tune.idle + )) + }) + } + /// Kill a watched git. Called before the child is reaped, so its group id - /// cannot have been reused. In serve mode the child leads its own process + /// cannot have been reused (after a clean exit only [`Self::receive`] calls + /// it, and only while a helper still holds a pipe, so the group is not + /// empty). In serve mode the child leads its own process /// group and the whole group is signalled, so `ssh` and `git-remote-*` - /// die with it; the system `kill` keeps this std-only, with no `unsafe`. + /// die with it; the system `kill` keeps this std-only, with no `unsafe`, and + /// serve mode on unix therefore needs a `kill` on the PATH. Without one only + /// git itself is killed, and a warning says so once. fn stop(&self, child: &mut Child) { #[cfg(unix)] - if self.group { - let _ = Command::new("kill") - .args(["-s", "KILL", "--"]) - .arg(format!("-{}", child.id())) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); + if self.group && !kill_group(self.tune.kill, child.id()) { + warn_once( + &KILL_WARNED, + &format!( + "sc: cannot run `{}` to stop a fetch's process group; only git itself is \ + killed, so ssh or git-remote-https may keep running (put a `kill` \ + program on the PATH)", + self.tune.kill + ), + ); } let _ = child.kill(); } @@ -967,7 +1037,9 @@ impl GitBucket { /// The download budget is [`FAMILY_BUDGET`]; a fetch of exactly one key /// goes through [`Self::fetch_one`] and gets that key's own, tighter one. fn fetch(&self, refspecs: &[String], family: bool) -> Result { - let what = "the 2 GiB budget of a family or batched fetch".to_string(); + let what = "the 2 GiB budget of a family or batched fetch; a legitimate store whose \ + log, checkpoint or idx entries together exceed it needs compaction" + .to_string(); self.fetch_watched(refspecs, family, FAMILY_BUDGET, what) } @@ -1325,6 +1397,9 @@ impl GitBucket { impl Drop for GitBucket { fn drop(&mut self) { + // Release the lock first: an open file keeps a directory from being + // removed on Windows. + drop(self.owner.take()); let _ = std::fs::remove_dir_all(&self.cache); } } @@ -1332,33 +1407,43 @@ impl Drop for GitBucket { /// Prefix of every cache directory name: `sc-git-bucket--`. const CACHE_PREFIX: &str = "sc-git-bucket-"; +/// Name of the file whose lock marks a cache directory as owned by a live +/// [`GitBucket::open_in`] bucket. +const OWNER_LOCK: &str = ".owner.lock"; + impl GitBucket { /// Remove cache directories a killed process left under `parent`, and return - /// how many were removed. Only a `sc-git-bucket--*` directory whose - /// owner process is provably gone is touched: several `sc serve --stdio` - /// processes can share one `parent`, and a live one's cache must survive. - /// A name that does not parse, this process's own directories, and a pid - /// that cannot be shown dead are all kept. Unix only; on other platforms it - /// removes nothing. + /// how many were removed. A `sc-git-bucket-*` directory is removed only when + /// its `.owner.lock` can be locked, which means no live bucket holds it: the + /// OS drops a lock with its holder, so liveness does not depend on pid + /// numbers (pid reuse, containers where every server is pid 1, several pid + /// namespaces on one host) or on any helper program. Several `sc serve + /// --stdio` processes can share one `parent`, and a live one's cache + /// survives. A directory with no lock file, a symlink, and any other name + /// are kept. The lock is an advisory file lock on one host: it is not + /// reliable over NFS and similar (as for `DirBucket`). Unix only; on other + /// platforms it removes nothing. pub fn reap_stale(parent: &Path) -> usize { #[cfg(unix)] { let mut removed = 0; for e in std::fs::read_dir(parent).into_iter().flatten().flatten() { - let name = e.file_name().to_string_lossy().into_owned(); - let Some(rest) = name.strip_prefix(CACHE_PREFIX) else { + // `file_type` does not follow a symlink, so only a real directory goes. + if !e.file_name().to_string_lossy().starts_with(CACHE_PREFIX) + || !e.file_type().is_ok_and(|t| t.is_dir()) + { continue; - }; - let Some(pid) = rest.split('-').next().and_then(|p| p.parse::().ok()) else { + } + let lock = e.path().join(OWNER_LOCK); + if !std::fs::symlink_metadata(&lock).is_ok_and(|m| m.is_file()) { + continue; + } + let Ok(held) = std::fs::OpenOptions::new().write(true).open(&lock) else { continue; }; - // `file_type` does not follow a symlink, so only a real directory goes. - let is_dir = e.file_type().is_ok_and(|t| t.is_dir()); - if is_dir - && pid != std::process::id() - && pid_is_gone(pid) - && std::fs::remove_dir_all(e.path()).is_ok() - { + // Hold the lock through the removal: an owner that is just being + // created blocks on it, then sees its lock file gone and starts over. + if held.try_lock().is_ok() && std::fs::remove_dir_all(e.path()).is_ok() { removed += 1; } } @@ -1372,20 +1457,33 @@ impl GitBucket { } } -/// Is `pid` provably not running? `kill -0` says "No such process" for a pid -/// that is gone; any other answer (alive, not permitted, unknown wording) is -/// not proof, so the caller keeps what it owns. -#[cfg(unix)] -fn pid_is_gone(pid: u32) -> bool { - Command::new("kill") - .args(["-0", &pid.to_string()]) - .env("LC_ALL", "C") - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .output() - .is_ok_and(|o| { - !o.status.success() && String::from_utf8_lossy(&o.stderr).contains("No such process") - }) +/// A fresh cache directory under `parent` with its owner lock taken. A sweep +/// ([`GitBucket::reap_stale`]) that raced the creation either blocks the lock +/// until it has removed the directory or removes it before the lock exists; both +/// show as a missing lock file, and the next attempt starts over. +fn new_owned_cache_dir_in(parent: &Path) -> Result<(PathBuf, std::fs::File)> { + for _ in 0..8 { + let dir = new_cache_dir_in(parent)?; + let lock = dir.join(OWNER_LOCK); + let held = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&lock) + .and_then(|f| f.lock().map(|()| f)); + match held { + Ok(f) if std::fs::symlink_metadata(&lock).is_ok() => return Ok((dir, f)), + Ok(_) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => { + let _ = std::fs::remove_dir_all(&dir); + return Err(e.into()); + } + } + let _ = std::fs::remove_dir_all(&dir); + } + Err(Error::Backend( + "cannot create a locked cache directory".into(), + )) } /// Fresh private directory `sc-git-bucket--` (mode 0700) under `parent`. @@ -3133,34 +3231,131 @@ mod tests { } /// A git that exits cleanly while a helper keeps its pipes open is an error - /// within `idle`, not a hang. + /// within `idle`, not a hang. In serve mode the helper is still in git's + /// process group, and the group is stopped before the error returns, so it + /// does not outlive the call; in CLI mode only git was ever its to kill. #[cfg(unix)] #[test] fn a_helper_holding_the_pipes_after_exit_is_an_error() { need_git!(); - let t = Tmp::new("held"); + for group in [true, false] { + let t = Tmp::new("held"); + let (_, url) = hub(&t); + let pids = t.0.join("pids"); + let body = format!( + "case \" $* \" in *\" fetch \"*) sleep 600 & echo $! > {}; exit 0;; esac", + pids.display() + ); + let sh = shim(&t, "held.sh", &body); + let mut b = GitBucket::open_full(&url, sh, CHUNK, None, group).unwrap(); + b.tune.idle = Duration::from_millis(800); + let started = Instant::now(); + let e = get_err(&b, "manifest"); + assert!(e.contains("left its"), "{e}"); + assert!(started.elapsed() < Duration::from_secs(90)); + let helper: u32 = std::fs::read_to_string(&pids) + .unwrap() + .trim() + .parse() + .unwrap(); + if !group { + kill_pid(helper); + } + wait_gone(helper); + } + } + + /// Both pipes share one deadline after git exits. Here stdout is held for + /// half of `idle` and stderr for 1.25 times `idle`: waiting `idle` for each + /// in turn (the old behavior) would let the second one finish in time. + #[cfg(unix)] + #[test] + fn the_pipe_wait_after_exit_has_one_deadline() { + need_git!(); + let t = Tmp::new("deadline"); + let (_, url) = hub(&t); + let body = "case \" $* \" in *\" fetch \"*) sleep 4 2>/dev/null & sleep 10 >/dev/null & exit 0;; esac"; + let sh = shim(&t, "deadline.sh", body); + let mut b = open_git(&url, &sh); + b.tune.idle = Duration::from_secs(8); + let e = get_err(&b, "manifest"); + assert!(e.contains("left its stderr open"), "{e}"); + } + + /// Without a `kill` program serve mode cannot signal the group. It falls back + /// to killing git alone, returns, and warns once instead of staying silent. + #[cfg(unix)] + #[test] + fn a_missing_kill_program_falls_back_to_killing_git_and_warns_once() { + need_git!(); + let flag = std::sync::atomic::AtomicBool::new(false); + assert!(warn_once(&flag, "first"), "the first report prints"); + assert!(!warn_once(&flag, "second"), "later reports stay quiet"); + assert!( + !kill_group("/no/such/kill", 1), + "an unrunnable program is reported" + ); + + let t = Tmp::new("nokill"); let (_, url) = hub(&t); let pids = t.0.join("pids"); let body = format!( - "case \" $* \" in *\" fetch \"*) sleep 600 & echo $! > {}; exit 0;; esac", + "case \" $* \" in *\" fetch \"*) sleep 600 & echo $! > {}; wait;; esac", pids.display() ); - let sh = shim(&t, "held.sh", &body); - let mut b = open_git(&url, &sh); + let sh = shim(&t, "nokill.sh", &body); + let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); b.tune.idle = Duration::from_millis(800); + b.tune.kill = "/no/such/kill"; let started = Instant::now(); let e = get_err(&b, "manifest"); - assert!(e.contains("left its"), "{e}"); + assert!(e.contains("no progress"), "{e}"); assert!(started.elapsed() < Duration::from_secs(90)); + // Only git was killed: the helper outlives the call, which is the + // documented cost of a missing `kill`. let helper: u32 = std::fs::read_to_string(&pids) .unwrap() .trim() .parse() .unwrap(); + let alive = Command::new("ps") + .args(["-o", "stat=", "-p", &helper.to_string()]) + .output() + .unwrap(); + let stat = String::from_utf8_lossy(&alive.stdout).trim().to_string(); + assert!(!stat.is_empty() && !stat.starts_with('Z'), "stat {stat:?}"); kill_pid(helper); wait_gone(helper); } + /// The family budget is a cliff for a legitimate store; the error says so. + #[test] + fn a_family_fetch_over_budget_names_the_cap_and_compaction() { + need_git!(); + let t = Tmp::new("family"); + let (hubdir, url) = hub(&t); + plant_big(&hubdir, "log/1", 3 << 20, false); + let mut b = open(&url); + b.tune.budget = Some(256 << 10); + let e = get_err(&b, "log/1"); + assert!( + e.contains("2 GiB budget of a family") && e.contains("needs compaction"), + "{e}" + ); + } + + #[test] + fn the_forced_config_pins_the_download_redirect_switches() { + for pinned in [ + "protocol.version=2", + "fetch.unpackLimit=1", + "fetch.uriProtocols=", + "transfer.bundleURI=false", + ] { + assert!(GIT_CONFIG.contains(&pinned), "{pinned}"); + } + } + /// Serve mode (own cache parent, own process group) is the same store. #[cfg(unix)] #[test] @@ -3178,49 +3373,103 @@ mod tests { assert_eq!(std::fs::read_dir(&parent).unwrap().count(), 0); } - /// The startup sweep removes only what a dead process left. A live foreign - /// process (another `sc serve --stdio` on the same home), this process, an - /// unparsable name and unrelated entries stay. + /// The startup sweep goes by the owner lock, not by pid numbers. A cache + /// whose lock is free goes even when its pid is ours or alive (pid reuse, + /// every server being pid 1 in a container); one whose lock is held stays + /// even when its pid looks dead; so do a directory with no lock file, a + /// symlink, and other names. This process's own live buckets stay too. #[cfg(unix)] #[test] - fn reap_stale_removes_only_caches_of_dead_processes() { + fn reap_stale_goes_by_the_owner_lock_not_by_pid() { need_git!(); let t = Tmp::new("reap"); let (_, url) = hub(&t); let parent = t.0.join("tmp"); std::fs::create_dir_all(&parent).unwrap(); - let mk = |name: &str| { + let mk = |name: &str, lock: bool| { let p = parent.join(name); std::fs::create_dir_all(p.join("objects")).unwrap(); + if lock { + std::fs::write(p.join(OWNER_LOCK), b"").unwrap(); + } p }; let mut gone = Command::new("true").spawn().unwrap(); let dead = gone.id(); gone.wait().unwrap(); - let mut live = Command::new("sleep").arg("30").spawn().unwrap(); - let stale = mk(&format!("{CACHE_PREFIX}{dead}-aaaa")); - let foreign = mk(&format!("{CACHE_PREFIX}{}-bbbb", live.id())); - let junk = mk(&format!("{CACHE_PREFIX}notapid-cccc")); - let other = mk("serve-bucket-1-0"); - // two live buckets of this process, and a symlink named like a stale cache + // Free lock, whatever the pid says: our own pid, a live one, a junk name. + let stale_own = mk(&format!("{CACHE_PREFIX}{}-aaaa", std::process::id()), true); + let stale_one = mk(&format!("{CACHE_PREFIX}1-bbbb"), true); + let stale_junk = mk(&format!("{CACHE_PREFIX}notapid-cccc"), true); + // Held lock, whatever the pid says: a dead one (another namespace, or a + // server that is alive under a pid we cannot see). + let held_dir = mk(&format!("{CACHE_PREFIX}{dead}-dddd"), true); + let held = std::fs::OpenOptions::new() + .write(true) + .open(held_dir.join(OWNER_LOCK)) + .unwrap(); + held.lock().unwrap(); + // Never reaped: no lock file, another name, a symlink into a stale-looking dir. + let no_lock = mk(&format!("{CACHE_PREFIX}{dead}-eeee"), false); + let other = mk("serve-bucket-1-0", true); + let link = parent.join(format!("{CACHE_PREFIX}{dead}-link")); + std::os::unix::fs::symlink(&other, &link).unwrap(); + // Two live buckets of this process. let (a, b) = ( GitBucket::open_in(&url, &parent).unwrap(), GitBucket::open_in(&url, &parent).unwrap(), ); - let link = parent.join(format!("{CACHE_PREFIX}{dead}-link")); - std::os::unix::fs::symlink(&other, &link).unwrap(); - assert_eq!(GitBucket::reap_stale(&parent), 1); - assert!(!stale.exists()); - for kept in [&foreign, &junk, &other, &link, &a.cache, &b.cache] { + assert_eq!(GitBucket::reap_stale(&parent), 3); + for gone in [&stale_own, &stale_one, &stale_junk] { + assert!( + !gone.exists(), + "{gone:?} has a free lock and must be reaped" + ); + } + for kept in [&held_dir, &no_lock, &other, &link, &a.cache, &b.cache] { assert!(kept.exists(), "{kept:?} must survive"); } + assert!(other.join(OWNER_LOCK).exists(), "a symlink is not followed"); assert!(a.put_new("k", b"v").unwrap() && b.exists("k").unwrap()); - live.kill().unwrap(); - live.wait().unwrap(); - // once its owner is dead the foreign cache goes too + + // The holder going away (a killed server) is what frees the cache. + drop(held); assert_eq!(GitBucket::reap_stale(&parent), 1); - assert!(!foreign.exists()); + assert!(!held_dir.exists()); + // So does dropping a bucket: it removes its own directory. + let cache = a.cache.clone(); + drop(a); + assert!(!cache.exists()); assert_eq!(GitBucket::reap_stale(&parent.join("missing")), 0); } + + /// A sweep that runs while buckets are being opened and dropped never takes + /// a live bucket's cache: the owner lock is taken before the bucket is used, + /// and a sweep that raced the creation makes the opener start over. + #[cfg(unix)] + #[test] + fn a_sweep_never_removes_a_live_bucket_while_others_open() { + need_git!(); + let t = Tmp::new("sweep"); + let (_, url) = hub(&t); + let parent = t.0.join("tmp"); + let done = std::sync::atomic::AtomicBool::new(false); + std::thread::scope(|s| { + s.spawn(|| { + while !done.load(std::sync::atomic::Ordering::Relaxed) { + GitBucket::reap_stale(&parent); + } + }); + for i in 0..12 { + let b = GitBucket::open_in(&url, &parent).unwrap(); + assert!( + b.put_new(&format!("k{i}"), b"v").unwrap(), + "cache {i} was swept" + ); + } + done.store(true, std::sync::atomic::Ordering::Relaxed); + }); + assert_eq!(std::fs::read_dir(&parent).unwrap().count(), 0); + } } diff --git a/crates/repo/src/bucket_transport.rs b/crates/repo/src/bucket_transport.rs index d11b0511..b1f23ace 100644 --- a/crates/repo/src/bucket_transport.rs +++ b/crates/repo/src/bucket_transport.rs @@ -352,14 +352,16 @@ impl BucketTransport { } /// The refusal for private-branch objects on a gated `sc+git::` store. The - /// text names the server-side flag; it never shows the store URL. + /// text names the server-side flag; it never shows the store URL. It does + /// not claim that the whole push uploaded nothing: public objects of the same + /// push may already sit on the host as an orphan pack. fn sealed_refusal() -> Error { Error::InvalidArgument( "this server stores into a Git repository (sc+git::) and was not started with \ --allow-private-branches, so it refuses private-branch objects: a Git host shows \ the branch name, recipient ids and sizes to everyone who can read the repository \ - and keeps the ciphertext after any later revoke. Nothing was uploaded. The \ - operator can restart `sc serve` with --allow-private-branches" + and keeps the ciphertext after any later revoke. No private-branch object was \ + uploaded. The operator can restart `sc serve` with --allow-private-branches" .into(), ) } diff --git a/crates/repo/src/bucket_transport/git_serve_tests.rs b/crates/repo/src/bucket_transport/git_serve_tests.rs index 01e4e7fb..a2875608 100644 --- a/crates/repo/src/bucket_transport/git_serve_tests.rs +++ b/crates/repo/src/bucket_transport/git_serve_tests.rs @@ -237,7 +237,7 @@ fn finish(client: Client, srv: Server) { fn assert_refused(e: &Error) { let m = e.to_string(); assert!(m.contains("--allow-private-branches"), "{m}"); - assert!(m.contains("Nothing was uploaded"), "{m}"); + assert!(m.contains("No private-branch object was uploaded"), "{m}"); } fn allowing() -> WirePolicy { diff --git a/crates/repo/src/wire.rs b/crates/repo/src/wire.rs index 9d3f10d5..00c14e3e 100644 --- a/crates/repo/src/wire.rs +++ b/crates/repo/src/wire.rs @@ -1143,9 +1143,11 @@ fn open_bucket_serve_transport( /// Remove Git cache directories a killed `sc serve --store sc+git::…` process /// left under the serve home's `.sc/tmp/`, and return how many were removed. Call -/// once at startup. Only a directory whose owner process is provably gone is -/// removed: many `sc serve --stdio` processes can share one home, and a live -/// session's cache must survive. Unix only; elsewhere it removes nothing. +/// once at startup. Only a directory whose owner lock is free is removed: many +/// `sc serve --stdio` processes can share one home, and a live session's cache, +/// whose bucket holds the lock, must survive. Liveness is that advisory file +/// lock on one host (unreliable over NFS and similar, as for `DirBucket`), not a +/// pid. Unix only; elsewhere it removes nothing. pub fn reap_stale_serve_scratch(home: &std::path::Path) -> usize { scl_objio::GitBucket::reap_stale(&crate::layout::Layout::at(home).tmp_dir()) } From 8bb74f83d31b444e1baed2642087f6d2a496799e Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 16:38:52 +0200 Subject: [PATCH 5/9] fix(objio): absolute cache path, and one terminate-and-discard path for every breach Sol review round 1 on serve --store sc+git::. S1: git runs with the cache directory as its working directory and GIT_DIR set to the same path, so a relative serve home (`.`) was resolved from inside the cache and every Git-backed session failed ("not a git repository: ./.sc/tmp/sc-git-bucket-..."). open_full now makes the cache parent absolute (std::path::absolute) before any git launch, for the serve home and for the system temp directory alike. S2: the final budget check discarded the download without handling a surviving helper, and the pipe-timeout path stopped the group without discarding. Every breach (budget during the run, budget at the final check, no progress, a pipe held open after exit) now takes terminate_and_discard: stop git (the group in serve mode), reap it, keep watching until the pipes have closed and objects/pack has stopped changing or a 5 s settle deadline passes, then discard what the fetch added. Tests, each failing without its fix (checked by mutation): - a_relative_cache_parent_works (objio) and serve_store_sc_git_works_with_a_relative_serve_home (cli, serve home `.` with the server's cwd set to it, push and clone through it). - a_clean_exit_over_budget_still_stops_a_helper_and_discards (final check). - a_helper_holding_the_pipes_after_exit_is_an_error and a_silent_fetch_hits_the_no_progress_deadline now also assert that the added files are gone; a_runaway_fetch_is_killed_with_its_descendants covers the in-run budget path. Serve mode asserts no surviving helper. ADR-0060 and THREAT-MODEL describe the single breach path and the absolute cache path. --- crates/cli/tests/git_bucket.rs | 60 ++++++++- crates/objio/src/git.rs | 219 +++++++++++++++++++++++++++------ 2 files changed, 239 insertions(+), 40 deletions(-) diff --git a/crates/cli/tests/git_bucket.rs b/crates/cli/tests/git_bucket.rs index fde1b660..7b2a48c7 100644 --- a/crates/cli/tests/git_bucket.rs +++ b/crates/cli/tests/git_bucket.rs @@ -162,15 +162,28 @@ fn serve_allow_private_branches_needs_a_git_store() { /// Spawn `sc serve --http 127.0.0.1:0 --store `; returns /// the child and its announced `host:port` (as `http_remote.rs` does). fn spawn_serve(home: &Path, store: &str, extra: &[&str]) -> (std::process::Child, String) { + spawn_serve_in(None, home.as_os_str(), store, extra) +} + +/// [`spawn_serve`] with the serve home given as supplied (`.` is a relative +/// home) and an optional working directory for the server. +fn spawn_serve_in( + cwd: Option<&Path>, + home: &std::ffi::OsStr, + store: &str, + extra: &[&str], +) -> (std::process::Child, String) { use std::io::BufRead; - let mut child = Command::new(env!("CARGO_BIN_EXE_sc")) - .args(["serve", "--http", "127.0.0.1:0", "--store", store]) + let mut cmd = Command::new(env!("CARGO_BIN_EXE_sc")); + cmd.args(["serve", "--http", "127.0.0.1:0", "--store", store]) .args(extra) .arg(home) .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .expect("spawn sc serve"); + .stderr(std::process::Stdio::piped()); + if let Some(cwd) = cwd { + cmd.current_dir(cwd); + } + let mut child = cmd.spawn().expect("spawn sc serve"); let mut line = String::new(); std::io::BufReader::new(child.stdout.take().unwrap()) .read_line(&mut line) @@ -278,6 +291,43 @@ fn serve_store_sc_git_gates_private_branches_until_the_operator_opts_in() { } } +/// The serve home is used as supplied, so `.` (with the server's working +/// directory set to it) must work: git runs with the cache as its working +/// directory, and a relative cache path used to be resolved from inside it. +#[test] +fn serve_store_sc_git_works_with_a_relative_serve_home() { + let root = tmp("serve-rel"); + let hub = root.join("hub.git"); + assert!(Command::new("git") + .args(["init", "--bare", "-q", hub.to_str().unwrap()]) + .status() + .unwrap() + .success()); + let store = format!("sc+git::file://{}", hub.display()); + let home = root.join("home"); + init_repo(&home); + let a = root.join("a"); + init_repo(&a); + std::fs::write(a.join("README.md"), "public\n").unwrap(); + ok(sc(&a, &["commit", "-m", "base"])); + + let (mut srv, addr) = spawn_serve_in(Some(&home), std::ffi::OsStr::new("."), &store, &[]); + let url = format!("sc+http://{addr}/x"); + ok(sc(&a, &["remote", "add", "origin", &url])); + ok(sc(&a, &["push", "origin"])); + assert!(refs_on_carrier(&hub).contains("refs/sc/manifest")); + let clone = root.join("clone"); + ok(sc(&root, &["clone", &url, clone.to_str().unwrap()])); + assert_eq!( + std::fs::read_to_string(clone.join("README.md")).unwrap(), + "public\n" + ); + wait_until_empty(&home.join(".sc").join("tmp")); + srv.kill().unwrap(); + srv.wait().unwrap(); + std::fs::remove_dir_all(&root).unwrap(); +} + /// A starting `sc serve --store sc+git::` removes the Git caches a killed /// earlier server left in the serve home, and only those nobody holds: another /// server on the same home may be live, and liveness is the owner lock, not a diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index 0e0134b3..9e925f70 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -62,6 +62,9 @@ const DELETE_ARG_BUDGET: usize = 24 << 10; const IDLE: Duration = Duration::from_secs(120); /// Download budget of a fetch of a whole key family or a batch of keys. const FAMILY_BUDGET: u64 = MAX_VALUE; +/// How long a terminated fetch is watched for a surviving helper (pipes still +/// open, pack directory still changing) before its download is discarded anyway. +const SETTLE: Duration = Duration::from_secs(5); /// `-c` overrides on every git call: transport allowlist, no hooks, no signing, /// no background maintenance, bounded stalls. `protocol.version=2` makes a @@ -237,6 +240,40 @@ impl Default for Tuning { } } +/// The two detached pipe readers of a watched git and what they have delivered. +struct Readers { + out: mpsc::Receiver<(Vec, bool)>, + err: mpsc::Receiver>, + got_out: Option<(Vec, bool)>, + got_err: Option>, +} + +impl Readers { + /// Collect what the readers deliver until `deadline`; the error names the + /// first pipe still open then. A reader that died counts as done and empty. + fn wait(&mut self, deadline: Instant) -> std::result::Result<(), &'static str> { + if self.got_out.is_none() { + let left = deadline.saturating_duration_since(Instant::now()); + match self.out.recv_timeout(left) { + Ok(v) => self.got_out = Some(v), + Err(mpsc::RecvTimeoutError::Disconnected) => { + self.got_out = Some(Default::default()) + } + Err(mpsc::RecvTimeoutError::Timeout) => return Err("stdout"), + } + } + if self.got_err.is_none() { + let left = deadline.saturating_duration_since(Instant::now()); + match self.err.recv_timeout(left) { + Ok(v) => self.got_err = Some(v), + Err(mpsc::RecvTimeoutError::Disconnected) => self.got_err = Some(Vec::new()), + Err(mpsc::RecvTimeoutError::Timeout) => return Err("stderr"), + } + } + Ok(()) + } +} + /// One watched fetch: its download budget and the name of the cap behind it. struct Watch { budget: u64, @@ -637,13 +674,20 @@ impl GitBucket { Self::validate_url(url)?; #[cfg(not(unix))] let _ = group; // process groups are unix-only; the watchdog kills the child alone + // The cache path must be absolute: git runs with the cache as its working + // directory and `GIT_DIR` set to the same path, and a relative one (a serve + // home given as `.`, a relative `TMPDIR`) would be resolved from inside it. let (cache, owner) = match parent { Some(p) => { - std::fs::create_dir_all(p)?; - let (cache, owner) = new_owned_cache_dir_in(p)?; + let p = std::path::absolute(p)?; + std::fs::create_dir_all(&p)?; + let (cache, owner) = new_owned_cache_dir_in(&p)?; (cache, Some(owner)) } - None => (new_cache_dir_in(&std::env::temp_dir())?, None), + None => ( + new_cache_dir_in(&std::path::absolute(std::env::temp_dir())?)?, + None, + ), }; let b = GitBucket { url: url.to_string(), @@ -843,24 +887,33 @@ impl GitBucket { pause = (pause * 2).min(Duration::from_millis(50)); } }; + let mut readers = Readers { + out: orx, + err: erx, + got_out: None, + got_err: None, + }; + // Every breach ends in `terminate_and_discard`: the budget during the + // run, the budget at the final check, no progress, a pipe that stays open. let Some(status) = status else { - // Signal before reaping, while the group id is still ours. - self.stop(&mut child); - let _ = child.wait(); - self.discard_download(&pack_dir, &before); + self.terminate_and_discard(&mut child, &mut readers, &pack_dir, &before); return Err(breach.expect("only a breach leaves the loop without a status")); }; if dir_bytes(&pack_dir).saturating_sub(base) > watch.budget { - self.discard_download(&pack_dir, &before); + self.terminate_and_discard(&mut child, &mut readers, &pack_dir, &before); return Err(over_budget(watch)); } // One deadline for both pipes. A helper that outlives git and holds one - // open is still in git's process group in serve mode, so the group is - // signalled before the error returns (the group is non-empty, so its id - // cannot have been reused). - let deadline = Instant::now() + self.tune.idle; - let (stdout, over) = self.receive(&mut child, &orx, deadline, "stdout")?; - let stderr = self.receive(&mut child, &erx, deadline, "stderr")?; + // open is still in git's process group in serve mode. + if let Err(what) = readers.wait(Instant::now() + self.tune.idle) { + self.terminate_and_discard(&mut child, &mut readers, &pack_dir, &before); + return Err(Error::Backend(format!( + "git exited but left its {what} open for {:?} (a helper outlived it)", + self.tune.idle + ))); + } + let (stdout, over) = readers.got_out.take().unwrap_or_default(); + let stderr = readers.got_err.take().unwrap_or_default(); if over { return Err(Error::Backend(format!( "git output exceeded {max_out} bytes (hostile or corrupt remote)" @@ -874,29 +927,40 @@ impl GitBucket { }) } - /// Wait for a pipe reader's result until `deadline`. Past it, a helper that - /// outlived git holds the pipe: stop the group (serve mode) and fail. - fn receive( + /// The one end of every breach. Stop git (its whole group in serve mode), + /// reap it, then keep watching until the pipes have closed and the pack + /// directory has stopped changing, or a short settle deadline passes (a + /// helper outside the group, in CLI mode, may still be writing), and only + /// then discard what the fetch added. Discarding first would let a surviving + /// helper write into the directory after it was cleaned. + fn terminate_and_discard( &self, child: &mut Child, - rx: &mpsc::Receiver, - deadline: Instant, - what: &str, - ) -> Result { - let left = deadline.saturating_duration_since(Instant::now()); - rx.recv_timeout(left).map_err(|_| { - self.stop(child); - Error::Backend(format!( - "git exited but left its {what} open for {:?} (a helper outlived it)", - self.tune.idle - )) - }) + readers: &mut Readers, + pack_dir: &Path, + before: &HashMap, + ) { + // Signal before reaping, while the group id is still ours. After a clean + // exit the leader is already reaped, so this relies on the group still + // having a member (a helper holding a pipe, or one still writing). + self.stop(child); + let _ = child.wait(); + let settle = Instant::now() + self.tune.idle.min(SETTLE); + let mut last = dir_bytes(pack_dir); + loop { + std::thread::sleep(Duration::from_millis(20)); + let closed = readers.wait(Instant::now()).is_ok(); + let now = dir_bytes(pack_dir); + if (closed && now == last) || Instant::now() >= settle { + break; + } + last = now; + } + self.discard_download(pack_dir, before); } - /// Kill a watched git. Called before the child is reaped, so its group id - /// cannot have been reused (after a clean exit only [`Self::receive`] calls - /// it, and only while a helper still holds a pipe, so the group is not - /// empty). In serve mode the child leads its own process + /// Kill a watched git. Called before the child is reaped where it can be, + /// so its group id cannot have been reused. In serve mode the child leads its own process /// group and the whole group is signalled, so `ssh` and `git-remote-*` /// die with it; the system `kill` keeps this std-only, with no `unsafe`, and /// serve mode on unix therefore needs a `kill` on the PATH. Without one only @@ -3198,8 +3262,10 @@ mod tests { let t = Tmp::new("silent"); let (_, url) = hub(&t); let pids = t.0.join("pids"); + // A little data first (under the budget), so the breach has + // something to discard. let body = format!( - "case \" $* \" in *\" fetch \"*) sleep 600 & echo $! > {}; wait;; esac", + "case \" $* \" in *\" fetch \"*) head -c 1000 /dev/zero > \"$GIT_DIR/objects/pack/tmp_pack_idle\"; sleep 600 & echo $! > {}; wait;; esac", pids.display() ); let sh = shim(&t, "silent.sh", &body); @@ -3208,6 +3274,10 @@ mod tests { let started = Instant::now(); let e = get_err(&b, "manifest"); let took = started.elapsed(); + assert!( + dir_files(&b.cache.join("objects").join("pack")).is_empty(), + "the no-progress breach must discard what the fetch added" + ); let helper: u32 = std::fs::read_to_string(&pids) .unwrap() .trim() @@ -3242,8 +3312,9 @@ mod tests { let t = Tmp::new("held"); let (_, url) = hub(&t); let pids = t.0.join("pids"); + // The fetch "downloads" something small and exits cleanly. let body = format!( - "case \" $* \" in *\" fetch \"*) sleep 600 & echo $! > {}; exit 0;; esac", + "case \" $* \" in *\" fetch \"*) head -c 1000 /dev/zero > \"$GIT_DIR/objects/pack/tmp_pack_held\"; sleep 600 & echo $! > {}; exit 0;; esac", pids.display() ); let sh = shim(&t, "held.sh", &body); @@ -3253,6 +3324,10 @@ mod tests { let e = get_err(&b, "manifest"); assert!(e.contains("left its"), "{e}"); assert!(started.elapsed() < Duration::from_secs(90)); + assert!( + dir_files(&b.cache.join("objects").join("pack")).is_empty(), + "the pipe-timeout breach must discard what the fetch added" + ); let helper: u32 = std::fs::read_to_string(&pids) .unwrap() .trim() @@ -3265,6 +3340,80 @@ mod tests { } } + /// git exits cleanly with its download over budget while a helper is still + /// alive (here holding the pipes): the final check must stop the helper + /// (serve mode) and discard the download, like every other breach. `poll` + /// is fixed longer than the shim runs, so git has already exited at the + /// second look. + #[cfg(unix)] + #[test] + fn a_clean_exit_over_budget_still_stops_a_helper_and_discards() { + need_git!(); + for group in [true, false] { + let t = Tmp::new("finalover"); + let (_, url) = hub(&t); + let pids = t.0.join("pids"); + let body = format!( + "case \" $* \" in *\" fetch \"*) sleep 0.4; head -c 2000000 /dev/zero > \"$GIT_DIR/objects/pack/pack-fake.pack\"; sleep 600 & echo $! > {}; exit 0;; esac", + pids.display() + ); + let sh = shim(&t, "finalover.sh", &body); + let mut b = GitBucket::open_full(&url, sh, CHUNK, None, group).unwrap(); + b.tune.budget = Some(64 << 10); + b.tune.poll = Some(Duration::from_millis(1500)); + b.tune.idle = Duration::from_secs(3); + let e = get_err(&b, "manifest"); + assert!(e.contains("65536 bytes"), "{e}"); + assert!( + dir_files(&b.cache.join("objects").join("pack")).is_empty(), + "the over-budget download must be discarded" + ); + let helper: u32 = std::fs::read_to_string(&pids) + .unwrap() + .trim() + .parse() + .unwrap(); + if !group { + kill_pid(helper); + } + wait_gone(helper); + } + } + + /// A relative serve home (`.`) must work: git runs with the cache as its + /// working directory, so a relative cache path would be resolved from inside + /// the cache. The path is relative to the current directory without + /// changing it, by climbing out to the root and down again. + #[cfg(unix)] + #[test] + fn a_relative_cache_parent_works() { + need_git!(); + let t = Tmp::new("relative"); + let (_, url) = hub(&t); + let cwd = std::env::current_dir().unwrap(); + let up = "../".repeat(cwd.components().count() - 1); + let target = t.0.join("home").join(".sc").join("tmp"); + let rel = PathBuf::from(format!( + "{up}{}", + target.strip_prefix("/").unwrap().display() + )); + assert!(rel.is_relative()); + let b = GitBucket::open_in(&url, &rel).unwrap(); + assert!(b.cache.is_absolute(), "{:?}", b.cache); + assert!(b.put_new("k", b"v").unwrap()); + let Fetched::New { bytes, .. } = b.get("k", None).unwrap() else { + panic!("the value just written must read back") + }; + assert_eq!(bytes, b"v"); + // and a non-create-only key goes through the fetch path too + let tag = b.put_if_tag("manifest", b"m", None).unwrap().unwrap(); + std::thread::sleep(FRESH + Duration::from_millis(100)); + assert!(matches!( + b.get("manifest", Some(&tag)).unwrap(), + Fetched::Unchanged + )); + } + /// Both pipes share one deadline after git exits. Here stdout is held for /// half of `idle` and stderr for 1.25 times `idle`: waiting `idle` for each /// in turn (the old behavior) would let the second one finish in time. From 1c2dd812fa8354609d17ff3f571e8b2de13ac9ad Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 16:49:35 +0200 Subject: [PATCH 6/9] fix(objio): keep measuring after git exits, and signal the group only with evidence Sol review round 2 on serve --store sc+git::. S2: the final size check ran before the pipe wait, so a helper that outlived git could cross the budget during that wait and close both pipes before the deadline, and the watchdog returned success without discarding the oversized download. After git exits it now polls objects/pack for the one deadline both pipes share, and measures once more after both pipes have closed, before it returns success. A breach there takes terminate_and_discard like every other. S3: once the leader is reaped, an empty process group's id can be reused by an unrelated group of the same user, and stop() signalled - anyway. The decision is now a small pure function, may_signal_group: free while the leader is unreaped (its pid pins the group id), and after the reap only while a pipe reader is still waiting (an open pipe has a live process holding its write end). With both pipes closed nothing is signalled; the watchdog only settles and discards. A try_wait error counts as an unknown reap state: no group signal. Tests, each failing without its fix (checked by mutation): - a_helper_that_crosses_the_budget_after_git_exits_is_caught (serve and CLI mode): the helper writes past the budget after git exited and closes its pipes. - the_group_is_signalled_only_when_its_id_is_pinned_or_a_member_is_proven (the decision's truth table). - a_reaped_leader_with_both_pipes_closed_gets_no_group_signal, with a recording kill program: no signal is sent, and the download is still discarded. - a_clean_exit_over_budget_still_stops_a_helper_and_discards now also asserts the one group signal in serve mode when a helper holds the pipes. ADR-0060 and THREAT-MODEL describe the post-exit measuring and the signalling rule. --- crates/objio/src/git.rs | 167 ++++++++++++++++++++++++++++++++++------ 1 file changed, 142 insertions(+), 25 deletions(-) diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index 9e925f70..df4ff6a0 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -456,6 +456,19 @@ fn remove_stale_locks(dir: &Path, depth: u32) { } } +/// Whether a watched git's process group may be signalled. While the leader is +/// unreaped its pid pins the group id, so the signal can only reach our own +/// group. Once the leader is reaped, an empty group's id is free to be reused by +/// an unrelated process group of the same user, so signal it only while a pipe +/// reader is still waiting: an open pipe has a live process holding its write +/// end, which is the evidence that the group still has a member. With both +/// pipes closed there is no such evidence, and the caller only settles and +/// discards. (A helper that left the group and still holds a pipe would fool +/// this; it has to have changed its own process group to do so.) +fn may_signal_group(leader_reaped: bool, a_pipe_is_open: bool) -> bool { + !leader_reaped || a_pipe_is_open +} + /// Signal every process of group `pgid` with SIGKILL through the system `kill`. /// False only when that program cannot be run; a non-zero exit (the group is /// already gone) is fine. @@ -866,7 +879,8 @@ impl GitBucket { Ok(Some(st)) => break Some(st), Ok(None) => {} Err(e) => { - self.stop(&mut child); + // Whether the leader is reaped is unknown: no group signal. + self.stop(&mut child, false); let _ = child.wait(); return Err(e.into()); } @@ -894,19 +908,31 @@ impl GitBucket { got_err: None, }; // Every breach ends in `terminate_and_discard`: the budget during the - // run, the budget at the final check, no progress, a pipe that stays open. + // run, the budget after git exited, no progress, a pipe that stays open. let Some(status) = status else { - self.terminate_and_discard(&mut child, &mut readers, &pack_dir, &before); + self.terminate_and_discard(&mut child, &mut readers, &pack_dir, &before, false); return Err(breach.expect("only a breach leaves the loop without a status")); }; - if dir_bytes(&pack_dir).saturating_sub(base) > watch.budget { - self.terminate_and_discard(&mut child, &mut readers, &pack_dir, &before); - return Err(over_budget(watch)); - } - // One deadline for both pipes. A helper that outlives git and holds one - // open is still in git's process group in serve mode. - if let Err(what) = readers.wait(Instant::now() + self.tune.idle) { - self.terminate_and_discard(&mut child, &mut readers, &pack_dir, &before); + // Git has exited and is reaped. A helper that outlived it may still be + // writing while it holds the pipes (in serve mode it is in git's group), + // so keep measuring `objects/pack` for the one deadline both pipes share, + // and once more after both have closed, before calling the fetch good. + let deadline = Instant::now() + self.tune.idle; + let held = loop { + let step = (Instant::now() + Duration::from_millis(20)).min(deadline); + let pending = readers.wait(step); + if dir_bytes(&pack_dir).saturating_sub(base) > watch.budget { + self.terminate_and_discard(&mut child, &mut readers, &pack_dir, &before, true); + return Err(over_budget(watch)); + } + match pending { + Ok(()) => break None, + Err(what) if Instant::now() >= deadline => break Some(what), + Err(_) => {} + } + }; + if let Some(what) = held { + self.terminate_and_discard(&mut child, &mut readers, &pack_dir, &before, true); return Err(Error::Backend(format!( "git exited but left its {what} open for {:?} (a helper outlived it)", self.tune.idle @@ -932,18 +958,19 @@ impl GitBucket { /// directory has stopped changing, or a short settle deadline passes (a /// helper outside the group, in CLI mode, may still be writing), and only /// then discard what the fetch added. Discarding first would let a surviving - /// helper write into the directory after it was cleaned. + /// helper write into the directory after it was cleaned. `reaped` says whether + /// the leader was already reaped; it decides whether the group may be + /// signalled ([`may_signal_group`]). fn terminate_and_discard( &self, child: &mut Child, readers: &mut Readers, pack_dir: &Path, before: &HashMap, + reaped: bool, ) { - // Signal before reaping, while the group id is still ours. After a clean - // exit the leader is already reaped, so this relies on the group still - // having a member (a helper holding a pipe, or one still writing). - self.stop(child); + let a_pipe_is_open = readers.wait(Instant::now()).is_err(); + self.stop(child, may_signal_group(reaped, a_pipe_is_open)); let _ = child.wait(); let settle = Instant::now() + self.tune.idle.min(SETTLE); let mut last = dir_bytes(pack_dir); @@ -959,15 +986,16 @@ impl GitBucket { self.discard_download(pack_dir, before); } - /// Kill a watched git. Called before the child is reaped where it can be, - /// so its group id cannot have been reused. In serve mode the child leads its own process - /// group and the whole group is signalled, so `ssh` and `git-remote-*` - /// die with it; the system `kill` keeps this std-only, with no `unsafe`, and - /// serve mode on unix therefore needs a `kill` on the PATH. Without one only - /// git itself is killed, and a warning says so once. - fn stop(&self, child: &mut Child) { + /// Kill a watched git, and with `signal_group` (see [`may_signal_group`]) its + /// whole process group in serve mode, where the child leads its own, so `ssh` + /// and `git-remote-*` die with it. The system `kill` keeps this std-only, with + /// no `unsafe`, and serve mode on unix therefore needs a `kill` on the PATH. + /// Without one only git itself is killed, and a warning says so once. + fn stop(&self, child: &mut Child, signal_group: bool) { + #[cfg(not(unix))] + let _ = signal_group; // no process groups: only the child is killed #[cfg(unix)] - if self.group && !kill_group(self.tune.kill, child.id()) { + if self.group && signal_group && !kill_group(self.tune.kill, child.id()) { warn_once( &KILL_WARNED, &format!( @@ -1925,9 +1953,15 @@ mod tests { #[cfg(unix)] fn shim(t: &Tmp, name: &str, body: &str) -> String { + shim_script(t, name, &format!("{body}\nexec git \"$@\"\n")) + } + + /// An executable `/bin/sh` script with exactly `body`. + #[cfg(unix)] + fn shim_script(t: &Tmp, name: &str, body: &str) -> String { use std::os::unix::fs::PermissionsExt; let p = t.0.join(name); - std::fs::write(&p, format!("#!/bin/sh\n{body}\nexec git \"$@\"\n")).unwrap(); + std::fs::write(&p, format!("#!/bin/sh\n{body}")).unwrap(); std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap(); p.to_str().unwrap().to_string() } @@ -3359,6 +3393,8 @@ mod tests { ); let sh = shim(&t, "finalover.sh", &body); let mut b = GitBucket::open_full(&url, sh, CHUNK, None, group).unwrap(); + let (kill, killed) = recording_kill(&t); + b.tune.kill = kill; b.tune.budget = Some(64 << 10); b.tune.poll = Some(Duration::from_millis(1500)); b.tune.idle = Duration::from_secs(3); @@ -3368,6 +3404,10 @@ mod tests { dir_files(&b.cache.join("objects").join("pack")).is_empty(), "the over-budget download must be discarded" ); + // The leader was reaped but a helper held the pipes: that is the + // evidence the group is alive, so serve mode signals it, once. + let signals = std::fs::read_to_string(&killed).unwrap_or_default(); + assert_eq!(signals.lines().count(), usize::from(group), "{signals:?}"); let helper: u32 = std::fs::read_to_string(&pids) .unwrap() .trim() @@ -3380,6 +3420,83 @@ mod tests { } } + /// A `kill` stand-in that records its arguments and then really kills: + /// returns the program path and the log it appends to. + #[cfg(unix)] + fn recording_kill(t: &Tmp) -> (&'static str, PathBuf) { + let log = t.0.join("kill.log"); + let sh = shim_script( + t, + "killrec.sh", + &format!("echo \"$@\" >> {}\nexec kill \"$@\"\n", log.display()), + ); + (Box::leak(sh.into_boxed_str()), log) + } + + /// Once git is reaped and both pipes are closed there is no evidence the + /// process group still has a member, so it must not be signalled: its id may + /// have been reused. The fetch is over budget here and is still discarded. + #[cfg(unix)] + #[test] + fn a_reaped_leader_with_both_pipes_closed_gets_no_group_signal() { + need_git!(); + let t = Tmp::new("nosignal"); + let (_, url) = hub(&t); + // No helper: the shim writes too much late, then exits, closing the pipes. + let body = "case \" $* \" in *\" fetch \"*) sleep 1; head -c 2000000 /dev/zero > \"$GIT_DIR/objects/pack/pack-fake.pack\"; exit 0;; esac"; + let sh = shim(&t, "nosignal.sh", body); + let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); + let (kill, killed) = recording_kill(&t); + b.tune.kill = kill; + b.tune.budget = Some(64 << 10); + // git has exited at the second look, so the breach comes after the reap. + b.tune.poll = Some(Duration::from_secs(3)); + let e = get_err(&b, "manifest"); + assert!(e.contains("65536 bytes"), "{e}"); + assert!( + dir_files(&b.cache.join("objects").join("pack")).is_empty(), + "the over-budget download must be discarded" + ); + assert!(!killed.exists(), "no group signal without a live member"); + } + + /// The signalling decision: free while the leader's pid pins the group id, + /// afterwards only with an open pipe as evidence of a live member. + #[test] + fn the_group_is_signalled_only_when_its_id_is_pinned_or_a_member_is_proven() { + assert!( + may_signal_group(false, false), + "unreaped leader pins the id" + ); + assert!(may_signal_group(false, true)); + assert!(may_signal_group(true, true), "an open pipe proves a member"); + assert!(!may_signal_group(true, false), "reaped and nothing proven"); + } + + /// A helper that outlives git can cross the budget after git has exited and + /// close both pipes before the shared deadline. The watchdog must still + /// catch it, and must not call the fetch good. + #[cfg(unix)] + #[test] + fn a_helper_that_crosses_the_budget_after_git_exits_is_caught() { + need_git!(); + for group in [true, false] { + let t = Tmp::new("late"); + let (_, url) = hub(&t); + // git exits at once; a helper holds the pipes, then writes too much and exits. + let body = "case \" $* \" in *\" fetch \"*) ( sleep 0.6; head -c 2000000 /dev/zero > \"$GIT_DIR/objects/pack/pack-late.pack\" ) & exit 0;; esac"; + let sh = shim(&t, "late.sh", body); + let mut b = GitBucket::open_full(&url, sh, CHUNK, None, group).unwrap(); + b.tune.budget = Some(64 << 10); + let e = get_err(&b, "manifest"); + assert!(e.contains("65536 bytes"), "{e}"); + assert!( + dir_files(&b.cache.join("objects").join("pack")).is_empty(), + "the late download must be discarded" + ); + } + } + /// A relative serve home (`.`) must work: git runs with the cache as its /// working directory, so a relative cache path would be resolved from inside /// the cache. The path is relative to the current directory without From 4dd4580a24805901ec171ac5d0168345c9000774 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 16:59:44 +0200 Subject: [PATCH 7/9] fix(objio): never signal a git process group after its leader is reaped Sol review round 3 on serve --store sc+git::. An open pipe proves a live group member only at the instant it is looked at, and the external `kill` runs later, so once the leader is reaped any group signal can hit a reused group id of an unrelated process of the same user. Holding the pid without reaping needs waitid(WNOWAIT), which is unsafe and not allowed here. may_signal_group now takes only the leader's state and returns true only while the leader is unreaped (its pid pins the group id); after the reap the group is never signalled, whatever the pipes say. A breach found after the reap still settles for a bounded time and discards what the fetch added. The cost is stated in the function's doc comment, ADR-0060, THREAT-MODEL, CLAUDE.md and ARCHITECTURE.md: a helper that outlives git's exit (a smart-HTTP v0/v1 helper chain, an ssh ControlPersist master, a hostile process) is not killed in any mode and may keep writing after the discard until it exits. Tests, each failing without the fix (checked by mutation): - the_group_is_signalled_only_while_the_leader_is_unreaped (truth table). - a_reaped_leader_gets_no_group_signal_even_with_a_pipe_open, with a recording kill program, for a helper that holds the pipes and for none; the download is discarded in both. - a_clean_exit_over_budget_discards_but_leaves_the_helper_alone: no group signal after a clean exit, the helper survives, the download is discarded. - a_helper_holding_the_pipes_after_exit_is_an_error: the helper now outlives the call in serve mode too. --- crates/objio/src/git.rs | 161 ++++++++++++++++++++++------------------ 1 file changed, 90 insertions(+), 71 deletions(-) diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index df4ff6a0..dd79839e 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -456,17 +456,21 @@ fn remove_stale_locks(dir: &Path, depth: u32) { } } -/// Whether a watched git's process group may be signalled. While the leader is -/// unreaped its pid pins the group id, so the signal can only reach our own -/// group. Once the leader is reaped, an empty group's id is free to be reused by -/// an unrelated process group of the same user, so signal it only while a pipe -/// reader is still waiting: an open pipe has a live process holding its write -/// end, which is the evidence that the group still has a member. With both -/// pipes closed there is no such evidence, and the caller only settles and -/// discards. (A helper that left the group and still holds a pipe would fool -/// this; it has to have changed its own process group to do so.) -fn may_signal_group(leader_reaped: bool, a_pipe_is_open: bool) -> bool { - !leader_reaped || a_pipe_is_open +/// Whether a watched git's process group may be signalled: only while the leader +/// is unreaped. Its pid pins the group id then, so the signal can only reach our +/// own group. Once the leader is reaped, the group's id is free to be reused by +/// an unrelated process group of the same user, and nothing observable can show +/// it has not been: an open pipe proves a live member only at the instant it is +/// looked at, and the external `kill` runs later. Holding the pid without +/// reaping it needs `waitid(WNOWAIT)`, which is `unsafe`, so after the reap the +/// group is never signalled, whatever the pipes say. +/// +/// The cost: a helper that outlives git's exit (a smart-HTTP v0/v1 helper chain, +/// an ssh `ControlPersist` master, a hostile process) is not killed and can keep +/// writing after the download was discarded, until it exits. The watchdog then +/// only settles for a bounded time and discards what the fetch added. +fn may_signal_group(leader_reaped: bool) -> bool { + !leader_reaped } /// Signal every process of group `pgid` with SIGKILL through the system `kill`. @@ -953,14 +957,14 @@ impl GitBucket { }) } - /// The one end of every breach. Stop git (its whole group in serve mode), - /// reap it, then keep watching until the pipes have closed and the pack - /// directory has stopped changing, or a short settle deadline passes (a - /// helper outside the group, in CLI mode, may still be writing), and only - /// then discard what the fetch added. Discarding first would let a surviving - /// helper write into the directory after it was cleaned. `reaped` says whether - /// the leader was already reaped; it decides whether the group may be - /// signalled ([`may_signal_group`]). + /// The one end of every breach. Stop git (its whole group in serve mode, but + /// only while the leader is unreaped: [`may_signal_group`]), reap it, then + /// keep watching until the pipes have closed and the pack directory has + /// stopped changing, or a short settle deadline passes, and only then discard + /// what the fetch added. Discarding first would let a surviving helper write + /// into the directory after it was cleaned. `reaped` says whether the leader + /// was already reaped when the breach was found. A helper that outlives that + /// reap is not killed and may write again after the discard. fn terminate_and_discard( &self, child: &mut Child, @@ -969,8 +973,7 @@ impl GitBucket { before: &HashMap, reaped: bool, ) { - let a_pipe_is_open = readers.wait(Instant::now()).is_err(); - self.stop(child, may_signal_group(reaped, a_pipe_is_open)); + self.stop(child, may_signal_group(reaped)); let _ = child.wait(); let settle = Instant::now() + self.tune.idle.min(SETTLE); let mut last = dir_bytes(pack_dir); @@ -3335,9 +3338,9 @@ mod tests { } /// A git that exits cleanly while a helper keeps its pipes open is an error - /// within `idle`, not a hang. In serve mode the helper is still in git's - /// process group, and the group is stopped before the error returns, so it - /// does not outlive the call; in CLI mode only git was ever its to kill. + /// within `idle`, not a hang, and what the fetch added is discarded. git is + /// already reaped, so the group is not signalled in either mode: the helper + /// outlives the call (the test kills it), which is the documented cost. #[cfg(unix)] #[test] fn a_helper_holding_the_pipes_after_exit_is_an_error() { @@ -3367,21 +3370,20 @@ mod tests { .trim() .parse() .unwrap(); - if !group { - kill_pid(helper); - } + kill_pid(helper); wait_gone(helper); } } /// git exits cleanly with its download over budget while a helper is still - /// alive (here holding the pipes): the final check must stop the helper - /// (serve mode) and discard the download, like every other breach. `poll` - /// is fixed longer than the shim runs, so git has already exited at the - /// second look. + /// alive (here holding the pipes): the download is discarded like for every + /// other breach, but the group is not signalled, because git is already + /// reaped and its id could have been reused. The helper survives (and is + /// killed here by the test). `poll` is fixed longer than the shim runs, so + /// git has already exited at the second look. #[cfg(unix)] #[test] - fn a_clean_exit_over_budget_still_stops_a_helper_and_discards() { + fn a_clean_exit_over_budget_discards_but_leaves_the_helper_alone() { need_git!(); for group in [true, false] { let t = Tmp::new("finalover"); @@ -3404,18 +3406,19 @@ mod tests { dir_files(&b.cache.join("objects").join("pack")).is_empty(), "the over-budget download must be discarded" ); - // The leader was reaped but a helper held the pipes: that is the - // evidence the group is alive, so serve mode signals it, once. - let signals = std::fs::read_to_string(&killed).unwrap_or_default(); - assert_eq!(signals.lines().count(), usize::from(group), "{signals:?}"); + assert!(!killed.exists(), "no group signal after the leader's reap"); let helper: u32 = std::fs::read_to_string(&pids) .unwrap() .trim() .parse() .unwrap(); - if !group { - kill_pid(helper); - } + let alive = Command::new("ps") + .args(["-o", "stat=", "-p", &helper.to_string()]) + .output() + .unwrap(); + let stat = String::from_utf8_lossy(&alive.stdout).trim().to_string(); + assert!(!stat.is_empty() && !stat.starts_with('Z'), "stat {stat:?}"); + kill_pid(helper); wait_gone(helper); } } @@ -3433,44 +3436,60 @@ mod tests { (Box::leak(sh.into_boxed_str()), log) } - /// Once git is reaped and both pipes are closed there is no evidence the - /// process group still has a member, so it must not be signalled: its id may - /// have been reused. The fetch is over budget here and is still discarded. + /// Once git is reaped the group is never signalled, whether a pipe is still + /// open (a helper holds it) or both are closed: its id may have been reused, + /// and an open pipe proves nothing by the time the external `kill` runs. The + /// fetch is over budget in both cases and is still discarded. #[cfg(unix)] #[test] - fn a_reaped_leader_with_both_pipes_closed_gets_no_group_signal() { + fn a_reaped_leader_gets_no_group_signal_even_with_a_pipe_open() { need_git!(); - let t = Tmp::new("nosignal"); - let (_, url) = hub(&t); - // No helper: the shim writes too much late, then exits, closing the pipes. - let body = "case \" $* \" in *\" fetch \"*) sleep 1; head -c 2000000 /dev/zero > \"$GIT_DIR/objects/pack/pack-fake.pack\"; exit 0;; esac"; - let sh = shim(&t, "nosignal.sh", body); - let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); - let (kill, killed) = recording_kill(&t); - b.tune.kill = kill; - b.tune.budget = Some(64 << 10); - // git has exited at the second look, so the breach comes after the reap. - b.tune.poll = Some(Duration::from_secs(3)); - let e = get_err(&b, "manifest"); - assert!(e.contains("65536 bytes"), "{e}"); - assert!( - dir_files(&b.cache.join("objects").join("pack")).is_empty(), - "the over-budget download must be discarded" - ); - assert!(!killed.exists(), "no group signal without a live member"); + for helper_holds_pipes in [true, false] { + let t = Tmp::new("nosignal"); + let (_, url) = hub(&t); + let pids = t.0.join("pids"); + let helper = if helper_holds_pipes { + format!("sleep 600 & echo $! > {};", pids.display()) + } else { + String::new() + }; + // The shim writes too much late, then exits; with a helper the pipes stay open. + let body = format!( + "case \" $* \" in *\" fetch \"*) sleep 1; head -c 2000000 /dev/zero > \"$GIT_DIR/objects/pack/pack-fake.pack\"; {helper} exit 0;; esac" + ); + let sh = shim(&t, "nosignal.sh", &body); + let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); + let (kill, killed) = recording_kill(&t); + b.tune.kill = kill; + b.tune.budget = Some(64 << 10); + b.tune.idle = Duration::from_secs(3); + // git has exited at the second look, so the breach comes after the reap. + b.tune.poll = Some(Duration::from_secs(3)); + let e = get_err(&b, "manifest"); + assert!(e.contains("65536 bytes"), "{e}"); + assert!( + dir_files(&b.cache.join("objects").join("pack")).is_empty(), + "the over-budget download must be discarded" + ); + assert!(!killed.exists(), "no group signal after the reap"); + if helper_holds_pipes { + let pid: u32 = std::fs::read_to_string(&pids) + .unwrap() + .trim() + .parse() + .unwrap(); + kill_pid(pid); + wait_gone(pid); + } + } } - /// The signalling decision: free while the leader's pid pins the group id, - /// afterwards only with an open pipe as evidence of a live member. + /// The signalling decision: only while the leader is unreaped, when its pid + /// pins the group id. After the reap, never. #[test] - fn the_group_is_signalled_only_when_its_id_is_pinned_or_a_member_is_proven() { - assert!( - may_signal_group(false, false), - "unreaped leader pins the id" - ); - assert!(may_signal_group(false, true)); - assert!(may_signal_group(true, true), "an open pipe proves a member"); - assert!(!may_signal_group(true, false), "reaped and nothing proven"); + fn the_group_is_signalled_only_while_the_leader_is_unreaped() { + assert!(may_signal_group(false), "an unreaped leader pins the id"); + assert!(!may_signal_group(true), "a reaped leader pins nothing"); } /// A helper that outlives git can cross the budget after git has exited and From 11018b424eb4e1edafc9daad5183002f13faaa52 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 17:04:49 +0200 Subject: [PATCH 8/9] docs: serve --store sc+git::, the filtered read and the watchdog, with compaction Merges PR 2's docs and demo with PR 1's compaction on sc+git:: (#370). Both now hold: `sc remote compact` works on a carrier, `sc serve --store sc+git::` works behind a server-side private-branch gate, the manifest read is one filtered fetch, and every fetch runs under a download watchdog. ADR-0060 gains "Serving a Git store", the fetch-based read with its abbreviation guard, and the watchdog with its exact limits: the group is signalled only while git is unreaped, so a helper that outlives git's exit is not killed and may keep writing after the discard. The planned fix, a group-leading `sh` supervisor, is noted there and as ROADMAP follow-on 10. THREAT-MODEL, CLAUDE.md, ARCHITECTURE.md, README and the sc-cli-reference skill carry the same facts. ROADMAP's `sc+git::` follow-ons mark (2) and (9) built and the manifest-read part of (7) done, next to PR 1's list. The carrier demo keeps PR 1's compaction step (4) and adds the serve step (7), the traced filtered-read step (8) and the zero-residue step (9), on the bare repository and sc-forge legs. --- .claude/skills/sc-cli-reference/SKILL.md | 18 +- ARCHITECTURE.md | 21 ++- CLAUDE.md | 32 +++- README.md | 5 +- ROADMAP.md | 35 ++-- demo/run_git_carrier_demo.sh | 146 ++++++++++++++++- docs/THREAT-MODEL.md | 76 ++++++++- docs/adr/0060-git-carried-bucket-remotes.md | 172 +++++++++++++++++++- 8 files changed, 454 insertions(+), 51 deletions(-) diff --git a/.claude/skills/sc-cli-reference/SKILL.md b/.claude/skills/sc-cli-reference/SKILL.md index 6ea2bec0..dbef5a7b 100644 --- a/.claude/skills/sc-cli-reference/SKILL.md +++ b/.claude/skills/sc-cli-reference/SKILL.md @@ -431,8 +431,8 @@ cargo run --bin sc -- remote add sc+git:: [--allow-private-branc # `git ls-remote` lists them; the same repo can also # hold a `--git` mirror of public branches. Anyone # with push to the repo holds bucket put + delete; - # deleting a ref is not erasure. `serve --store` - # refuses it in v1; `remote compact` works (below); + # deleting a ref is not erasure. `remote compact` + # and `serve --store` work (below); # clone --filter refused as for any bucket. Untested # against real GitHub cargo run --bin sc -- remote compact [--dry-run] [--prune-expire 24h] [--allow-short-grace] @@ -468,6 +468,20 @@ cargo run --bin sc -- serve --http --store sc+wal://|sc+s3:// # all content lives in the bucket (instance is # disposable); still needs `.sc/` for tokens, # the TLS identity and scratch space +cargo run --bin sc -- serve --http --store sc+git:: [--allow-private-branches] + # the same in front of a Git repository (ADR-0060): + # git must authenticate WITHOUT prompting (ssh agent, + # key without passphrase, credential helper). Without + # --allow-private-branches (refused for any other + # store) the server refuses every private-branch + # object and a ref at a manifest before anything + # reaches the host; only the operator can lift it. + # The git cache lives under /.sc/tmp/, locked + # while a session lives; a start reaps caches whose + # lock is free (unix, one host, not NFS). Every + # fetch runs under a download watchdog (disk per + # fetch; 120 s no-progress deadline); serve mode + # needs a `kill` on the PATH to kill git's group bash demo/run_git_carrier_demo.sh # sc+git:: proof (P39): a bare Git repo as the host; # protected path + secret + private branch pushed # (private branch refused without the opt-in), plain diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d0585364..d915eecf 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1372,10 +1372,15 @@ length. Git is the system binary (`SC_GIT` overrides) with a scrubbed environmen transport allowlist (`https`, `http`, `ssh`, `file`; `ext`, `fd` and `git` forced off), a URL allowlist and no userinfo on `http(s)` URLs (credentials go through git's credential helper); a private bare repo in `sc-git-bucket--` caches fetched -objects and is removed on `Drop`. Scaling: every `ls-remote` and push downloads the -whole ref advertisement (the host does not filter it server-side), and a cold read fetches -every `log/` and `checkpoints/` entry behind the last checkpoint; refs grow by at least three -per push until `sc remote compact` runs. +objects and is removed on `Drop`. A mutable key (the manifest) is read with one `git fetch` +of exactly `refs/sc/K` under protocol v2, so a v2 host lists that ref alone, and every fetch +runs under a watchdog that kills git past its download budget (disk bytes per fetch: a +key's value cap, 2 GiB for a family) or after 120 s without progress (only serve mode on +unix kills git's whole process group, through the system `kill`, and only while git is +unreaped; elsewhere, and for any helper that outlives git's exit, the helpers that write the +pack may keep writing). Scaling: `ls-remote` and push download the whole ref advertisement, +and a cold read fetches every `log/` and `checkpoints/` entry behind the last checkpoint; +refs grow by at least three per push until `sc remote compact` runs. `sc remote compact` works on a carrier (ADR-0057 addendum). A Git host reports no trustworthy object age (committer time is writer-controlled), so the compactor keeps its @@ -1394,8 +1399,12 @@ A private branch goes to an `sc+git::` remote only under the per-remote opt-in `--allow-private-branches` (`sc remote set --allow-private-branches` for an existing remote), since its name, recipient ids and sizes become readable by every reader of the repository and the ciphertext cannot be taken back. -`sc serve --store` (a daemon cannot answer prompts, and it would bypass the opt-in) -refuses an `sc+git::` store in v1. +`sc serve --store sc+git::` serves a carrier, and the opt-in moves to the operator +(`--allow-private-branches`): without it `BucketTransport` refuses objects of the +sealed-branch kinds in `PutPack` and `PutObject` before anything is uploaded or staged, and +an `UpdateRef` to a manifest, so no private-branch ciphertext reaches the Git host through +the server (the daemon needs non-interactive git auth, and keeps its git cache under the +serve home's `.sc/tmp/`). sc-forge accepts exactly the `refs/sc/` prefix (ADR-0058 addendum). The dependency rule is amended: `objio` may spawn the system `git`, and only `gitio::bridge` and objio's git backend spawn it; `gix` stays in `gitio`. Proven by `crates/cli/tests/git_bucket.rs`, diff --git a/CLAUDE.md b/CLAUDE.md index 6bc8beb4..339a2d0f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -164,7 +164,7 @@ the code, those win. | P37c | Merge, cherry-pick and rebase on a partial clone: a tree-recursive engine (`merge_trees`) carries a subtree by id when the sides agree and no protection rule applies, reads only where they differ or a rule intersects, and on a lazy, online clone fills what it asks for by id outside the store lock (a non-lazy clone gets the gap error with `sc backfill `); a carried protected subtree keeps the complete input wrap union; a conflict inside the view is marked and resolved like on a full clone (`sc conflicts`/`sc resolve`/`merge --abort`/`sc commit`; the out-of-view part is grafted from the decided root), while a conflict outside the view, or a result that must be sealed outside it, is refused before any ref, merge-state or working-tree change (fetched objects stay). A subtree identical on all three sides is carried even under a protection rule. Cherry-pick and rebase use the same engine (`replay_commit`): the fold's accumulator may hold absent subtrees, the view is filled before the final write, and a stopped pick or rebase keeps its carried subtrees in the decided root. Mode-only edits now merge and a file/directory collision fails before any write, on full clones too. | [0049](docs/adr/0049-merge-on-partial-clones.md) | | P37d | Agent sessions on a partial clone: `sc ws fork`/`run`/`harvest`/`abandon` and `sc work` work; a workspace or agent sees the host's sparse view only (`sc work` no longer gets a complete tree there), the view closure is filled (lazy, online) or refused with the gap error before anything is written, a workspace commit grafts the out-of-view subtrees from the base without reading them, and harvest lands through the ADR-0049 merge (any conflict falls back to a `work-` branch as on a full clone) | [0050](docs/adr/0050-sessions-on-partial-clones.md) | | P38 | Gitea-compatible forge over bare Git and smart HTTP: recorded API subset, scoped auth, protected merges, maintenance and container operations; 24 recorded scenarios and six first-consumer acceptance scenarios pass. Hosted repos have none of the `.sc/` features; SSH and Actions remain outside v1. | [0058](docs/adr/0058-sc-forge-gitea-compatible-git-forge.md) | -| P39 | `sc+git::` remotes: the bucket WAL stored as one parentless commit per key at `refs/sc/` in any Git repository (system `git` via `objio::GitBucket`, lease CAS, no `+` refspec), so private branches, protected paths and secrets travel through a Git host (local bare repo, sc-forge; GitHub untested); private-branch push needs per-remote `--allow-private-branches`; `remote compact` works on it (first-sighting age record, leased deletes; ADR-0057 addendum) while `serve --store` refuses it in v1; sc-forge accepts `refs/sc/` | [0060](docs/adr/0060-git-carried-bucket-remotes.md) | +| P39 | `sc+git::` remotes: the bucket WAL stored as one parentless commit per key at `refs/sc/` in any Git repository (system `git` via `objio::GitBucket`, lease CAS, no `+` refspec), so private branches, protected paths and secrets travel through a Git host (local bare repo, sc-forge; GitHub untested); private-branch push needs per-remote `--allow-private-branches`; `remote compact` works on it (first-sighting age record, leased deletes; ADR-0057 addendum); `sc serve --store sc+git::` serves it and refuses private-branch objects unless the operator passes `--allow-private-branches`, and every fetch runs under a download watchdog; sc-forge accepts `refs/sc/` | [0060](docs/adr/0060-git-carried-bucket-remotes.md) | ## Standing boundaries & gotchas @@ -228,7 +228,7 @@ transport-adjacent. The rest, imperatively: error on it and it never reaches the P18 bridge. Pushing a private branch needs the per-remote opt-in `--allow-private-branches` (`remote add`/`clone`; enforced in `repo`, refuses before any upload); protected paths and secrets - need none. `sc serve --store` refuses it (deferred: no prompts in a daemon). + need none. `sc serve --store` serves it behind a server-side gate (next bullet). `sc remote compact` works on it with its own age source (a Git host has no trustworthy object age): a first-sighting record at `refs/sc/compact/sightings` (` `, generation-aware, strict @@ -245,7 +245,33 @@ transport-adjacent. The rest, imperatively: and forks may keep objects by id). Public content is plaintext, and private branch names, recipient ids and sizes are readable by every reader of the repository. A killed process can leave `sc-git-bucket--` in the - temp dir. Not tested against real GitHub. + temp dir (`sc serve` keeps it under the serve home's `.sc/tmp/`, locks + `/.owner.lock` for the bucket's life, and reaps at startup the caches + whose lock is free, unix). Not tested against real GitHub. +- **`sc serve --store sc+git::` (ADR-0060) refuses private-branch objects + unless the operator passes `--allow-private-branches`** (refused on any other + store). The gate is `BucketTransport.refuse_sealed`, set by `open_for_serve`: + `put_pack` stops at the first object of a sealed-branch kind (`Sealed`, + `Manifest`, `transport::is_sealed_branch_object`) before `upload_pack`, + `put_object` refuses before staging, and `update_ref` refuses a manifest tip + even when it already sits on the chain; the error is `InvalidArgument` + (`EC_OTHER`, no wire change) and a client cannot lift it. The daemon needs + non-interactive git auth. A mutable-key read is one `git fetch` of exactly + `refs/sc/` under protocol v2 (the host filters the advertisement; negative + refspecs stop git's abbreviation rules from letting a branch or tag stand in + for the key). Every fetch runs under a watchdog (`fetch.unpackLimit=1`, the + growth of `objects/pack` polled from 2 ms backing off to 50 ms): killed over the + budget (the key's value cap plus 0.1% and 1 MiB, 2 GiB for a family) or after + 120 s with no progress, what it added removed. The bound is disk per fetch, not + RAM inside `index-pack`, not the total cache, and not the refs a family fetch + writes. Only serve mode on unix bounds disk against a hostile HTTP host: it runs + git in its own process group and kills the group through the system `kill`, + but only while git is unreaped (after the reap the group id may have been + reused, so it is never signalled; `may_signal_group`). Without a `kill` it + warns once and kills git alone. The CLI and Windows kill git alone, and the + helpers that write the pack may keep writing; so may any helper that outlives + git's exit, in any mode, until it exits. Only fetches are watched; `ls-remote` + and push are not. - **`DirBucket` (`sc+wal://`) is a single-host, single-owner backend (#264).** Its CAS takes an OS advisory file lock (`File::lock`: `flock` on unix, `LockFileEx` on Windows) on `/.cas-lock`, a file that is never removed and diff --git a/README.md b/README.md index d57a0298..8f5a6ae7 100644 --- a/README.md +++ b/README.md @@ -143,8 +143,9 @@ branches, protected paths and secrets travel through a Git host that a plain `git clone` never sees into (ADR-0060). Pushing a private branch there needs `--allow-private-branches` (its name and recipient ids become readable by everyone who can read the repository). `sc remote compact` works on it (run it -from one host; the first run only records when it first saw each object); -`serve --store` is not supported on it yet. Proven by +from one host; the first run only records when it first saw each object). +`sc serve --store sc+git::` serves it and refuses private branches +unless the operator passes `--allow-private-branches`. Proven by `demo/run_git_carrier_demo.sh`; not tested against GitHub itself. ## How Phase 1 works diff --git a/ROADMAP.md b/ROADMAP.md index 2d8b7a6d..6221deea 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -704,10 +704,11 @@ across every phase. `refs/sc/*`, and the same repository can also mirror public branches through a `--git` remote. A private-branch push needs the per-remote opt-in `--allow-private-branches`; `sc remote compact` works on `sc+git::` (a - first-sighting age record, leased deletes; ADR-0057 addendum) while - `sc serve --store` refuses it in v1; sc-forge accepts the `refs/sc/` prefix. No - format, wire or crypto change. Proven by `demo/run_git_carrier_demo.sh`. - (ADR-0060.) + first-sighting age record, leased deletes; ADR-0057 addendum); `sc serve --store + sc+git::` serves it behind a server-side private-branch gate (the operator's + `--allow-private-branches`) and every fetch runs under a download watchdog; + sc-forge accepts the `refs/sc/` prefix. No format, wire or crypto change. Proven + by `demo/run_git_carrier_demo.sh`. (ADR-0060.) ## Active @@ -1028,16 +1029,22 @@ P28 review follow-ons, the port de-flake, the clippy allow-list) are recorded in come from the idx; `list_meta` itself still downloads every listed value); (7) in part: compaction folds small packs and deletes what nothing names, which bounds the ref growth of at least three per push and the log/checkpoint - entries a cold read fetches, but between runs every `ls-remote` and push still - downloads the whole ref advertisement (the host does not filter it by prefix - server-side, a packet trace showed the full list). Still deferred: (2) - `sc serve --store sc+git::...` (a daemon cannot answer ssh or credential - prompts, and it would sit between clients and the private-branch opt-in); (4) - validation against real GitHub (per-ref and per-push limits, rulesets, hosts - that refuse unknown ref namespaces), untested so far; (9) a download budget for - fetches into the temp cache (`git fetch` has no input-size limit, so an - oversized value is written before the read caps refuse it). Declined, with - reasons: (3) rollback and rewind of `refs/sc/manifest` by anyone with push + entries a cold read fetches, and the manifest read is one `git fetch` that a + protocol v2 host answers with that ref alone, but between runs `ls-remote` and + push still download the whole ref advertisement; (2) `sc serve --store + sc+git::...`, which refuses private-branch objects unless the operator passes + `--allow-private-branches` (the daemon needs non-interactive git auth); (9) a + download budget: every fetch into the cache runs under a watchdog that kills + git over its budget (disk bytes per fetch; not RAM inside `index-pack`, not the + total cache). Still deferred: (4) validation against real GitHub (per-ref and + per-push limits, rulesets, hosts that refuse unknown ref namespaces), untested + so far; (10) a group-leading supervisor for the serve-mode watchdog: run git + under a std-only `sh` leader (`sh -c 'git "$@"; echo $? > ; read + _' sh `, stdin piped from sc) so the process group id stays pinned after + git exits, and the watchdog can group-kill a helper that outlives git (the + post-exit gap ADR-0060 documents: a smart-HTTP v0/v1 helper chain, an ssh + `ControlPersist` master) before it closes the leader's stdin and reaps it. + Declined, with reasons: (3) rollback and rewind of `refs/sc/manifest` by anyone with push access is the cross-cutting signed-ref effort already tracked under P34 follow-on (6) and ADR-0059 follow-on (2), now reachable by more people; a client-side "highest manifest seq seen" floor was reviewed and judged a false diff --git a/demo/run_git_carrier_demo.sh b/demo/run_git_carrier_demo.sh index bf3df49b..a80f98b4 100755 --- a/demo/run_git_carrier_demo.sh +++ b/demo/run_git_carrier_demo.sh @@ -23,8 +23,16 @@ # plaintext, no embargoed file, and cannot read the secret value. # 6. KEYED: an sc clone with alice's identity reads the protected file, the # secret and the embargoed branch. -# 7. Zero residue: no leftover `sc-git-bucket-*` temp dirs (TMPDIR is private -# to each run, so any leftover is ours) and no .sc/tmp residue. +# 7. SERVE: `sc serve --http --store sc+git::` in front of the same host. +# A public push and a keyless clone work through it; a private push is +# refused by the SERVER (no private-branch object reaches the host) until +# the operator restarts it with --allow-private-branches. +# 8. FILTERED READ: every manifest fetch sc makes (traced through an SC_GIT +# shim with GIT_TRACE_PACKET) sends `ref-prefix refs/sc/manifest`, and the +# host lists that one ref, not its branches or the other refs/sc/* keys. +# 9. Zero residue: no leftover `sc-git-bucket-*` temp dirs (TMPDIR is private +# to each run, so any leftover is ours), nothing in the serve home's +# .sc/tmp and no .sc/tmp residue. # # Self-checking: every claim is an assertion; any failure exits non-zero before # the RESULT line. Run twice to prove repeatability. The forge leg needs @@ -53,7 +61,14 @@ fail() { echo "FAIL: $1"; exit 1; } WORK="" FORGE_PID="" +SERVE_PID="" +SERVE_ADDR="" cleanup() { + if [ -n "$SERVE_PID" ]; then + kill "$SERVE_PID" 2>/dev/null || true + wait "$SERVE_PID" 2>/dev/null || true + SERVE_PID="" + fi if [ -n "$FORGE_PID" ]; then kill "$FORGE_PID" 2>/dev/null || true wait "$FORGE_PID" 2>/dev/null || true @@ -75,6 +90,46 @@ no_cache_residue() { fi } +# start_serve [extra sc serve flags]: runs `sc serve --http` +# on a free loopback port in the background; sets SERVE_PID and SERVE_ADDR. +start_serve() { + local home="$1" store="$2" i + shift 2 + : > "$WORK/serve.out" + "$SC" serve --http 127.0.0.1:0 --store "$store" "$@" "$home" \ + > "$WORK/serve.out" 2> "$WORK/serve.err" & + SERVE_PID=$! + SERVE_ADDR="" + for i in $(seq 1 100); do + SERVE_ADDR=$(sed -n 's/^listening on //p' "$WORK/serve.out" | head -1) + [ -n "$SERVE_ADDR" ] && return 0 + kill -0 "$SERVE_PID" 2>/dev/null || { cat "$WORK/serve.err"; fail "sc serve exited"; } + sleep 0.1 + done + fail "sc serve did not announce an address" +} + +# wait_serve_idle : every session removes its Git cache when it ends, but +# the client can return before the server thread has finished; wait for that so +# killing the server cannot leave a cache behind (a killed server's leftovers +# are only reaped by the next start). +wait_serve_idle() { + local i + for i in $(seq 1 100); do + [ -z "$(ls -A "$1/.sc/tmp" 2>/dev/null)" ] && return 0 + sleep 0.1 + done + fail "the server's sessions did not clean up under $1/.sc/tmp" +} + +stop_serve() { + if [ -n "$SERVE_PID" ]; then + kill "$SERVE_PID" 2>/dev/null || true + wait "$SERVE_PID" 2>/dev/null || true + SERVE_PID="" + fi +} + # scenario