From fab473e76fb3dbc13856a9380c0d54a75c07cf75 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sun, 22 Nov 2026 10:34:40 +0200 Subject: [PATCH 1/9] feat(objio): run serve-mode fetches under a group-leading supervisor In serve mode on unix the fetch watchdog now spawns /bin/sh as the leader of git's process group instead of git itself. The shell runs git in a subshell, writes git's exit code to /.fetch-status-, closes its own stdout and stderr, and blocks reading a stdin pipe only sc holds. When sc closes that pipe (or dies) the shell kills its whole group with the builtin kill. sc never waits for the leader before its group is dead: git's exit comes from the status file (strict parse), so the leader's pid keeps pinning the group id. Every end of a fetch, normal or breach, drops the pipe, signals the group once through the system kill as a backup while the leader is unreaped, reaps the leader within a bound, then settles and measures. A helper that stays in git's group, including one that outlives git, can no longer write after the discard. A helper that leaves the group (setsid) still escapes. Exit status 126 or 127 maps to the "cannot run git" error; a signalled git reads as 128 + n. The CLI and non-unix builds are unchanged. Tests rewrite the old "no group signal after git exits" cases for the new rule and add: a quiet writer that outlives git is dead and stops growing the pack directory, a mid-run breach kills the group with no kill program, exit codes propagate, 126/127, a leader that dies early, and the strict status parse. --- crates/objio/src/git.rs | 730 +++++++++++++++++++++++----- crates/repo/src/bucket_transport.rs | 5 +- 2 files changed, 611 insertions(+), 124 deletions(-) diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index dfe2050b..e7b2d37d 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -70,6 +70,82 @@ const FAMILY_BUDGET: u64 = MAX_VALUE; /// open, pack directory still changing) before its download is discarded anyway. const SETTLE: Duration = Duration::from_secs(5); +/// The serve-mode group leader: `/bin/sh -c LEADER sc-fetch +/// `. It runs git in a subshell with stdin from `/dev/null` and writes +/// git's exit code to the status file, drops its own stdout and stderr so only +/// git and its helpers hold the pipes, then blocks reading its stdin, which only +/// sc holds. When sc closes that pipe (or dies) it kills its whole group, itself +/// included, with the builtin `kill`: no group id and no external program are +/// involved. The parentheses matter: a `{ ...; } &` group lets bash exec the +/// command in place and the status line never runs. `command` skips shell +/// functions, so an exported `BASH_FUNC_*` cannot stand in for git. +#[cfg_attr(not(unix), allow(dead_code))] +const LEADER: &str = "s=$1; shift; ( command \"$@\" \"$s\" ) & exec >/dev/null 2>&1; read _; kill -s KILL 0"; + +/// The leader's status file in the private cache directory, removed on drop so +/// every exit path of a fetch cleans it up. +#[cfg_attr(not(unix), allow(dead_code))] +struct StatusFile(PathBuf); + +#[cfg_attr(not(unix), allow(dead_code))] +impl StatusFile { + /// A fresh path `/.fetch-status-`; the leader creates the file. + fn new(cache: &Path) -> StatusFile { + use std::sync::atomic::{AtomicU64, Ordering}; + static SEQ: AtomicU64 = AtomicU64::new(0); + let n = SEQ.fetch_add(1, Ordering::Relaxed); + let p = cache.join(format!(".fetch-status-{n}")); + let _ = std::fs::remove_file(&p); + StatusFile(p) + } + + /// git's exit code once the leader has written it, else `None`. + fn read(&self) -> Option { + let mut buf = Vec::new(); + std::fs::File::open(&self.0) + .ok()? + .take(8) + .read_to_end(&mut buf) + .ok()?; + parse_status(&buf) + } +} + +impl Drop for StatusFile { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +/// An exit code from the status file: one to three ASCII digits and a newline, +/// at most 255. Anything else (empty, partial, longer, signed) is not yet a +/// status. +#[cfg_attr(not(unix), allow(dead_code))] +fn parse_status(b: &[u8]) -> Option { + let d = b.strip_suffix(b"\n")?; + if d.is_empty() || d.len() > 3 || !d.iter().all(u8::is_ascii_digit) { + return None; + } + let n: i32 = std::str::from_utf8(d).ok()?.parse().ok()?; + (n <= 255).then_some(n) +} + +/// How a watched git ended. +struct Exit { + success: bool, + code: Option, +} + +/// An exit code for error text. A git killed by signal `n` reports `128 + n` +/// through the serve-mode leader and no code at all when run directly. +fn describe_code(code: Option) -> String { + match code { + None => "signal".to_string(), + Some(c) if c > 128 => format!("{c}, signal {} if it was killed", c - 128), + Some(c) => c.to_string(), + } +} + /// `-c` overrides on every git call: transport allowlist, no hooks, no signing, /// no background maintenance, bounded stalls. `protocol.version=2` makes a /// fetch send `ref-prefix`, so the host filters its ref advertisement down to @@ -229,8 +305,8 @@ struct Tuning { /// Fixed polling interval; `None` polls after 2 ms and backs off to 50 ms /// (tests fix it to make a fetch finish between two polls). poll: Option, - /// The program that signals a process group in serve mode (tests point it - /// at a missing one). + /// The program that signals a process group in serve mode, as a backup to + /// the supervisor's own kill (tests point it at a missing one). #[cfg(unix)] kill: &'static str, } @@ -309,8 +385,10 @@ pub struct GitBucket { /// life. It is how [`GitBucket::reap_stale`] tells a live cache from one a /// killed process left behind (the OS drops the lock with its holder). owner: Option, - /// Serve mode: every fetch leads its own process group, so the watchdog - /// can kill `ssh` and `git-remote-*` along with git. Off for the CLI, where + /// Serve mode: every fetch runs under a `/bin/sh` supervisor ([`LEADER`]) + /// that leads its own process group and kills it when the fetch ends, so + /// `ssh`, `git-remote-*` and any helper that outlives git die with it. Off for + /// the CLI, where /// the terminal's Ctrl-C must still reach git and ssh must still be able to /// prompt on the controlling terminal. #[cfg(unix)] @@ -474,15 +552,17 @@ fn remove_stale_locks(dir: &Path, depth: u32) { /// is unreaped. Its pid pins the group id then, so the signal can only reach our /// own group. Once the leader is reaped, the group's id is free to be reused by /// an unrelated process group of the same user, and nothing observable can show -/// it has not been: an open pipe proves a live member only at the instant it is -/// looked at, and the external `kill` runs later. Holding the pid without -/// reaping it needs `waitid(WNOWAIT)`, which is `unsafe`, so after the reap the -/// group is never signalled, whatever the pipes say. +/// it has not been. Holding the pid without reaping it needs `waitid(WNOWAIT)`, +/// which is `unsafe`, so after the reap the group is never signalled. /// -/// The cost: a helper that outlives git's exit (a smart-HTTP v0/v1 helper chain, -/// an ssh `ControlPersist` master, a hostile process) is not killed and can keep -/// writing after the download was discarded, until it exits. The watchdog then -/// only settles for a bounded time and discards what the fetch added. +/// In serve mode on unix the leader is the `/bin/sh` supervisor ([`LEADER`]), +/// and sc never reaps it before the group is dead: git's exit is learned from +/// the status file, not from `wait`. So the rule is always met until the one +/// final reap, and every helper that stays in git's process group, including +/// one that outlives git, is killed. The signal here is the backup to the +/// leader's own `kill 0`, which needs no group id and no external program. A +/// process that leaves the group (`setsid`: an ssh `ControlPersist` master, a +/// credential-cache daemon) escapes both. fn may_signal_group(leader_reaped: bool) -> bool { !leader_reaped } @@ -702,9 +782,10 @@ impl GitBucket { /// [`GitBucket::open`] for a long-running server (`sc serve --store`): the /// cache directory is made under `parent` (the serve home's `.sc/tmp/`, created - /// if missing) instead of the system temp directory, and every fetch - /// leads its own process group so the watchdog, when it kills one, takes `ssh` or - /// `git-remote-https` with it (which needs a `kill` program on the PATH). The + /// if missing) instead of the system temp directory, and on unix every fetch + /// runs under a `/bin/sh` supervisor that leads its own process group, which + /// is killed when the fetch ends, so `ssh`, `git-remote-https` and any helper + /// that stays in the group die with git (a system `kill` is only a backup). The /// bucket holds a lock on `/.owner.lock` until it is dropped. A killed /// server can leave the cache behind; [`GitBucket::reap_stale`] removes such /// leftovers at the next start. @@ -797,7 +878,27 @@ impl GitBucket { /// The git command line in the cache repository with the hardened /// environment, stdout and stderr piped. fn command(&self, args: &[&str], env: &[(&str, &str)], piped_stdin: bool) -> Command { - let mut cmd = Command::new(&self.git); + self.hardened(Command::new(&self.git), args, env, piped_stdin) + } + + /// Whether watched fetches run under the group-leading supervisor. + fn leads(&self) -> bool { + #[cfg(unix)] + return self.group; + #[cfg(not(unix))] + false + } + + /// `cmd` (git, or the `/bin/sh` leader that has the git program already + /// among its arguments) with the `-c` overrides, `args`, the cache as + /// working directory and the scrubbed environment. + fn hardened( + &self, + mut cmd: Command, + args: &[&str], + env: &[(&str, &str)], + piped_stdin: bool, + ) -> Command { for c in GIT_CONFIG { cmd.arg("-c").arg(c); } @@ -910,12 +1011,26 @@ impl GitBucket { let pack_dir = self.cache.join("objects").join("pack"); let before = dir_files(&pack_dir); let base: u64 = before.values().sum(); - let mut cmd = self.command(args, &[], false); + // Serve mode on unix: `/bin/sh` leads git's group and holds its stdin + // pipe ([`LEADER`]); git's exit comes from the status file, because the + // leader must stay unreaped until its group is dead. + let status_file = self.leads().then(|| StatusFile::new(&self.cache)); + let mut cmd = match &status_file { + Some(sf) => { + let mut sh = Command::new("/bin/sh"); + sh.args(["-c", LEADER, "sc-fetch"]) + .arg(&sf.0) + .arg(&self.git); + self.hardened(sh, args, &[], true) + } + None => self.command(args, &[], false), + }; #[cfg(unix)] - if self.group { + if status_file.is_some() { std::os::unix::process::CommandExt::process_group(&mut cmd, 0); } let mut child = self.spawn(&mut cmd)?; + let mut lead_in = child.stdin.take(); let mut sout = child.stdout.take().expect("piped"); let mut serr = child.stderr.take().expect("piped"); // Detached readers: a descendant that outlives a killed git and keeps a @@ -932,15 +1047,31 @@ impl GitBucket { let (mut last, mut moved) = (0u64, Instant::now()); let mut pause = self.tune.poll.unwrap_or(Duration::from_millis(2)); let mut breach = None; - let status = loop { - match child.try_wait() { - Ok(Some(st)) => break Some(st), - Ok(None) => {} - Err(e) => { - // Whether the leader is reaped is unknown: no group signal. - self.stop(&mut child, false); - let _ = child.wait(); - return Err(e.into()); + let exit = loop { + if let Some(sf) = &status_file { + // Never `wait` for the leader here: a reaped leader no longer + // pins its group's id. + if let Some(code) = sf.read() { + break Some(Exit { + success: code == 0, + code: Some(code), + }); + } + } else { + match child.try_wait() { + Ok(Some(st)) => { + break Some(Exit { + success: st.success(), + code: st.code(), + }) + } + Ok(None) => {} + Err(e) => { + // Whether the leader is reaped is unknown: no group signal. + self.stop(&mut child, &mut lead_in, false); + let _ = child.wait(); + return Err(e.into()); + } } } let grown = dir_bytes(&pack_dir).saturating_sub(base); @@ -967,20 +1098,41 @@ impl GitBucket { }; // Every breach ends in `terminate_and_discard`: the budget during the // run, the budget after git exited, no progress, a pipe that stays open. - let Some(status) = status else { - self.terminate_and_discard(&mut child, &mut readers, &pack_dir, &before, false); - return Err(breach.expect("only a breach leaves the loop without a status")); + let Some(exit) = exit else { + self.terminate_and_discard( + &mut child, + &mut lead_in, + &mut readers, + &pack_dir, + &before, + false, + ); + return Err(breach.expect("only a breach leaves the loop without an exit")); }; - // Git has exited and is reaped. A helper that outlived it may still be - // writing while it holds the pipes (in serve mode it is in git's group), - // so keep measuring `objects/pack` for the one deadline both pipes share, - // and once more after both have closed, before calling the fetch good. + if status_file.is_some() { + // Git is done and the leader is still unreaped: whatever is left in + // its group outlived git, so the group always ends here, the same + // way as on a breach. + self.terminate(&mut child, &mut lead_in, &mut readers, &pack_dir, false); + } + // Git has exited and the leader is reaped. A holder of the pipes that + // left the group (or, in the CLI, a helper that outlived git) may still + // be writing, so keep measuring `objects/pack` for the one deadline both + // pipes share, and once more after both have closed, before calling the + // fetch good. let deadline = Instant::now() + self.tune.idle; let held = loop { let step = (Instant::now() + Duration::from_millis(20)).min(deadline); let pending = readers.wait(step); if dir_bytes(&pack_dir).saturating_sub(base) > watch.budget { - self.terminate_and_discard(&mut child, &mut readers, &pack_dir, &before, true); + self.terminate_and_discard( + &mut child, + &mut lead_in, + &mut readers, + &pack_dir, + &before, + true, + ); return Err(over_budget(watch)); } match pending { @@ -990,12 +1142,26 @@ impl GitBucket { } }; if let Some(what) = held { - self.terminate_and_discard(&mut child, &mut readers, &pack_dir, &before, true); + self.terminate_and_discard( + &mut child, + &mut lead_in, + &mut readers, + &pack_dir, + &before, + true, + ); return Err(Error::Backend(format!( "git exited but left its {what} open for {:?} (a helper outlived it)", self.tune.idle ))); } + // `command` reports a git it cannot find or start as 127 or 126. + if let (Some(_), Some(c @ (126 | 127))) = (&status_file, exit.code) { + return Err(Error::Backend(format!( + "cannot run git ({}): the shell exited with status {c} (install git or set SC_GIT)", + self.git + ))); + } let (stdout, over) = readers.got_out.take().unwrap_or_default(); let stderr = readers.got_err.take().unwrap_or_default(); if over { @@ -1004,30 +1170,45 @@ impl GitBucket { ))); } Ok(Out { - success: status.success(), - code: status.code(), + success: exit.success, + code: exit.code, stdout, stderr: self.scrub(&String::from_utf8_lossy(&stderr)), }) } - /// The one end of every breach. Stop git (its whole group in serve mode, but - /// only while the leader is unreaped: [`may_signal_group`]), reap it, then - /// keep watching until the pipes have closed and the pack directory has - /// stopped changing, or a short settle deadline passes, and only then discard - /// what the fetch added. Discarding first would let a surviving helper write - /// into the directory after it was cleaned. `reaped` says whether the leader - /// was already reaped when the breach was found. A helper that outlives that - /// reap is not killed and may write again after the discard. + /// The one end of every breach. [`Self::terminate`], then discard what the + /// fetch added, only after the group has been given time to die and stop + /// writing: discarding first would let a surviving helper write into the + /// directory after it was cleaned. `reaped` says whether the leader was + /// already reaped when the breach was found (the CLI, after git exited, and + /// serve mode after the group ended). fn terminate_and_discard( &self, child: &mut Child, + lead_in: &mut Option, readers: &mut Readers, pack_dir: &Path, before: &HashMap, reaped: bool, ) { - self.stop(child, may_signal_group(reaped)); + self.terminate(child, lead_in, readers, pack_dir, reaped); + self.discard_download(pack_dir, before); + } + + /// Stop git (its whole group in serve mode, but only while the leader is + /// unreaped: [`may_signal_group`]), reap it, then keep watching until the + /// pipes have closed and the pack directory has stopped changing, or a short + /// settle deadline passes. + fn terminate( + &self, + child: &mut Child, + lead_in: &mut Option, + readers: &mut Readers, + pack_dir: &Path, + reaped: bool, + ) { + self.stop(child, lead_in, may_signal_group(reaped)); let _ = child.wait(); let settle = Instant::now() + self.tune.idle.min(SETTLE); let mut last = dir_bytes(pack_dir); @@ -1040,28 +1221,44 @@ impl GitBucket { } last = now; } - self.discard_download(pack_dir, before); } - /// Kill a watched git, and with `signal_group` (see [`may_signal_group`]) its - /// whole process group in serve mode, where the child leads its own, so `ssh` - /// and `git-remote-*` die with it. The system `kill` keeps this std-only, with - /// no `unsafe`, and serve mode on unix therefore needs a `kill` on the PATH. - /// Without one only git itself is killed, and a warning says so once. - fn stop(&self, child: &mut Child, signal_group: bool) { + /// Stop a watched git. In serve mode (the child is the [`LEADER`]) closing + /// its stdin makes it kill the whole group, itself included, with no help + /// from outside; with `signal_group` (see [`may_signal_group`]) the system + /// `kill` also signals the group as a backup, which still works when the + /// leader died early. That backup is the only external program, and without + /// it a warning is printed once. The leader is then reaped within a bound. + /// Without a leader (the CLI) git is just killed. + fn stop( + &self, + child: &mut Child, + lead_in: &mut Option, + signal_group: bool, + ) { #[cfg(not(unix))] - let _ = signal_group; // no process groups: only the child is killed + let _ = (&lead_in, signal_group); // no process groups: only the child is killed #[cfg(unix)] - if self.group && signal_group && !kill_group(self.tune.kill, child.id()) { - warn_once( - &KILL_WARNED, - &format!( - "sc: cannot run `{}` to stop a fetch's process group; only git itself is \ - killed, so ssh or git-remote-https may keep running (put a `kill` \ - program on the PATH)", - self.tune.kill - ), - ); + if self.group { + drop(lead_in.take()); + if signal_group && !kill_group(self.tune.kill, child.id()) { + warn_once( + &KILL_WARNED, + &format!( + "sc: cannot run `{}` to signal a fetch's process group; the fetch's \ + supervisor still kills it, but a helper may survive a supervisor that \ + died early (put a `kill` program on the PATH)", + self.tune.kill + ), + ); + } + let end = Instant::now() + SETTLE; + while Instant::now() < end { + if !matches!(child.try_wait(), Ok(None)) { + return; + } + std::thread::sleep(Duration::from_millis(2)); + } } let _ = child.kill(); } @@ -1111,7 +1308,7 @@ impl GitBucket { fn fail(&self, what: &str, out: &Out) -> Error { Error::Backend(format!( "git {what} failed (exit {}): {}", - out.code.map_or("signal".to_string(), |c| c.to_string()), + describe_code(out.code), out.stderr )) } @@ -1267,7 +1464,7 @@ impl GitBucket { .unwrap_or_default(); Ok(Pushed::Refused(format!( "git push failed (exit {}): {} {}", - out.code.map_or("signal".to_string(), |c| c.to_string()), + describe_code(out.code), out.stderr, first ))) @@ -2119,7 +2316,7 @@ impl Bucket for GitBucket { .unwrap_or_default(); let refusal = format!( "git push failed (exit {}): {} {}", - out.code.map_or("signal".to_string(), |c| c.to_string()), + describe_code(out.code), out.stderr, first ); @@ -3904,21 +4101,42 @@ mod tests { } } - /// A git that exits cleanly while a helper keeps its pipes open is an error - /// within `idle`, not a hang, and what the fetch added is discarded. git is - /// already reaped, so the group is not signalled in either mode: the helper - /// outlives the call (the test kills it), which is the documented cost. + /// Whether `perl` can run `setsid`, which is how a test shim leaves its + /// process group (macOS has no `setsid` program). + #[cfg(unix)] + fn have_setsid_perl() -> bool { + Command::new("perl") + .args(["-MPOSIX", "-e", "POSIX::setsid(); exit 0"]) + .status() + .is_ok_and(|s| s.success()) + } + + /// A git that exits cleanly while something keeps its pipes open is an error + /// within `idle`, not a hang, and what the fetch added is discarded. In the + /// CLI that is any helper. In serve mode the group is killed when git exits, + /// so only a process that left the group (`setsid`, like an ssh + /// `ControlPersist` master) can still hold them, and it outlives the call: + /// the documented escape (the test kills it). #[cfg(unix)] #[test] - fn a_helper_holding_the_pipes_after_exit_is_an_error() { + fn a_holder_of_the_pipes_that_outlives_git_is_an_error() { need_git!(); for group in [true, false] { + if group && !have_setsid_perl() { + continue; + } let t = Tmp::new("held"); let (_, url) = hub(&t); let pids = t.0.join("pids"); - // The fetch "downloads" something small and exits cleanly. + let sleeper = if group { + "perl -MPOSIX -e 'POSIX::setsid(); exec @ARGV' sleep 600" + } else { + "sleep 600" + }; + // The fetch "downloads" something small and exits cleanly, once the + // helper has had time to leave the group. let body = format!( - "case \" $* \" in *\" fetch \"*) head -c 1000 /dev/zero > \"$GIT_DIR/objects/pack/tmp_pack_held\"; sleep 600 & echo $! > {}; exit 0;; esac", + "case \" $* \" in *\" fetch \"*) head -c 1000 /dev/zero > \"$GIT_DIR/objects/pack/tmp_pack_held\"; {sleeper} & echo $! > {}; sleep 0.5; exit 0;; esac", pids.display() ); let sh = shim(&t, "held.sh", &body); @@ -3932,6 +4150,7 @@ mod tests { dir_files(&b.cache.join("objects").join("pack")).is_empty(), "the pipe-timeout breach must discard what the fetch added" ); + assert!(no_status_files(&b)); let helper: u32 = std::fs::read_to_string(&pids) .unwrap() .trim() @@ -3942,15 +4161,25 @@ mod tests { } } + /// No `.fetch-status-*` file is left in the cache. + #[cfg(unix)] + fn no_status_files(b: &GitBucket) -> bool { + std::fs::read_dir(&b.cache).unwrap().flatten().all(|e| { + !e.file_name() + .to_string_lossy() + .starts_with(".fetch-status-") + }) + } + /// git exits cleanly with its download over budget while a helper is still /// alive (here holding the pipes): the download is discarded like for every - /// other breach, but the group is not signalled, because git is already - /// reaped and its id could have been reused. The helper survives (and is - /// killed here by the test). `poll` is fixed longer than the shim runs, so - /// git has already exited at the second look. + /// other breach. In serve mode the helper is in git's group, which is killed + /// when git exits, so it is dead afterwards; in the CLI nothing signals it + /// and it survives (the test kills it). `poll` is fixed longer than the shim + /// runs, so git has already exited at the second look. #[cfg(unix)] #[test] - fn a_clean_exit_over_budget_discards_but_leaves_the_helper_alone() { + fn a_clean_exit_over_budget_discards_and_serve_mode_kills_the_helper() { need_git!(); for group in [true, false] { let t = Tmp::new("finalover"); @@ -3973,20 +4202,29 @@ mod tests { dir_files(&b.cache.join("objects").join("pack")).is_empty(), "the over-budget download must be discarded" ); - assert!(!killed.exists(), "no group signal after the leader's reap"); + assert!(no_status_files(&b)); let helper: u32 = std::fs::read_to_string(&pids) .unwrap() .trim() .parse() .unwrap(); - let alive = Command::new("ps") - .args(["-o", "stat=", "-p", &helper.to_string()]) - .output() - .unwrap(); - let stat = String::from_utf8_lossy(&alive.stdout).trim().to_string(); - assert!(!stat.is_empty() && !stat.starts_with('Z'), "stat {stat:?}"); - kill_pid(helper); - wait_gone(helper); + if group { + assert!( + killed.exists(), + "serve mode signals the group at git's exit" + ); + wait_gone(helper); + } else { + assert!(!killed.exists(), "the CLI never signals a group"); + let alive = Command::new("ps") + .args(["-o", "stat=", "-p", &helper.to_string()]) + .output() + .unwrap(); + let stat = String::from_utf8_lossy(&alive.stdout).trim().to_string(); + assert!(!stat.is_empty() && !stat.starts_with('Z'), "stat {stat:?}"); + kill_pid(helper); + wait_gone(helper); + } } } @@ -4003,13 +4241,13 @@ mod tests { (Box::leak(sh.into_boxed_str()), log) } - /// Once git is reaped the group is never signalled, whether a pipe is still - /// open (a helper holds it) or both are closed: its id may have been reused, - /// and an open pipe proves nothing by the time the external `kill` runs. The - /// fetch is over budget in both cases and is still discarded. + /// The group is signalled once per fetch, while the leader is unreaped: at + /// git's exit. The fetch is then found over budget after the leader's reap, + /// and that breach must not signal the group again (its id may be reused), + /// whether a pipe is still held or not. The download is still discarded. #[cfg(unix)] #[test] - fn a_reaped_leader_gets_no_group_signal_even_with_a_pipe_open() { + fn the_group_is_signalled_once_and_never_after_the_leader_is_reaped() { need_git!(); for helper_holds_pipes in [true, false] { let t = Tmp::new("nosignal"); @@ -4030,7 +4268,7 @@ mod tests { b.tune.kill = kill; b.tune.budget = Some(64 << 10); b.tune.idle = Duration::from_secs(3); - // git has exited at the second look, so the breach comes after the reap. + // git has exited at the second look, so the breach comes after it. b.tune.poll = Some(Duration::from_secs(3)); let e = get_err(&b, "manifest"); assert!(e.contains("65536 bytes"), "{e}"); @@ -4038,14 +4276,18 @@ mod tests { dir_files(&b.cache.join("objects").join("pack")).is_empty(), "the over-budget download must be discarded" ); - assert!(!killed.exists(), "no group signal after the reap"); + let calls = std::fs::read_to_string(&killed).unwrap_or_default(); + assert_eq!( + calls.lines().count(), + 1, + "one signal while the leader is unreaped, none after: {calls:?}" + ); if helper_holds_pipes { let pid: u32 = std::fs::read_to_string(&pids) .unwrap() .trim() .parse() .unwrap(); - kill_pid(pid); wait_gone(pid); } } @@ -4059,27 +4301,279 @@ mod tests { assert!(!may_signal_group(true), "a reaped leader pins nothing"); } - /// A helper that outlives git can cross the budget after git has exited and - /// close both pipes before the shared deadline. The watchdog must still - /// catch it, and must not call the fetch good. + /// In the CLI a helper that outlives git can cross the budget after git has + /// exited and close both pipes before the shared deadline. The watchdog must + /// still catch it, and must not call the fetch good. #[cfg(unix)] #[test] fn a_helper_that_crosses_the_budget_after_git_exits_is_caught() { + need_git!(); + let t = Tmp::new("late"); + let (_, url) = hub(&t); + // git exits at once; a helper holds the pipes, then writes too much and exits. + let body = "case \" $* \" in *\" fetch \"*) ( sleep 0.6; head -c 2000000 /dev/zero > \"$GIT_DIR/objects/pack/pack-late.pack\" ) & exit 0;; esac"; + let sh = shim(&t, "late.sh", body); + let mut b = GitBucket::open_full(&url, sh, CHUNK, None, false).unwrap(); + b.tune.budget = Some(64 << 10); + let e = get_err(&b, "manifest"); + assert!(e.contains("65536 bytes"), "{e}"); + assert!( + dir_files(&b.cache.join("objects").join("pack")).is_empty(), + "the late download must be discarded" + ); + } + + /// One `fetch` of `refs/sc/manifest` through the watchdog, for tests that + /// look at its raw result. + #[cfg(unix)] + fn watched_fetch(b: &GitBucket) -> (Result, bool) { + b.fetch_flagged( + &["refs/sc/manifest:refs/sc/manifest".to_string()], + false, + 1 << 20, + "manifest".to_string(), + ) + } + + /// Serve mode kills every process still in git's group when git exits, even + /// one that closed its pipes and keeps writing into `objects/pack`: it is dead + /// afterwards and the directory stops growing. The download is discarded only + /// where it crossed the budget (the second case: git exits at once, a fast + /// writer crosses the budget before the first poll that sees the exit). + #[cfg(unix)] + #[test] + fn a_helper_that_closes_its_pipes_and_keeps_writing_is_dead_after_git_exits() { + need_git!(); + for (budget, crosses) in [(32u64 << 20, false), (1 << 20, true)] { + let t = Tmp::new("quietwriter"); + let (_, url) = hub(&t); + let pids = t.0.join("pids"); + let (pause, hold) = if crosses { + ("0.01", "") + } else { + ("0.05", "sleep 0.4;") + }; + let body = format!( + "case \" $* \" in *\" fetch \"*) ( exec >/dev/null 2>&1; while :; do head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/pack-late.pack\"; sleep {pause}; done ) & echo $! > {}; {hold} exit 0;; esac", + pids.display() + ); + let sh = shim(&t, "quiet.sh", &body); + let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); + let (kill, killed) = recording_kill(&t); + b.tune.kill = kill; + b.tune.budget = Some(budget); + if crosses { + b.tune.poll = Some(Duration::from_millis(1500)); + } + let pack = b.cache.join("objects").join("pack"); + let (out, over) = watched_fetch(&b); + let helper: u32 = std::fs::read_to_string(&pids) + .unwrap() + .trim() + .parse() + .unwrap(); + wait_gone(helper); + let after_call = dir_bytes(&pack); + std::thread::sleep(Duration::from_millis(400)); + assert_eq!(dir_bytes(&pack), after_call, "the writer kept going"); + assert!(no_status_files(&b)); + // The group is ended by git's exit, not left to the pipe closing when + // the call returns: the backup signal went out exactly once. + let calls = std::fs::read_to_string(&killed).unwrap_or_default(); + assert_eq!(calls.lines().count(), 1, "{calls:?}"); + if crosses { + assert!(over && out.is_err(), "crossed the budget: an error"); + assert!(dir_files(&pack).is_empty(), "and the download is discarded"); + } else { + let out = out.unwrap(); + assert!(out.success && out.code == Some(0)); + assert!(after_call > 0, "within the budget nothing is discarded"); + } + } + } + + /// The group dies on a mid-run breach with no external `kill` at all: closing + /// the leader's stdin is what ends it. + #[cfg(unix)] + #[test] + fn a_runaway_fetch_dies_without_a_kill_program() { + need_git!(); + let t = Tmp::new("nokillrun"); + let (_, url) = hub(&t); + let pids = t.0.join("pids"); + let body = format!( + "case \" $* \" in *\" fetch \"*) sleep 600 & echo $! > {}; while :; do head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/tmp_pack_x\"; sleep 0.01; done;; esac", + pids.display() + ); + let sh = shim(&t, "runaway2.sh", &body); + let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); + b.tune.kill = "/no/such/kill"; + b.tune.budget = Some(1 << 20); + let e = get_err(&b, "manifest"); + assert!(e.contains("1048576 bytes"), "{e}"); + let helper: u32 = std::fs::read_to_string(&pids) + .unwrap() + .trim() + .parse() + .unwrap(); + wait_gone(helper); + let tmp_pack = b.cache.join("objects").join("pack").join("tmp_pack_x"); + std::thread::sleep(Duration::from_millis(300)); + assert!(!tmp_pack.exists(), "the writing loop must be dead too"); + assert!(no_status_files(&b)); + } + + /// git's exit code reaches the caller through the leader (a failing fetch is + /// `!success` with that code), and a git killed by a signal reads as 128 + n. + /// The CLI reports the same codes, and no code for a signal. + #[cfg(unix)] + #[test] + fn the_exit_code_of_git_is_propagated() { need_git!(); for group in [true, false] { - let t = Tmp::new("late"); + for (body, success, code) in [ + ("exit 0", true, Some(0)), + ("exit 1", false, Some(1)), + ("exit 3", false, Some(3)), + ("exit 128", false, Some(128)), + ("exit 255", false, Some(255)), + ("kill -s KILL $$", false, group.then_some(137)), + ] { + let t = Tmp::new("exitcode"); + let (_, url) = hub(&t); + let script = format!("case \" $* \" in *\" fetch \"*) {body};; esac"); + let sh = shim(&t, "exit.sh", &script); + let b = GitBucket::open_full(&url, sh, CHUNK, None, group).unwrap(); + let (out, over) = watched_fetch(&b); + let out = out.unwrap(); + assert!(!over); + assert_eq!( + (out.success, out.code), + (success, code), + "group {group}: {body}" + ); + assert!(no_status_files(&b)); + } + } + assert_eq!(describe_code(Some(137)), "137, signal 9 if it was killed"); + assert_eq!(describe_code(Some(3)), "3"); + assert_eq!(describe_code(None), "signal"); + } + + /// `command` exits 127 for a git it cannot find and 126 for one it cannot + /// run: the same "cannot run git" error the CLI gets from `spawn`. The status + /// file is removed. + #[cfg(unix)] + #[test] + fn a_git_the_shell_cannot_run_is_the_cannot_run_error() { + need_git!(); + use std::os::unix::fs::PermissionsExt; + for (what, status) in [("missing", "127"), ("not executable", "126")] { + let t = Tmp::new("cannotrun"); let (_, url) = hub(&t); - // git exits at once; a helper holds the pipes, then writes too much and exits. - let body = "case \" $* \" in *\" fetch \"*) ( sleep 0.6; head -c 2000000 /dev/zero > \"$GIT_DIR/objects/pack/pack-late.pack\" ) & exit 0;; esac"; - let sh = shim(&t, "late.sh", body); - let mut b = GitBucket::open_full(&url, sh, CHUNK, None, group).unwrap(); - b.tune.budget = Some(64 << 10); - let e = get_err(&b, "manifest"); - assert!(e.contains("65536 bytes"), "{e}"); + let sh = shim(&t, "gone.sh", ""); + let b = GitBucket::open_full(&url, sh.clone(), CHUNK, None, true).unwrap(); + if what == "missing" { + std::fs::remove_file(&sh).unwrap(); + } else { + std::fs::set_permissions(&sh, std::fs::Permissions::from_mode(0o644)).unwrap(); + } + let started = Instant::now(); + let (out, over) = watched_fetch(&b); + let e = out.err().expect("the fetch cannot run").to_string(); + assert!(!over); assert!( - dir_files(&b.cache.join("objects").join("pack")).is_empty(), - "the late download must be discarded" + e.contains("cannot run git") && e.contains("SC_GIT") && e.contains(status), + "{what}: {e}" + ); + assert!(started.elapsed() < Duration::from_secs(30)); + assert!(no_status_files(&b)); + } + // A shim that exits 127 by itself reads the same way. + let t = Tmp::new("exit127"); + let (_, url) = hub(&t); + let sh = shim( + &t, + "e127.sh", + "case \" $* \" in *\" fetch \"*) exit 127;; esac", + ); + let b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); + let e = watched_fetch(&b).0.err().unwrap().to_string(); + assert!(e.contains("cannot run git"), "{e}"); + } + + /// A leader that dies on its own (here a shim that kills the supervisor and + /// the subshell holding git's status line) never produces a status. That is + /// a bounded error through the no-progress deadline, and the group is still + /// killed from outside while the dead leader is unreaped. With no `kill` + /// program that backup is gone and the helper survives, the documented cost. + #[cfg(unix)] + #[test] + fn a_leader_that_dies_early_is_a_bounded_error() { + need_git!(); + for with_kill in [true, false] { + let t = Tmp::new("leaderdies"); + let (_, url) = hub(&t); + let pids = t.0.join("pids"); + let body = format!( + "case \" $* \" in *\" fetch \"*) l=$(ps -o pgid= -p $$ | tr -d ' '); kill -s KILL $PPID $l; sleep 600 & echo $! > {}; wait;; esac", + pids.display() ); + let sh = shim(&t, "orphan.sh", &body); + let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); + b.tune.idle = Duration::from_millis(800); + if !with_kill { + b.tune.kill = "/no/such/kill"; + } + let started = Instant::now(); + let e = get_err(&b, "manifest"); + assert!(e.contains("no progress"), "{e}"); + assert!(started.elapsed() < Duration::from_secs(30)); + assert!(no_status_files(&b)); + let helper: u32 = std::fs::read_to_string(&pids) + .unwrap() + .trim() + .parse() + .unwrap(); + if with_kill { + wait_gone(helper); + } else { + let alive = Command::new("ps") + .args(["-o", "stat=", "-p", &helper.to_string()]) + .output() + .unwrap(); + let stat = String::from_utf8_lossy(&alive.stdout).trim().to_string(); + assert!(!stat.is_empty() && !stat.starts_with('Z'), "stat {stat:?}"); + kill_pid(helper); + wait_gone(helper); + } + } + } + + /// The status line is read strictly: one to three digits and a newline, at + /// most 255. Anything else is not yet a status. + #[test] + fn the_status_line_is_parsed_strictly() { + for (text, want) in [ + ("0\n", Some(0)), + ("7\n", Some(7)), + ("128\n", Some(128)), + ("255\n", Some(255)), + ("007\n", Some(7)), + ("", None), + ("\n", None), + ("1", None), + ("12", None), + ("256\n", None), + ("1000\n", None), + ("-1\n", None), + ("+1\n", None), + ("1 \n", None), + ("1\n2\n", None), + ("1\r\n", None), + ("x\n", None), + ] { + assert_eq!(parse_status(text.as_bytes()), want, "{text:?}"); } } @@ -4134,11 +4628,12 @@ mod tests { assert!(e.contains("left its stderr open"), "{e}"); } - /// Without a `kill` program serve mode cannot signal the group. It falls back - /// to killing git alone, returns, and warns once instead of staying silent. + /// A missing `kill` program is reported once, and only costs the backup + /// signal: the leader still kills the group on its own, so a fetch that + /// stalls returns and its helper is dead. #[cfg(unix)] #[test] - fn a_missing_kill_program_falls_back_to_killing_git_and_warns_once() { + fn a_missing_kill_program_warns_once_and_the_leader_still_kills_the_group() { need_git!(); let flag = std::sync::atomic::AtomicBool::new(false); assert!(warn_once(&flag, "first"), "the first report prints"); @@ -4163,20 +4658,11 @@ mod tests { let e = get_err(&b, "manifest"); assert!(e.contains("no progress"), "{e}"); assert!(started.elapsed() < Duration::from_secs(90)); - // Only git was killed: the helper outlives the call, which is the - // documented cost of a missing `kill`. let helper: u32 = std::fs::read_to_string(&pids) .unwrap() .trim() .parse() .unwrap(); - let alive = Command::new("ps") - .args(["-o", "stat=", "-p", &helper.to_string()]) - .output() - .unwrap(); - let stat = String::from_utf8_lossy(&alive.stdout).trim().to_string(); - assert!(!stat.is_empty() && !stat.starts_with('Z'), "stat {stat:?}"); - kill_pid(helper); wait_gone(helper); } diff --git a/crates/repo/src/bucket_transport.rs b/crates/repo/src/bucket_transport.rs index dcf3a329..eb1407cd 100644 --- a/crates/repo/src/bucket_transport.rs +++ b/crates/repo/src/bucket_transport.rs @@ -343,8 +343,9 @@ impl BucketTransport { /// Open the store behind `sc serve --store` (ADR-0060). For `sc+wal://` and /// `sc+s3://` this is [`BucketTransport::open`]. For `sc+git::` the Git /// cache lives under `scratch` (the serve home's `.sc/tmp/`), every fetch - /// call runs in its own process group so the fetch watchdog can take a - /// stuck `ssh` or `git-remote-https` down with it, and private-branch + /// call runs under a supervisor that leads its own process group and kills + /// it when the call ends (a stuck `ssh`, `git-remote-https` or any helper + /// that stays in the group dies with git), and private-branch /// objects are refused unless `allow_private_branches` is set: a Git host /// shows the branch name, recipient ids and sizes to every reader and keeps /// the ciphertext, so the opt-in belongs to the operator, not to a client. From 016f5cd3b45c0ee9e958da6afa738dbb1d09c571 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sun, 22 Nov 2026 10:34:41 +0200 Subject: [PATCH 2/9] docs: record the group-leading supervisor for the serve-mode fetch watchdog ADR-0060, THREAT-MODEL, ARCHITECTURE, CLAUDE.md, the sc-cli-reference skill and ROADMAP now say that serve mode on unix kills every helper that stays in git's process group, including after git exits, and when sc serve dies. A helper that leaves the group (setsid: an ssh ControlPersist master, a credential-cache daemon) escapes; the status file trusts local executables; SC_GIT is resolved by /bin/sh through command. Follow-on (10) moves to built, and the claims that a helper outliving git is never killed, that serve mode needs a kill program, and that a fetch outlives a killed sc serve are removed for serve mode. --- .claude/skills/sc-cli-reference/SKILL.md | 4 +- ARCHITECTURE.md | 6 +- CLAUDE.md | 22 +++-- ROADMAP.md | 15 ++- docs/THREAT-MODEL.md | 41 +++++--- docs/adr/0060-git-carried-bucket-remotes.md | 101 +++++++++++++------- 6 files changed, 118 insertions(+), 71 deletions(-) diff --git a/.claude/skills/sc-cli-reference/SKILL.md b/.claude/skills/sc-cli-reference/SKILL.md index 30787ae5..89d67a19 100644 --- a/.claude/skills/sc-cli-reference/SKILL.md +++ b/.claude/skills/sc-cli-reference/SKILL.md @@ -481,7 +481,9 @@ cargo run --bin sc -- serve --http --store sc+git:: [--al # lock is free (unix, one host, not NFS). Every # fetch runs under a download watchdog (disk per # fetch; 120 s no-progress deadline); serve mode - # needs a `kill` on the PATH to kill git's group + # runs git under a /bin/sh group leader and kills + # its group when the fetch ends (a `kill` on the + # PATH is only a backup) bash demo/run_git_carrier_demo.sh # sc+git:: proof (P39): a bare Git repo as the host; # protected path + secret + private branch pushed # (private branch refused without the opt-in), plain diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d915eecf..ec97e878 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1376,9 +1376,9 @@ objects and is removed on `Drop`. A mutable key (the manifest) is read with one of exactly `refs/sc/K` under protocol v2, so a v2 host lists that ref alone, and every fetch runs under a watchdog that kills git past its download budget (disk bytes per fetch: a key's value cap, 2 GiB for a family) or after 120 s without progress (only serve mode on -unix kills git's whole process group, through the system `kill`, and only while git is -unreaped; elsewhere, and for any helper that outlives git's exit, the helpers that write the -pack may keep writing). Scaling: `ls-remote` and push download the whole ref advertisement, +unix runs git under a `/bin/sh` group leader and kills the whole process group when the +fetch ends, helpers that outlive git included; elsewhere the helpers that write the pack may +keep writing, and a helper that leaves the group with `setsid` escapes). Scaling: `ls-remote` and push download the whole ref advertisement, and a cold read fetches every `log/` and `checkpoints/` entry behind the last checkpoint; refs grow by at least three per push until `sc remote compact` runs. diff --git a/CLAUDE.md b/CLAUDE.md index 71aca96e..17780657 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -266,14 +266,22 @@ transport-adjacent. The rest, imperatively: instance, so a big store can still compact) or after 120 s with no progress, what it added removed. The bound is disk per fetch, not RAM inside `index-pack`, not the total cache, and not the refs a family fetch - writes. Only serve mode on unix bounds disk against a hostile HTTP host: it runs - git in its own process group and kills the group through the system `kill`, - but only while git is unreaped (after the reap the group id may have been - reused, so it is never signalled; `may_signal_group`). Without a `kill` it - warns once and kills git alone. The CLI and Windows kill git alone, and the + writes. Only serve mode on unix bounds disk against a hostile HTTP host: git + runs under a `/bin/sh` group leader that holds a stdin pipe only sc has and + writes git's exit code to `/.fetch-status-`; when the fetch ends (a + normal exit or any breach) sc closes the pipe and the leader kills the whole + group with its builtin `kill`, so every helper that stays in git's process + group, including one that outlives git, dies (also when `sc serve` dies). The + system `kill` signals the group too as a backup, only while the leader is + unreaped (`may_signal_group`: sc never waits for the leader before the group is + dead; after the final reap the id may be reused, so it is never signalled); + without a `kill` it warns once. 126 or 127 from `/bin/sh` is the "cannot run + git" error. A helper that leaves the group (`setsid`: an ssh `ControlPersist` + master, a credential-cache daemon) escapes; the status file trusts local + executables; `SC_GIT` is resolved by `/bin/sh` through `command` (no shell + functions). The CLI and Windows spawn git directly and kill git alone, and the helpers that write the pack may keep writing; so may any helper that outlives - git's exit, in any mode, until it exits. Only fetches are watched; `ls-remote` - and push are not. + git's exit there. Only fetches are watched; `ls-remote` and push are not. - **`DirBucket` (`sc+wal://`) is a single-host, single-owner backend (#264).** Its CAS takes an OS advisory file lock (`File::lock`: `flock` on unix, `LockFileEx` on Windows) on `/.cas-lock`, a file that is never removed and diff --git a/ROADMAP.md b/ROADMAP.md index ab737f3a..5d05ace3 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1049,16 +1049,15 @@ P28 review follow-ons, the port de-flake, the clippy allow-list) are recorded in `update_ref`, remain because only `has_object` may use the memo). Not in scope: `ref_guard::check_ref_move` still downloads the old tip's pack to compare kinds on every push after the first (bandwidth, not round trips; the idx carries no - kind). (4) validated against a private GitHub repository on 2026-11-21 (custom + kind). (10) a group-leading supervisor for the serve-mode watchdog (unix: + git runs under a `/bin/sh` leader, and the group is killed when the fetch ends, + so a helper that stays in git's process group cannot outlive it or write after + the discard; a helper that leaves the group with `setsid`, such as an ssh + `ControlPersist` master or a credential-cache daemon, escapes; ADR-0060). + (4) validated against a private GitHub repository on 2026-11-21 (custom refs, lease CAS with one winner of two racing pushers, filtered fetch, compaction 18 → 5 refs); still unexercised there: per-push and per-blob limits, - rulesets on custom refs, and CAS across GitHub replicas beyond one race. Still - deferred: (10) a group-leading supervisor for the serve-mode watchdog: run git - under a std-only `sh` leader (`sh -c 'git "$@"; echo $? > ; read - _' sh `, stdin piped from sc) so the process group id stays pinned after - git exits, and the watchdog can group-kill a helper that outlives git (the - post-exit gap ADR-0060 documents: a smart-HTTP v0/v1 helper chain, an ssh - `ControlPersist` master) before it closes the leader's stdin and reaps it. + rulesets on custom refs, and CAS across GitHub replicas beyond one race. Declined, with reasons: (3) rollback and rewind of `refs/sc/manifest` by anyone with push access is the cross-cutting signed-ref effort already tracked under P34 follow-on (6) and ADR-0059 follow-on (2), now reachable by more people; a diff --git a/docs/THREAT-MODEL.md b/docs/THREAT-MODEL.md index be10b6c2..d9adfc79 100644 --- a/docs/THREAT-MODEL.md +++ b/docs/THREAT-MODEL.md @@ -771,10 +771,13 @@ The bucket model above applies unchanged, with the Git repository as the bucket after it exits, and the measuring goes on while a helper still holds git's pipes and once more after they close. Every breach (budget, budget after exit, no progress, a pipe held open after exit) stops git, waits up to 5 s for helpers to go and the directory to settle, then - discards. The process group is signalled only while git is unreaped, when its pid pins the + discards; in serve mode a normal exit takes the same stop and settle steps first. The + process group is signalled only while its leader is unreaped, when the leader's pid pins the group id; after the reap it is never signalled, because the id may have been reused by an unrelated process group of the same user and nothing observable shows otherwise (pinning the - pid without reaping needs `unsafe`). The limits, plainly: + pid without reaping needs `unsafe`). In serve mode the leader is a `/bin/sh` supervisor that + sc never waits for before the group is dead, so that is always the case until the final reap. + The limits, plainly: - The bound is disk bytes per fetch. It does not cover git's memory while `index-pack` resolves a pack, and it does not cover the total size of the cache, which grows by what each fetch keeps and lives as long as the process. An HTTP daemon opens one cold cache per @@ -788,19 +791,27 @@ The bucket model above applies unchanged, with the Git repository as the bucket store that has grown past the family budget stays readable and can compact itself. A hostile host can force that slower path (one git round trip per key) by serving a big family; it cannot lift any per-key cap by doing so. - - Only serve mode on unix is disk-bounded against a hostile HTTP host, and only while git - runs. There git runs in its own process group and the whole group is killed while git is - unreaped (through the system `kill`; without one the watchdog warns once and kills git - alone), so `ssh`, `git-remote-https` and `index-pack` die with it. Outside serve mode (the - unix CLI, Windows), and possibly against a smart-HTTP host on protocol v0 or v1, the kill may - not reach the helper chain that writes the pack, which can keep writing after git is killed: - those cases are not disk-bounded. The call still returns, since detached pipe readers keep a - surviving helper from blocking it. - - A helper that outlives git's exit (a smart-HTTP v0/v1 helper chain, an ssh `ControlPersist` - master, a hostile process) is not killed in any mode, because the process group is never - signalled after the leader is reaped. It may keep writing after the discard until it exits. - - A fetch in its own process group outlives a killed `sc serve`; a supervisor that kills the - whole cgroup covers it. + - Only serve mode on unix is disk-bounded against a hostile HTTP host. There git runs under + a `/bin/sh` supervisor that leads its process group, holds a stdin pipe only sc has, and + writes git's exit code to a status file in the private cache directory. When the fetch ends + (a normal exit or any breach) sc closes that pipe and the supervisor kills the whole group + with its builtin `kill`, itself included; the system `kill` signals the group too, as a + backup while the supervisor is unreaped (without one the watchdog warns once). So `ssh`, + `git-remote-https`, `index-pack` and any helper that outlives git, a smart-HTTP v0/v1 + helper chain included, die and cannot write after the discard, and a group also dies when + `sc serve` itself does. Outside serve mode (the unix CLI, Windows), and possibly against a + smart-HTTP host on protocol v0 or v1, the kill may not reach the helper chain that writes + the pack, which can keep writing after git is killed, and a helper that outlives git is not + killed: those cases are not disk-bounded. The call still returns, since detached pipe + readers keep a surviving helper from blocking it. + - A process that leaves git's process group (`setsid`: an ssh `ControlPersist` master, a + credential-cache daemon, a hostile process) escapes serve mode's kill. If it holds git's + pipes the fetch fails after the shared deadline; if it closes them and writes into + `objects/pack`, only the measuring bounds it. + - The status file trusts local executables: any process running as the user can write it, and + a forged exit code ends the fetch early with no new authority. git is run by the supervisor + through `command` with the `SC_GIT` program resolved by `/bin/sh` (a bare name through the + PATH, a path as is; shell functions are skipped). - Only fetches are watched. `ls-remote` and push run unwatched: over HTTP `http.lowSpeedLimit`/`lowSpeedTime` bound them, over ssh the operator should set `ServerAliveInterval` for the daemon's ssh user. diff --git a/docs/adr/0060-git-carried-bucket-remotes.md b/docs/adr/0060-git-carried-bucket-remotes.md index ecfd346d..17c6367c 100644 --- a/docs/adr/0060-git-carried-bucket-remotes.md +++ b/docs/adr/0060-git-carried-bucket-remotes.md @@ -241,7 +241,9 @@ under a watchdog: Every breach (the budget during the run, the budget after git exited, no progress, a pipe a helper keeps open after git exits) takes one path: stop git (the group in serve mode), reap it, keep watching until the pipes have closed - and `objects/pack` has stopped changing or 5 s have passed, then discard. The + and `objects/pack` has stopped changing or 5 s have passed, then discard; in + serve mode a normal exit runs the same stop, reap and settle steps before the + budget is checked. The files the fetch added to `objects/pack` and every local ref under `refs/sc/` are removed (the cache only mirrors the host; the next read fetches again) and the error names the cap. Discarding before the helpers are gone would let a @@ -249,35 +251,57 @@ under a watchdog: - The cache path is made absolute before any git runs: git runs with the cache as its working directory and `GIT_DIR` set to it, so a relative serve home (`.`) would be resolved from inside the cache. -- Serve mode, on unix, spawns git in its own process group and signals the group - (`kill -s KILL -- -`, std only, no `unsafe`), so `ssh` and - `git-remote-https` die with it. The group is signalled only while the leader is - unreaped: its pid pins the group id then. Once the leader is reaped, the id of - the group is free to be reused by an unrelated process group of the same user, - and nothing observable shows that it has not been (an open pipe proves a live - member only at the instant it is looked at, and the external `kill` runs - later). Pinning the pid without reaping needs `waitid(WNOWAIT)`, which is - `unsafe` and not allowed here. So after the reap the group is never signalled, - whatever the pipes say; the decision is `may_signal_group`. That needs a `kill` - program on the PATH; if it cannot be run, the watchdog falls back to killing - git alone and prints one warning. The CLI does not use a process group: Ctrl-C - must still reach git and ssh must still be able to prompt on the terminal. It - kills git alone, and the pipe readers are detached threads, so a helper that - outlives git and holds the pipes cannot block the call. A git that exits while - a helper holds its pipes is an error once one shared 120 s deadline for both - pipes has passed, after which the watchdog settles for a bounded time and - discards what the fetch added. Windows kills the child only. -- A helper that outlives git's exit is therefore not killed, in either mode, and - may keep writing after the discard until it exits. That covers a smart-HTTP - v0/v1 helper chain, an ssh `ControlPersist` master, and a hostile process. The - watchdog bounds disk only while git itself is running (or its group can still - be signalled); after git exits it measures, settles and discards, and cannot - stop a survivor. The planned fix (ROADMAP, `sc+git::` follow-on 10) is a - group-leading supervisor: in serve mode, run git under a std-only `sh` leader - (`sh -c 'git "$@"; echo $? > ; read _' sh `, with stdin piped - from sc) so the process group id stays pinned after git exits. The watchdog could - then group-kill a helper that outlives git, which closes this gap, before it - closes the leader's stdin and reaps it. Not built in v1. +- Serve mode, on unix, runs git under a group-leading supervisor, std only, no + `unsafe`. The child is `/bin/sh` (an absolute path, not looked up on the PATH) + with its own process group, running a constant script with the git program, its + `-c` options and its arguments as positional parameters. The script runs git in a + subshell with stdin from `/dev/null`; the subshell writes git's exit code to + `/.fetch-status-`; the shell closes its own stdout and stderr, so only + git and its helpers hold the pipes, and then blocks reading its stdin, a pipe + only sc holds. When that pipe closes, because sc dropped it or died, the shell + runs the builtin `kill -s KILL 0`: its whole group dies, itself included, and no + group id and no external program are involved. git runs through `command`, so a + shell function exported by the environment cannot stand in for it; `SC_GIT` is + resolved by `/bin/sh` the same way (a bare name through the PATH, a path as is). + The script uses a parenthesized subshell on purpose: with a `{ ...; } &` group + bash execs the command in place and the status line never runs. +- sc never waits for the leader before the group is dead. It learns git's exit + from the status file (one to three ASCII digits and a newline, at most 255; + anything else, such as a partial write, counts as not yet), so the leader stays + unreaped and its pid keeps pinning the group id. A reaped leader would free the + id for reuse by an unrelated process group of the same user, and pinning a pid + without reaping it needs `waitid(WNOWAIT)`, which is `unsafe` and not allowed + here. A leader that dies early writes no status; the no-progress deadline ends + the fetch. Every end of a fetch, a normal exit or any breach, takes the same + steps: close the stdin pipe; signal the group through the system `kill` as a + backup while the leader is still unreaped (`may_signal_group`; it also reaches + helpers when the leader died before it saw the end of input); reap the leader, + within a bound; then settle and measure as below. After a successful fetch the + group is killed too, so every helper that stays in git's process group, a + smart-HTTP v0/v1 helper chain included, dies when git exits and cannot write + after the discard. After the reap the group is never signalled. `kill` is only + the backup: without it a warning is printed once and the leader's own `kill 0` + still ends the group, unless the leader died early. +- A `/bin/sh` exit status of 126 or 127 means `command` could not run git and + gives the usual "cannot run git (...): install git or set SC_GIT" error. A git + killed by signal `n` reads as exit `128 + n`. +- What escapes, and what is trusted. A process that leaves git's group with + `setsid` (an ssh `ControlPersist` master, a credential-cache daemon) is not + killed. If it holds a pipe, the fetch is an error after the shared deadline, and + the discard follows; if it closes its pipes and writes into `objects/pack`, the + watchdog bounds disk only while it measures. The status file is a channel that + trusts local executables: any process of the same user can write it (the cache + directory is private, mode 0700), and a forged "0" ends the fetch early without + any new authority. The CLI and Windows spawn git directly, as before: the CLI + kills git alone, the pipe readers are detached threads, so a helper that + outlives git and holds the pipes cannot block the call, and a helper that + outlives git is not killed and may keep writing after the discard until it + exits. The CLI does not use a process group: Ctrl-C must still reach git and ssh + must still be able to prompt on the terminal. A git that exits while something + holds its pipes is an error once one shared 120 s deadline for both pipes has + passed (in serve mode that is only a process outside the group), after which + the watchdog settles for a bounded time and discards what the fetch added. + Windows kills the child only. The exact limits: @@ -288,13 +312,16 @@ The exact limits: caches keep. - Refs a family fetch writes are not measured. A hostile host can advertise a great many refs; only the 120 s no-progress deadline bounds that. -- Only serve mode on unix actually bounds disk against a hostile HTTP host. Outside - it (the unix CLI, Windows), and possibly against a smart-HTTP host on protocol - v0 or v1, the kill may not reach the helper chain that writes the pack - (`git-remote-https`, `index-pack`), which can keep writing after git is killed. - Those cases are not disk-bounded; the watchdog still frees the call. -- A fetch in its own process group outlives a killed `sc serve`: a supervisor - that kills the whole cgroup covers it. +- Only serve mode on unix actually bounds disk against a hostile HTTP host: every + process that stays in git's group, a smart-HTTP v0/v1 helper chain included, is + killed when the fetch ends, and a process that leaves the group (`setsid`) + escapes. Outside serve mode (the unix CLI, Windows) the kill may not reach the + helper chain that writes the pack (`git-remote-https`, `index-pack`), which can + keep writing after git is killed. Those cases are not disk-bounded; the + watchdog still frees the call. +- A fetch's group ends when `sc serve` dies, whatever the signal: the leader sees + the end of its stdin and kills the group. Only a process that left the group + outlives it. ### Private-branch opt-in From 99b3c76eb84fa9d67282bad4fd181a013c47e1d0 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sun, 22 Nov 2026 10:42:48 +0200 Subject: [PATCH 3/9] test(objio): handshake the escaped-helper tests and restore the serve leg The escaped helper now writes its pid after setsid succeeds and the shim waits for it, instead of sleeping, which raced the group kill. The late writer test regains its serve-mode leg (a writer that left the group and holds the pipes). describe_code words the 128 + n case as a hint, and the THREAT-MODEL no longer says a v0/v1 helper chain may survive in serve mode. --- crates/objio/src/git.rs | 79 ++++++++++++++++++++++++++++------------- docs/THREAT-MODEL.md | 7 ++-- 2 files changed, 57 insertions(+), 29 deletions(-) diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index e7b2d37d..e935b740 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -141,7 +141,7 @@ struct Exit { fn describe_code(code: Option) -> String { match code { None => "signal".to_string(), - Some(c) if c > 128 => format!("{c}, signal {} if it was killed", c - 128), + Some(c) if c > 128 => format!("{c} (128 + signal {}, if git was killed)", c - 128), Some(c) => c.to_string(), } } @@ -4111,6 +4111,17 @@ mod tests { .is_ok_and(|s| s.success()) } + /// Shell text that starts `cmd` in the background in a new session, so it + /// leaves git's process group, and then waits until it has: the perl wrapper + /// writes its pid to `pids` once `setsid` has succeeded. + #[cfg(unix)] + fn escaped(cmd: &str, pids: &Path) -> String { + format!( + "perl -MPOSIX -e 'POSIX::setsid() or exit 1; open(F, \">\", shift); print F \"$$\\n\"; close F; exec @ARGV' {p} {cmd} & until [ -s {p} ]; do sleep 0.01; done;", + p = pids.display() + ) + } + /// A git that exits cleanly while something keeps its pipes open is an error /// within `idle`, not a hang, and what the fetch added is discarded. In the /// CLI that is any helper. In serve mode the group is killed when git exits, @@ -4128,16 +4139,15 @@ mod tests { let t = Tmp::new("held"); let (_, url) = hub(&t); let pids = t.0.join("pids"); - let sleeper = if group { - "perl -MPOSIX -e 'POSIX::setsid(); exec @ARGV' sleep 600" + // The fetch "downloads" something small and exits cleanly (in serve + // mode once the helper has left the group). + let launch = if group { + escaped("sleep 600", &pids) } else { - "sleep 600" + format!("sleep 600 & echo $! > {};", pids.display()) }; - // The fetch "downloads" something small and exits cleanly, once the - // helper has had time to leave the group. let body = format!( - "case \" $* \" in *\" fetch \"*) head -c 1000 /dev/zero > \"$GIT_DIR/objects/pack/tmp_pack_held\"; {sleeper} & echo $! > {}; sleep 0.5; exit 0;; esac", - pids.display() + "case \" $* \" in *\" fetch \"*) head -c 1000 /dev/zero > \"$GIT_DIR/objects/pack/tmp_pack_held\"; {launch} exit 0;; esac" ); let sh = shim(&t, "held.sh", &body); let mut b = GitBucket::open_full(&url, sh, CHUNK, None, group).unwrap(); @@ -4301,26 +4311,42 @@ mod tests { assert!(!may_signal_group(true), "a reaped leader pins nothing"); } - /// In the CLI a helper that outlives git can cross the budget after git has - /// exited and close both pipes before the shared deadline. The watchdog must - /// still catch it, and must not call the fetch good. + /// A helper that outlives git can cross the budget after git has exited and + /// close both pipes before the shared deadline. The watchdog must still catch + /// it, and must not call the fetch good. In the CLI any helper can do this; in + /// serve mode only one that left git's group (`setsid`) can, since the group + /// is killed at git's exit. #[cfg(unix)] #[test] fn a_helper_that_crosses_the_budget_after_git_exits_is_caught() { need_git!(); - let t = Tmp::new("late"); - let (_, url) = hub(&t); - // git exits at once; a helper holds the pipes, then writes too much and exits. - let body = "case \" $* \" in *\" fetch \"*) ( sleep 0.6; head -c 2000000 /dev/zero > \"$GIT_DIR/objects/pack/pack-late.pack\" ) & exit 0;; esac"; - let sh = shim(&t, "late.sh", body); - let mut b = GitBucket::open_full(&url, sh, CHUNK, None, false).unwrap(); - b.tune.budget = Some(64 << 10); - let e = get_err(&b, "manifest"); - assert!(e.contains("65536 bytes"), "{e}"); - assert!( - dir_files(&b.cache.join("objects").join("pack")).is_empty(), - "the late download must be discarded" - ); + for group in [true, false] { + if group && !have_setsid_perl() { + continue; + } + let t = Tmp::new("late"); + let (_, url) = hub(&t); + let pids = t.0.join("pids"); + let write = + "sleep 0.6; head -c 2000000 /dev/zero > \"$GIT_DIR/objects/pack/pack-late.pack\""; + // git exits at once; a helper holds the pipes, then writes too much and exits. + let launch = if group { + escaped(&format!("sh -c '{write}'"), &pids) + } else { + format!("( {write} ) &") + }; + let body = format!("case \" $* \" in *\" fetch \"*) {launch} exit 0;; esac"); + let sh = shim(&t, "late.sh", &body); + let mut b = GitBucket::open_full(&url, sh, CHUNK, None, group).unwrap(); + b.tune.budget = Some(64 << 10); + let e = get_err(&b, "manifest"); + assert!(e.contains("65536 bytes"), "{e}"); + assert!( + dir_files(&b.cache.join("objects").join("pack")).is_empty(), + "the late download must be discarded" + ); + assert!(no_status_files(&b)); + } } /// One `fetch` of `refs/sc/manifest` through the watchdog, for tests that @@ -4455,7 +4481,10 @@ mod tests { assert!(no_status_files(&b)); } } - assert_eq!(describe_code(Some(137)), "137, signal 9 if it was killed"); + assert_eq!( + describe_code(Some(137)), + "137 (128 + signal 9, if git was killed)" + ); assert_eq!(describe_code(Some(3)), "3"); assert_eq!(describe_code(None), "signal"); } diff --git a/docs/THREAT-MODEL.md b/docs/THREAT-MODEL.md index d9adfc79..d2f57c34 100644 --- a/docs/THREAT-MODEL.md +++ b/docs/THREAT-MODEL.md @@ -799,10 +799,9 @@ The bucket model above applies unchanged, with the Git repository as the bucket backup while the supervisor is unreaped (without one the watchdog warns once). So `ssh`, `git-remote-https`, `index-pack` and any helper that outlives git, a smart-HTTP v0/v1 helper chain included, die and cannot write after the discard, and a group also dies when - `sc serve` itself does. Outside serve mode (the unix CLI, Windows), and possibly against a - smart-HTTP host on protocol v0 or v1, the kill may not reach the helper chain that writes - the pack, which can keep writing after git is killed, and a helper that outlives git is not - killed: those cases are not disk-bounded. The call still returns, since detached pipe + `sc serve` itself does. Outside serve mode (the unix CLI, Windows) the kill may not reach the + helper chain that writes the pack, which can keep writing after git is killed, and a helper + that outlives git is not killed: those cases are not disk-bounded. The call still returns, since detached pipe readers keep a surviving helper from blocking it. - A process that leaves git's process group (`setsid`: an ssh `ControlPersist` master, a credential-cache daemon, a hostile process) escapes serve mode's kill. If it holds git's From d50156b6a97b2d8d55fe37cb9d1b9738087c7d6a Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sun, 22 Nov 2026 11:08:04 +0200 Subject: [PATCH 4/9] fix(objio): harden the fetch leader against shell startup state and unbounded waits The leader's environment loses SHELLOPTS, BASHOPTS, BASH_ENV, ENV and every BASH_FUNC_* variable (bash is /bin/sh on macOS: monitor moves the job out of the group, errexit skips the status line and the final kill, exported functions override command, echo, read and kill). The script itself now tolerates errexit and uses `command --`. The backup kill program is spawned and waited for at most 2 s. A missing /bin/sh is reported as that, not as a missing git. The 128 + n hint appears only in the text of fetches the leader ran, and a 126/127 from the shell includes git's own stderr. Tests: a hostile shell environment (set through a test seam on the leader command), a hanging kill program, the leader must still exist when the group is signalled, the quiet-writer test waits for the helper's first write and has a variant without the backup kill, and the perl setsid probe needs a positive session id. Docs soften the claims about sc serve dying, say the shell functions are stripped, and note the per-fetch cost. --- CLAUDE.md | 8 +- crates/objio/src/git.rs | 285 ++++++++++++++++---- crates/repo/src/bucket_transport.rs | 8 +- docs/THREAT-MODEL.md | 13 +- docs/adr/0060-git-carried-bucket-remotes.md | 39 ++- 5 files changed, 284 insertions(+), 69 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 17780657..aa72b663 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -271,15 +271,17 @@ transport-adjacent. The rest, imperatively: writes git's exit code to `/.fetch-status-`; when the fetch ends (a normal exit or any breach) sc closes the pipe and the leader kills the whole group with its builtin `kill`, so every helper that stays in git's process - group, including one that outlives git, dies (also when `sc serve` dies). The + group, including one that outlives git, dies (the group also ends when `sc serve` + exits, since the leader's stdin closes). The system `kill` signals the group too as a backup, only while the leader is unreaped (`may_signal_group`: sc never waits for the leader before the group is dead; after the final reap the id may be reused, so it is never signalled); without a `kill` it warns once. 126 or 127 from `/bin/sh` is the "cannot run git" error. A helper that leaves the group (`setsid`: an ssh `ControlPersist` master, a credential-cache daemon) escapes; the status file trusts local - executables; `SC_GIT` is resolved by `/bin/sh` through `command` (no shell - functions). The CLI and Windows spawn git directly and kill git alone, and the + executables; `SC_GIT` is resolved by `/bin/sh` through `command --`, with + `SHELLOPTS`, `BASHOPTS`, `BASH_ENV`, `ENV` and `BASH_FUNC_*` removed from its + environment (so no shell functions). The CLI and Windows spawn git directly and kill git alone, and the helpers that write the pack may keep writing; so may any helper that outlives git's exit there. Only fetches are watched; `ls-remote` and push are not. - **`DirBucket` (`sc+wal://`) is a single-host, single-owner backend (#264).** diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index e935b740..4d9ca8ef 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -74,13 +74,39 @@ const SETTLE: Duration = Duration::from_secs(5); /// `. It runs git in a subshell with stdin from `/dev/null` and writes /// git's exit code to the status file, drops its own stdout and stderr so only /// git and its helpers hold the pipes, then blocks reading its stdin, which only -/// sc holds. When sc closes that pipe (or dies) it kills its whole group, itself -/// included, with the builtin `kill`: no group id and no external program are -/// involved. The parentheses matter: a `{ ...; } &` group lets bash exec the -/// command in place and the status line never runs. `command` skips shell -/// functions, so an exported `BASH_FUNC_*` cannot stand in for git. +/// sc holds. When that pipe closes (sc dropped it, or exited) it kills its whole +/// group, itself included, with the builtin `kill`: no group id and no external +/// program are involved. The parentheses matter: a `{ ...; } &` group lets bash +/// exec the command in place and the status line never runs. `|| r=$?` and +/// `|| :` keep the status write and the final `kill` alive under `errexit`; +/// `command --` skips shell functions and takes a program named like an option. +/// The shell's own startup state is removed from its environment +/// ([`scrub_shell_env`]), because it would change all of that. #[cfg_attr(not(unix), allow(dead_code))] -const LEADER: &str = "s=$1; shift; ( command \"$@\" \"$s\" ) & exec >/dev/null 2>&1; read _; kill -s KILL 0"; +const LEADER: &str = "s=$1; shift; ( r=0; command -- \"$@\" \"$s\" ) & exec >/dev/null 2>&1; read _ || :; kill -s KILL 0"; + +/// Remove from `cmd` what makes a POSIX shell behave differently from the +/// [`LEADER`] script: `SHELLOPTS` (`monitor` moves the background job into +/// another process group, `errexit` skips the status line), `BASHOPTS`, +/// `BASH_ENV` and `ENV` (startup files), and every exported function +/// (`BASH_FUNC_*`), which would override `command`, `echo`, `read` or `kill`. +/// `extra` names variables a test put on the command and the scrub must also +/// catch. +#[cfg_attr(not(unix), allow(dead_code))] +fn scrub_shell_env(cmd: &mut Command, extra: &[&str]) { + let names: Vec = std::env::vars_os() + .map(|(k, _)| k) + .chain(extra.iter().map(OsString::from)) + .collect(); + for k in names { + let s = k.to_string_lossy(); + if matches!(&*s, "SHELLOPTS" | "BASHOPTS" | "BASH_ENV" | "ENV") + || s.starts_with("BASH_FUNC_") + { + cmd.env_remove(&k); + } + } +} /// The leader's status file in the private cache directory, removed on drop so /// every exit path of a fetch cleans it up. @@ -136,12 +162,15 @@ struct Exit { code: Option, } -/// An exit code for error text. A git killed by signal `n` reports `128 + n` -/// through the serve-mode leader and no code at all when run directly. -fn describe_code(code: Option) -> String { - match code { +/// An exit code for error text. A git killed by signal `n` has no code when run +/// directly and reports `128 + n` through the serve-mode leader, which only +/// that case's text says (`led`). +fn describe_exit(out: &Out) -> String { + match out.code { None => "signal".to_string(), - Some(c) if c > 128 => format!("{c} (128 + signal {}, if git was killed)", c - 128), + Some(c) if out.led && c > 128 => { + format!("{c} (128 + signal {}, if git was killed)", c - 128) + } Some(c) => c.to_string(), } } @@ -281,6 +310,8 @@ struct Known { struct Out { success: bool, code: Option, + /// Run by the serve-mode leader, whose status reports a signal as `128 + n`. + led: bool, stdout: Vec, stderr: String, } @@ -309,6 +340,10 @@ struct Tuning { /// the supervisor's own kill (tests point it at a missing one). #[cfg(unix)] kill: &'static str, + /// Variables put on the serve-mode leader's command before its environment + /// is scrubbed, standing in for a hostile parent environment (tests only). + #[cfg(unix)] + leader_env: &'static [(&'static str, &'static str)], } impl Default for Tuning { @@ -320,6 +355,8 @@ impl Default for Tuning { poll: None, #[cfg(unix)] kill: "kill", + #[cfg(unix)] + leader_env: &[], } } } @@ -569,17 +606,30 @@ fn may_signal_group(leader_reaped: bool) -> bool { /// Signal every process of group `pgid` with SIGKILL through the system `kill`. /// False only when that program cannot be run; a non-zero exit (the group is -/// already gone) is fine. +/// already gone) is fine. The program is waited for at most 2 s and then killed, +/// so a `kill` that hangs cannot park the caller. #[cfg(unix)] fn kill_group(program: &str, pgid: u32) -> bool { - Command::new(program) + let Ok(mut c) = Command::new(program) .args(["-s", "KILL", "--"]) .arg(format!("-{pgid}")) .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::null()) - .status() - .is_ok() + .spawn() + else { + return false; + }; + let end = Instant::now() + Duration::from_secs(2); + while Instant::now() < end { + if !matches!(c.try_wait(), Ok(None)) { + return true; + } + std::thread::sleep(Duration::from_millis(2)); + } + let _ = c.kill(); + let _ = c.wait(); + true } /// Set once a missing `kill` has been reported. @@ -976,6 +1026,7 @@ impl GitBucket { Ok(Out { success: status.success(), code: status.code(), + led: false, stdout, stderr: self.scrub(&String::from_utf8_lossy(&stderr)), }) @@ -1021,7 +1072,17 @@ impl GitBucket { sh.args(["-c", LEADER, "sc-fetch"]) .arg(&sf.0) .arg(&self.git); - self.hardened(sh, args, &[], true) + let mut sh = self.hardened(sh, args, &[], true); + #[cfg(unix)] + for (k, v) in self.tune.leader_env { + sh.env(k, v); + } + #[cfg(unix)] + let extra: Vec<&str> = self.tune.leader_env.iter().map(|(k, _)| *k).collect(); + #[cfg(not(unix))] + let extra: Vec<&str> = Vec::new(); + scrub_shell_env(&mut sh, &extra); + sh } None => self.command(args, &[], false), }; @@ -1029,7 +1090,15 @@ impl GitBucket { if status_file.is_some() { std::os::unix::process::CommandExt::process_group(&mut cmd, 0); } - let mut child = self.spawn(&mut cmd)?; + let mut child = if status_file.is_some() { + cmd.spawn().map_err(|e| { + Error::Backend(format!( + "cannot run /bin/sh, which supervises git in serve mode: {e}" + )) + })? + } else { + self.spawn(&mut cmd)? + }; let mut lead_in = child.stdin.take(); let mut sout = child.stdout.take().expect("piped"); let mut serr = child.stderr.take().expect("piped"); @@ -1155,15 +1224,16 @@ impl GitBucket { self.tune.idle ))); } + let (stdout, over) = readers.got_out.take().unwrap_or_default(); + let stderr = readers.got_err.take().unwrap_or_default(); // `command` reports a git it cannot find or start as 127 or 126. if let (Some(_), Some(c @ (126 | 127))) = (&status_file, exit.code) { return Err(Error::Backend(format!( - "cannot run git ({}): the shell exited with status {c} (install git or set SC_GIT)", - self.git + "cannot run git ({}): exit status {c}: {} (install git or set SC_GIT)", + self.git, + self.scrub(&String::from_utf8_lossy(&stderr)) ))); } - let (stdout, over) = readers.got_out.take().unwrap_or_default(); - let stderr = readers.got_err.take().unwrap_or_default(); if over { return Err(Error::Backend(format!( "git output exceeded {max_out} bytes (hostile or corrupt remote)" @@ -1172,6 +1242,7 @@ impl GitBucket { Ok(Out { success: exit.success, code: exit.code, + led: status_file.is_some(), stdout, stderr: self.scrub(&String::from_utf8_lossy(&stderr)), }) @@ -1308,7 +1379,7 @@ impl GitBucket { fn fail(&self, what: &str, out: &Out) -> Error { Error::Backend(format!( "git {what} failed (exit {}): {}", - describe_code(out.code), + describe_exit(out), out.stderr )) } @@ -1464,7 +1535,7 @@ impl GitBucket { .unwrap_or_default(); Ok(Pushed::Refused(format!( "git push failed (exit {}): {} {}", - describe_code(out.code), + describe_exit(&out), out.stderr, first ))) @@ -2316,7 +2387,7 @@ impl Bucket for GitBucket { .unwrap_or_default(); let refusal = format!( "git push failed (exit {}): {} {}", - describe_code(out.code), + describe_exit(&out), out.stderr, first ); @@ -4105,19 +4176,27 @@ mod tests { /// process group (macOS has no `setsid` program). #[cfg(unix)] fn have_setsid_perl() -> bool { - Command::new("perl") - .args(["-MPOSIX", "-e", "POSIX::setsid(); exit 0"]) + // Run like the fixture: as a background child, never a group leader. + Command::new("sh") + .args(["-c", &format!("{PERL_SETSID} exit 0' & wait $!")]) .status() .is_ok_and(|s| s.success()) } + /// The start of a perl one-liner that leaves the process group: it needs a + /// positive session id (`setsid` fails with undef or -1 otherwise). The + /// caller closes the quote. + #[cfg(unix)] + const PERL_SETSID: &str = + "perl -MPOSIX -e 'my $s = POSIX::setsid(); exit 1 unless defined $s && $s > 0;"; + /// Shell text that starts `cmd` in the background in a new session, so it /// leaves git's process group, and then waits until it has: the perl wrapper /// writes its pid to `pids` once `setsid` has succeeded. #[cfg(unix)] fn escaped(cmd: &str, pids: &Path) -> String { format!( - "perl -MPOSIX -e 'POSIX::setsid() or exit 1; open(F, \">\", shift); print F \"$$\\n\"; close F; exec @ARGV' {p} {cmd} & until [ -s {p} ]; do sleep 0.01; done;", + "{PERL_SETSID} open(F, \">\", shift); print F \"$$\\n\"; close F; exec @ARGV' {p} {cmd} & i=0; until [ -s {p} ] || [ $i -ge 500 ]; do sleep 0.01; i=$((i+1)); done;", p = pids.display() ) } @@ -4238,15 +4317,20 @@ mod tests { } } - /// A `kill` stand-in that records its arguments and then really kills: - /// returns the program path and the log it appends to. + /// A `kill` stand-in that records its arguments and the state of the group + /// leader at that moment (`leader=` the `ps` state of the pid the group id + /// names, empty when it is gone; a zombie counts as there) and then really + /// kills: returns the program path and the log it appends to. #[cfg(unix)] fn recording_kill(t: &Tmp) -> (&'static str, PathBuf) { let log = t.0.join("kill.log"); let sh = shim_script( t, "killrec.sh", - &format!("echo \"$@\" >> {}\nexec kill \"$@\"\n", log.display()), + &format!( + "echo \"$@ leader=$(ps -o stat= -p ${{4#-}} | tr -d ' ')\" >> {}\nexec kill \"$@\"\n", + log.display() + ), ); (Box::leak(sh.into_boxed_str()), log) } @@ -4292,6 +4376,10 @@ mod tests { 1, "one signal while the leader is unreaped, none after: {calls:?}" ); + assert!( + !calls.trim_end().ends_with("leader="), + "the leader must still exist (unreaped) when it is signalled: {calls:?}" + ); if helper_holds_pipes { let pid: u32 = std::fs::read_to_string(&pids) .unwrap() @@ -4365,28 +4453,33 @@ mod tests { /// one that closed its pipes and keeps writing into `objects/pack`: it is dead /// afterwards and the directory stops growing. The download is discarded only /// where it crossed the budget (the second case: git exits at once, a fast - /// writer crosses the budget before the first poll that sees the exit). + /// writer crosses the budget before the first poll that sees the exit). The + /// writer acknowledges its first write before git exits. With the backup + /// `kill` missing the leader's own `kill 0` does the same. #[cfg(unix)] #[test] fn a_helper_that_closes_its_pipes_and_keeps_writing_is_dead_after_git_exits() { need_git!(); - for (budget, crosses) in [(32u64 << 20, false), (1 << 20, true)] { + for (budget, crosses, backup) in [ + (32u64 << 20, false, true), + (1 << 20, true, true), + (32 << 20, false, false), + (1 << 20, true, false), + ] { let t = Tmp::new("quietwriter"); let (_, url) = hub(&t); let pids = t.0.join("pids"); - let (pause, hold) = if crosses { - ("0.01", "") - } else { - ("0.05", "sleep 0.4;") - }; + let ack = t.0.join("ack"); + let pause = if crosses { "0.01" } else { "0.05" }; let body = format!( - "case \" $* \" in *\" fetch \"*) ( exec >/dev/null 2>&1; while :; do head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/pack-late.pack\"; sleep {pause}; done ) & echo $! > {}; {hold} exit 0;; esac", - pids.display() + "case \" $* \" in *\" fetch \"*) ( exec >/dev/null 2>&1; head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/pack-late.pack\"; echo > {ack}; while :; do head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/pack-late.pack\"; sleep {pause}; done ) & echo $! > {p}; until [ -e {ack} ]; do sleep 0.01; done; exit 0;; esac", + p = pids.display(), + ack = ack.display() ); let sh = shim(&t, "quiet.sh", &body); let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); let (kill, killed) = recording_kill(&t); - b.tune.kill = kill; + b.tune.kill = if backup { kill } else { "/no/such/kill" }; b.tune.budget = Some(budget); if crosses { b.tune.poll = Some(Duration::from_millis(1500)); @@ -4406,7 +4499,7 @@ mod tests { // The group is ended by git's exit, not left to the pipe closing when // the call returns: the backup signal went out exactly once. let calls = std::fs::read_to_string(&killed).unwrap_or_default(); - assert_eq!(calls.lines().count(), 1, "{calls:?}"); + assert_eq!(calls.lines().count(), usize::from(backup), "{calls:?}"); if crosses { assert!(over && out.is_err(), "crossed the budget: an error"); assert!(dir_files(&pack).is_empty(), "and the download is discarded"); @@ -4449,6 +4542,94 @@ mod tests { assert!(no_status_files(&b)); } + /// A parent environment that changes how a POSIX shell behaves must not reach + /// the leader: `SHELLOPTS=errexit:monitor` (the status line and the final + /// `kill` would be skipped, the background job would leave the group), + /// `BASH_ENV` and `ENV` startup files, and exported functions that override + /// `command`, `echo`, `read` and `kill`. The fetch fails with exit 3: its code + /// is reported, the helper is dead and no startup file ran. + #[cfg(unix)] + #[test] + fn a_hostile_shell_environment_does_not_change_the_leader() { + need_git!(); + let t = Tmp::new("hostileenv"); + let (_, url) = hub(&t); + let pids = t.0.join("pids"); + let marker = t.0.join("marker"); + let startup = shim_script( + &t, + "startup.sh", + &format!("echo ran > {}\n", marker.display()), + ); + let startup: &'static str = Box::leak(startup.into_boxed_str()); + let body = format!( + "case \" $* \" in *\" fetch \"*) sleep 600 & echo $! > {}; exit 3;; esac", + pids.display() + ); + let sh = shim(&t, "hostile.sh", &body); + let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); + b.tune.leader_env = Box::leak(Box::new([ + ("SHELLOPTS", "errexit:monitor"), + ("BASHOPTS", "login_shell"), + ("BASH_ENV", startup), + ("ENV", startup), + ("BASH_FUNC_echo%%", "() { :; }"), + ("BASH_FUNC_kill%%", "() { :; }"), + ("BASH_FUNC_read%%", "() { :; }"), + ("BASH_FUNC_command%%", "() { :; }"), + ])); + let (out, over) = watched_fetch(&b); + let out = out.unwrap(); + assert!( + !over && !out.success && out.code == Some(3), + "{:?}", + out.code + ); + let helper: u32 = std::fs::read_to_string(&pids) + .unwrap() + .trim() + .parse() + .unwrap(); + wait_gone(helper); + assert!(!marker.exists(), "a startup file ran in the leader"); + assert!(no_status_files(&b)); + } + + /// A `kill` program that hangs cannot park a breach: it is waited for a + /// bounded time and killed, and the leader's own `kill 0` still ends the + /// group. + #[cfg(unix)] + #[test] + fn a_hanging_kill_program_is_bounded() { + need_git!(); + let t = Tmp::new("hangkill"); + let (_, url) = hub(&t); + let pids = t.0.join("pids"); + let body = format!( + "case \" $* \" in *\" fetch \"*) sleep 600 & echo $! > {}; while :; do head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/tmp_pack_x\"; sleep 0.01; done;; esac", + pids.display() + ); + let sh = shim(&t, "hangrun.sh", &body); + let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); + let hang = shim_script(&t, "hangkill.sh", "exec sleep 600\n"); + b.tune.kill = Box::leak(hang.into_boxed_str()); + b.tune.budget = Some(1 << 20); + let started = Instant::now(); + let e = get_err(&b, "manifest"); + assert!(e.contains("1048576 bytes"), "{e}"); + assert!( + started.elapsed() < Duration::from_secs(20), + "{:?}", + started.elapsed() + ); + let helper: u32 = std::fs::read_to_string(&pids) + .unwrap() + .trim() + .parse() + .unwrap(); + wait_gone(helper); + } + /// git's exit code reaches the caller through the leader (a failing fetch is /// `!success` with that code), and a git killed by a signal reads as 128 + n. /// The CLI reports the same codes, and no code for a signal. @@ -4481,12 +4662,21 @@ mod tests { assert!(no_status_files(&b)); } } + let out = |code, led| Out { + success: false, + code, + led, + stdout: Vec::new(), + stderr: String::new(), + }; assert_eq!( - describe_code(Some(137)), + describe_exit(&out(Some(137), true)), "137 (128 + signal 9, if git was killed)" ); - assert_eq!(describe_code(Some(3)), "3"); - assert_eq!(describe_code(None), "signal"); + // Direct runs (the CLI, push, ls-remote) keep the plain code. + assert_eq!(describe_exit(&out(Some(137), false)), "137"); + assert_eq!(describe_exit(&out(Some(3), true)), "3"); + assert_eq!(describe_exit(&out(None, false)), "signal"); } /// `command` exits 127 for a git it cannot find and 126 for one it cannot @@ -4524,11 +4714,14 @@ mod tests { let sh = shim( &t, "e127.sh", - "case \" $* \" in *\" fetch \"*) exit 127;; esac", + "case \" $* \" in *\" fetch \"*) echo 'no such helper' >&2; exit 127;; esac", ); let b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); let e = watched_fetch(&b).0.err().unwrap().to_string(); - assert!(e.contains("cannot run git"), "{e}"); + assert!( + e.contains("cannot run git") && e.contains("no such helper"), + "git's own message must be in the error: {e}" + ); } /// A leader that dies on its own (here a shim that kills the supervisor and diff --git a/crates/repo/src/bucket_transport.rs b/crates/repo/src/bucket_transport.rs index eb1407cd..a6fba58a 100644 --- a/crates/repo/src/bucket_transport.rs +++ b/crates/repo/src/bucket_transport.rs @@ -342,10 +342,10 @@ impl BucketTransport { /// Open the store behind `sc serve --store` (ADR-0060). For `sc+wal://` and /// `sc+s3://` this is [`BucketTransport::open`]. For `sc+git::` the Git - /// cache lives under `scratch` (the serve home's `.sc/tmp/`), every fetch - /// call runs under a supervisor that leads its own process group and kills - /// it when the call ends (a stuck `ssh`, `git-remote-https` or any helper - /// that stays in the group dies with git), and private-branch + /// cache lives under `scratch` (the serve home's `.sc/tmp/`), on unix every + /// fetch call runs under a supervisor that leads its own process group and + /// kills it when the call ends (a stuck `ssh`, `git-remote-https` or any + /// helper that stays in the group dies with git), and private-branch /// objects are refused unless `allow_private_branches` is set: a Git host /// shows the branch name, recipient ids and sizes to every reader and keeps /// the ciphertext, so the opt-in belongs to the operator, not to a client. diff --git a/docs/THREAT-MODEL.md b/docs/THREAT-MODEL.md index d2f57c34..ef52fd5b 100644 --- a/docs/THREAT-MODEL.md +++ b/docs/THREAT-MODEL.md @@ -798,8 +798,10 @@ The bucket model above applies unchanged, with the Git repository as the bucket with its builtin `kill`, itself included; the system `kill` signals the group too, as a backup while the supervisor is unreaped (without one the watchdog warns once). So `ssh`, `git-remote-https`, `index-pack` and any helper that outlives git, a smart-HTTP v0/v1 - helper chain included, die and cannot write after the discard, and a group also dies when - `sc serve` itself does. Outside serve mode (the unix CLI, Windows) the kill may not reach the + helper chain included, die and cannot write after the discard. The group also ends when + `sc serve` exits, because the supervisor's stdin closes (if the supervisor itself was + killed first only the backup `kill` remains, and a child spawned at the same moment can + briefly hold the pipe open on macOS). Outside serve mode (the unix CLI, Windows) the kill may not reach the helper chain that writes the pack, which can keep writing after git is killed, and a helper that outlives git is not killed: those cases are not disk-bounded. The call still returns, since detached pipe readers keep a surviving helper from blocking it. @@ -809,8 +811,11 @@ The bucket model above applies unchanged, with the Git repository as the bucket `objects/pack`, only the measuring bounds it. - The status file trusts local executables: any process running as the user can write it, and a forged exit code ends the fetch early with no new authority. git is run by the supervisor - through `command` with the `SC_GIT` program resolved by `/bin/sh` (a bare name through the - PATH, a path as is; shell functions are skipped). + through `command --` with the `SC_GIT` program resolved by `/bin/sh` (a bare name through the + PATH, a path as is). The supervisor's environment loses `SHELLOPTS`, `BASHOPTS`, `BASH_ENV`, + `ENV` and every `BASH_FUNC_*`, so a hostile parent environment cannot change the shell's + behavior or override its builtins with exported functions. Each fetch costs an extra `sh`, a + `kill` spawn and about 20 ms. - Only fetches are watched. `ls-remote` and push run unwatched: over HTTP `http.lowSpeedLimit`/`lowSpeedTime` bound them, over ssh the operator should set `ServerAliveInterval` for the daemon's ssh user. diff --git a/docs/adr/0060-git-carried-bucket-remotes.md b/docs/adr/0060-git-carried-bucket-remotes.md index 17c6367c..859f56c3 100644 --- a/docs/adr/0060-git-carried-bucket-remotes.md +++ b/docs/adr/0060-git-carried-bucket-remotes.md @@ -258,13 +258,20 @@ under a watchdog: subshell with stdin from `/dev/null`; the subshell writes git's exit code to `/.fetch-status-`; the shell closes its own stdout and stderr, so only git and its helpers hold the pipes, and then blocks reading its stdin, a pipe - only sc holds. When that pipe closes, because sc dropped it or died, the shell + only sc holds. When that pipe closes, because sc dropped it or exited, the shell runs the builtin `kill -s KILL 0`: its whole group dies, itself included, and no - group id and no external program are involved. git runs through `command`, so a - shell function exported by the environment cannot stand in for it; `SC_GIT` is - resolved by `/bin/sh` the same way (a bare name through the PATH, a path as is). - The script uses a parenthesized subshell on purpose: with a `{ ...; } &` group - bash execs the command in place and the status line never runs. + group id and no external program are involved. git runs through `command --`; + `SC_GIT` is resolved by `/bin/sh` the same way (a bare name through the PATH, a + path as is). On macOS `/bin/sh` is bash, so the shell's own startup state is + removed from the leader's environment (and so from git's): `SHELLOPTS` (`monitor` + would move the background job into another process group, `errexit` would skip + the status line), `BASHOPTS`, `BASH_ENV`, `ENV`, and every `BASH_FUNC_*` + exported function, which would otherwise override `command`, `echo`, `read` or + `kill`. With that, shell functions cannot stand in for git. The script also + tolerates `errexit` (`|| r=$?`, `|| :`). The script uses a parenthesized subshell + on purpose: with a `{ ...; } &` group bash execs the command in place and the + status line never runs. It was checked on `/bin/sh`, `dash`, `bash` and `zsh` + (exit code propagated, group killed at stdin close, also mid-run). - sc never waits for the leader before the group is dead. It learns git's exit from the status file (one to three ASCII digits and a newline, at most 255; anything else, such as a partial write, counts as not yet), so the leader stays @@ -281,10 +288,15 @@ under a watchdog: smart-HTTP v0/v1 helper chain included, dies when git exits and cannot write after the discard. After the reap the group is never signalled. `kill` is only the backup: without it a warning is printed once and the leader's own `kill 0` - still ends the group, unless the leader died early. + still ends the group, unless the leader died early. The backup is waited for at + most 2 s and then killed, and the leader gets 5 s to exit before it is killed + itself. Each serve-mode fetch costs an extra `sh`, a `kill` spawn and about + 20 ms of settle time. - A `/bin/sh` exit status of 126 or 127 means `command` could not run git and - gives the usual "cannot run git (...): install git or set SC_GIT" error. A git - killed by signal `n` reads as exit `128 + n`. + gives the usual "cannot run git (...): install git or set SC_GIT" error, with + git's own message. A missing `/bin/sh` is reported as that. A git killed by + signal `n` reads as exit `128 + n`, which only the text of a fetch run by the + leader says. - What escapes, and what is trusted. A process that leaves git's group with `setsid` (an ssh `ControlPersist` master, a credential-cache daemon) is not killed. If it holds a pipe, the fetch is an error after the shared deadline, and @@ -319,9 +331,12 @@ The exact limits: helper chain that writes the pack (`git-remote-https`, `index-pack`), which can keep writing after git is killed. Those cases are not disk-bounded; the watchdog still frees the call. -- A fetch's group ends when `sc serve` dies, whatever the signal: the leader sees - the end of its stdin and kills the group. Only a process that left the group - outlives it. +- The leader kills the group when its stdin closes, which happens when sc drops the + pipe or exits, so a group normally ends with `sc serve`. If the leader itself is + killed first, only the backup `kill` remains, and a dead `sc serve` runs none. + On macOS a child spawned concurrently by another thread can briefly inherit the + stdin write end, which delays the end of input until it execs. A process that + left the group outlives all of this. ### Private-branch opt-in From 5ad85d06fa63bf7ddeda56de9426d330a60d6d04 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sun, 22 Nov 2026 11:28:48 +0200 Subject: [PATCH 5/9] refactor(objio): drop the external backup kill; the leader's builtin kill is the only group kill An external kill program waited on by sc kills only that program's own process, and signalling a group id after the leader is reaped could hit a reused id. The leader's `kill -s KILL 0`, run when its stdin closes, is now the one group kill. stop() closes the pipe, gives the leader 5 s to exit, then kills it alone; terminate() reaps it, settles and measures. Removed: kill_group, may_signal_group, KILL_WARNED, warn_once, Tuning.kill, the `reaped` parameters, and the tests that existed only for the backup (hanging kill, signalled-once ordering, missing kill, runaway-without-kill). The quiet-writer test now also asserts the leader is reaped when the call returns. A leader killed from outside first leaves its group's helpers running: documented in ADR-0060, THREAT-MODEL and CLAUDE.md, with every claim about a backup kill or a needed kill program removed. --- .claude/skills/sc-cli-reference/SKILL.md | 4 +- CLAUDE.md | 9 +- crates/objio/src/git.rs | 452 ++++---------------- docs/THREAT-MODEL.md | 31 +- docs/adr/0060-git-carried-bucket-remotes.md | 45 +- 5 files changed, 113 insertions(+), 428 deletions(-) diff --git a/.claude/skills/sc-cli-reference/SKILL.md b/.claude/skills/sc-cli-reference/SKILL.md index 89d67a19..d004605a 100644 --- a/.claude/skills/sc-cli-reference/SKILL.md +++ b/.claude/skills/sc-cli-reference/SKILL.md @@ -482,8 +482,8 @@ cargo run --bin sc -- serve --http --store sc+git:: [--al # fetch runs under a download watchdog (disk per # fetch; 120 s no-progress deadline); serve mode # runs git under a /bin/sh group leader and kills - # its group when the fetch ends (a `kill` on the - # PATH is only a backup) + # its group when the fetch ends (no `kill` program + # needed) bash demo/run_git_carrier_demo.sh # sc+git:: proof (P39): a bare Git repo as the host; # protected path + secret + private branch pushed # (private branch refused without the opt-in), plain diff --git a/CLAUDE.md b/CLAUDE.md index aa72b663..0666c9e2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -272,11 +272,10 @@ transport-adjacent. The rest, imperatively: normal exit or any breach) sc closes the pipe and the leader kills the whole group with its builtin `kill`, so every helper that stays in git's process group, including one that outlives git, dies (the group also ends when `sc serve` - exits, since the leader's stdin closes). The - system `kill` signals the group too as a backup, only while the leader is - unreaped (`may_signal_group`: sc never waits for the leader before the group is - dead; after the final reap the id may be reused, so it is never signalled); - without a `kill` it warns once. 126 or 127 from `/bin/sh` is the "cannot run + exits, since the leader's stdin closes). That builtin is the only group kill: sc + runs no external `kill` and signals no group id. If the leader is killed from + outside first, nothing kills the group: the fetch fails and its helpers may + survive. 126 or 127 from `/bin/sh` is the "cannot run git" error. A helper that leaves the group (`setsid`: an ssh `ControlPersist` master, a credential-cache daemon) escapes; the status file trusts local executables; `SC_GIT` is resolved by `/bin/sh` through `command --`, with diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index 4d9ca8ef..9b0be0bf 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -336,10 +336,6 @@ struct Tuning { /// Fixed polling interval; `None` polls after 2 ms and backs off to 50 ms /// (tests fix it to make a fetch finish between two polls). poll: Option, - /// The program that signals a process group in serve mode, as a backup to - /// the supervisor's own kill (tests point it at a missing one). - #[cfg(unix)] - kill: &'static str, /// Variables put on the serve-mode leader's command before its environment /// is scrubbed, standing in for a hostile parent environment (tests only). #[cfg(unix)] @@ -354,8 +350,6 @@ impl Default for Tuning { idle: IDLE, poll: None, #[cfg(unix)] - kill: "kill", - #[cfg(unix)] leader_env: &[], } } @@ -585,68 +579,6 @@ fn remove_stale_locks(dir: &Path, depth: u32) { } } -/// Whether a watched git's process group may be signalled: only while the leader -/// is unreaped. Its pid pins the group id then, so the signal can only reach our -/// own group. Once the leader is reaped, the group's id is free to be reused by -/// an unrelated process group of the same user, and nothing observable can show -/// it has not been. Holding the pid without reaping it needs `waitid(WNOWAIT)`, -/// which is `unsafe`, so after the reap the group is never signalled. -/// -/// In serve mode on unix the leader is the `/bin/sh` supervisor ([`LEADER`]), -/// and sc never reaps it before the group is dead: git's exit is learned from -/// the status file, not from `wait`. So the rule is always met until the one -/// final reap, and every helper that stays in git's process group, including -/// one that outlives git, is killed. The signal here is the backup to the -/// leader's own `kill 0`, which needs no group id and no external program. A -/// process that leaves the group (`setsid`: an ssh `ControlPersist` master, a -/// credential-cache daemon) escapes both. -fn may_signal_group(leader_reaped: bool) -> bool { - !leader_reaped -} - -/// Signal every process of group `pgid` with SIGKILL through the system `kill`. -/// False only when that program cannot be run; a non-zero exit (the group is -/// already gone) is fine. The program is waited for at most 2 s and then killed, -/// so a `kill` that hangs cannot park the caller. -#[cfg(unix)] -fn kill_group(program: &str, pgid: u32) -> bool { - let Ok(mut c) = Command::new(program) - .args(["-s", "KILL", "--"]) - .arg(format!("-{pgid}")) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - else { - return false; - }; - let end = Instant::now() + Duration::from_secs(2); - while Instant::now() < end { - if !matches!(c.try_wait(), Ok(None)) { - return true; - } - std::thread::sleep(Duration::from_millis(2)); - } - let _ = c.kill(); - let _ = c.wait(); - true -} - -/// Set once a missing `kill` has been reported. -#[cfg(unix)] -static KILL_WARNED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); - -/// Print `msg` to stderr unless `flag` says it was already printed; true when it -/// printed. -#[cfg(unix)] -fn warn_once(flag: &std::sync::atomic::AtomicBool, msg: &str) -> bool { - let first = !flag.swap(true, std::sync::atomic::Ordering::Relaxed); - if first { - eprintln!("{msg}"); - } - first -} - /// Read up to `cap` bytes from `r` and discard the rest, so the writer never /// blocks on a full pipe. The flag says more than `cap` bytes came. fn drain(r: &mut impl Read, cap: usize) -> (Vec, bool) { @@ -835,7 +767,7 @@ impl GitBucket { /// if missing) instead of the system temp directory, and on unix every fetch /// runs under a `/bin/sh` supervisor that leads its own process group, which /// is killed when the fetch ends, so `ssh`, `git-remote-https` and any helper - /// that stays in the group die with git (a system `kill` is only a backup). The + /// that stays in the group die with git. The /// bucket holds a lock on `/.owner.lock` until it is dropped. A killed /// server can leave the cache behind; [`GitBucket::reap_stale`] removes such /// leftovers at the next start. @@ -1064,7 +996,7 @@ impl GitBucket { let base: u64 = before.values().sum(); // Serve mode on unix: `/bin/sh` leads git's group and holds its stdin // pipe ([`LEADER`]); git's exit comes from the status file, because the - // leader must stay unreaped until its group is dead. + // leader stays until its stdin closes. let status_file = self.leads().then(|| StatusFile::new(&self.cache)); let mut cmd = match &status_file { Some(sf) => { @@ -1118,8 +1050,8 @@ impl GitBucket { let mut breach = None; let exit = loop { if let Some(sf) = &status_file { - // Never `wait` for the leader here: a reaped leader no longer - // pins its group's id. + // The leader does not exit when git does (it waits for its + // stdin to close), so git's exit comes from the status file. if let Some(code) = sf.read() { break Some(Exit { success: code == 0, @@ -1136,8 +1068,7 @@ impl GitBucket { } Ok(None) => {} Err(e) => { - // Whether the leader is reaped is unknown: no group signal. - self.stop(&mut child, &mut lead_in, false); + self.stop(&mut child, &mut lead_in); let _ = child.wait(); return Err(e.into()); } @@ -1168,23 +1099,15 @@ impl GitBucket { // Every breach ends in `terminate_and_discard`: the budget during the // run, the budget after git exited, no progress, a pipe that stays open. let Some(exit) = exit else { - self.terminate_and_discard( - &mut child, - &mut lead_in, - &mut readers, - &pack_dir, - &before, - false, - ); + self.terminate_and_discard(&mut child, &mut lead_in, &mut readers, &pack_dir, &before); return Err(breach.expect("only a breach leaves the loop without an exit")); }; if status_file.is_some() { - // Git is done and the leader is still unreaped: whatever is left in - // its group outlived git, so the group always ends here, the same - // way as on a breach. - self.terminate(&mut child, &mut lead_in, &mut readers, &pack_dir, false); + // Git is done: whatever is left in its group outlived git, so the + // group always ends here, the same way as on a breach. + self.terminate(&mut child, &mut lead_in, &mut readers, &pack_dir); } - // Git has exited and the leader is reaped. A holder of the pipes that + // Git has exited and the leader is gone. A holder of the pipes that // left the group (or, in the CLI, a helper that outlived git) may still // be writing, so keep measuring `objects/pack` for the one deadline both // pipes share, and once more after both have closed, before calling the @@ -1200,7 +1123,6 @@ impl GitBucket { &mut readers, &pack_dir, &before, - true, ); return Err(over_budget(watch)); } @@ -1211,14 +1133,7 @@ impl GitBucket { } }; if let Some(what) = held { - self.terminate_and_discard( - &mut child, - &mut lead_in, - &mut readers, - &pack_dir, - &before, - true, - ); + self.terminate_and_discard(&mut child, &mut lead_in, &mut readers, &pack_dir, &before); return Err(Error::Backend(format!( "git exited but left its {what} open for {:?} (a helper outlived it)", self.tune.idle @@ -1251,9 +1166,7 @@ impl GitBucket { /// The one end of every breach. [`Self::terminate`], then discard what the /// fetch added, only after the group has been given time to die and stop /// writing: discarding first would let a surviving helper write into the - /// directory after it was cleaned. `reaped` says whether the leader was - /// already reaped when the breach was found (the CLI, after git exited, and - /// serve mode after the group ended). + /// directory after it was cleaned. fn terminate_and_discard( &self, child: &mut Child, @@ -1261,25 +1174,22 @@ impl GitBucket { readers: &mut Readers, pack_dir: &Path, before: &HashMap, - reaped: bool, ) { - self.terminate(child, lead_in, readers, pack_dir, reaped); + self.terminate(child, lead_in, readers, pack_dir); self.discard_download(pack_dir, before); } - /// Stop git (its whole group in serve mode, but only while the leader is - /// unreaped: [`may_signal_group`]), reap it, then keep watching until the - /// pipes have closed and the pack directory has stopped changing, or a short - /// settle deadline passes. + /// Stop git (its whole group in serve mode), reap it, then keep watching + /// until the pipes have closed and the pack directory has stopped changing, + /// or a short settle deadline passes. fn terminate( &self, child: &mut Child, lead_in: &mut Option, readers: &mut Readers, pack_dir: &Path, - reaped: bool, ) { - self.stop(child, lead_in, may_signal_group(reaped)); + self.stop(child, lead_in); let _ = child.wait(); let settle = Instant::now() + self.tune.idle.min(SETTLE); let mut last = dir_bytes(pack_dir); @@ -1295,34 +1205,18 @@ impl GitBucket { } /// Stop a watched git. In serve mode (the child is the [`LEADER`]) closing - /// its stdin makes it kill the whole group, itself included, with no help - /// from outside; with `signal_group` (see [`may_signal_group`]) the system - /// `kill` also signals the group as a backup, which still works when the - /// leader died early. That backup is the only external program, and without - /// it a warning is printed once. The leader is then reaped within a bound. - /// Without a leader (the CLI) git is just killed. - fn stop( - &self, - child: &mut Child, - lead_in: &mut Option, - signal_group: bool, - ) { + /// its stdin makes it kill the whole group, itself included, with the shell + /// builtin: that is the only group kill, with no group id and no external + /// program. The leader gets [`SETTLE`] to exit and is then killed alone, so a + /// leader that cannot run leaves its group behind. A leader that died on its + /// own before it killed the group (killed from outside) also leaves the + /// group's helpers running. Without a leader (the CLI) git is just killed. + fn stop(&self, child: &mut Child, lead_in: &mut Option) { #[cfg(not(unix))] - let _ = (&lead_in, signal_group); // no process groups: only the child is killed + let _ = &lead_in; // no process groups: only the child is killed #[cfg(unix)] if self.group { drop(lead_in.take()); - if signal_group && !kill_group(self.tune.kill, child.id()) { - warn_once( - &KILL_WARNED, - &format!( - "sc: cannot run `{}` to signal a fetch's process group; the fetch's \ - supervisor still kills it, but a helper may survive a supervisor that \ - died early (put a `kill` program on the PATH)", - self.tune.kill - ), - ); - } let end = Instant::now() + SETTLE; while Instant::now() < end { if !matches!(child.try_wait(), Ok(None)) { @@ -4280,8 +4174,6 @@ mod tests { ); let sh = shim(&t, "finalover.sh", &body); let mut b = GitBucket::open_full(&url, sh, CHUNK, None, group).unwrap(); - let (kill, killed) = recording_kill(&t); - b.tune.kill = kill; b.tune.budget = Some(64 << 10); b.tune.poll = Some(Duration::from_millis(1500)); b.tune.idle = Duration::from_secs(3); @@ -4298,13 +4190,8 @@ mod tests { .parse() .unwrap(); if group { - assert!( - killed.exists(), - "serve mode signals the group at git's exit" - ); wait_gone(helper); } else { - assert!(!killed.exists(), "the CLI never signals a group"); let alive = Command::new("ps") .args(["-o", "stat=", "-p", &helper.to_string()]) .output() @@ -4317,88 +4204,6 @@ mod tests { } } - /// A `kill` stand-in that records its arguments and the state of the group - /// leader at that moment (`leader=` the `ps` state of the pid the group id - /// names, empty when it is gone; a zombie counts as there) and then really - /// kills: returns the program path and the log it appends to. - #[cfg(unix)] - fn recording_kill(t: &Tmp) -> (&'static str, PathBuf) { - let log = t.0.join("kill.log"); - let sh = shim_script( - t, - "killrec.sh", - &format!( - "echo \"$@ leader=$(ps -o stat= -p ${{4#-}} | tr -d ' ')\" >> {}\nexec kill \"$@\"\n", - log.display() - ), - ); - (Box::leak(sh.into_boxed_str()), log) - } - - /// The group is signalled once per fetch, while the leader is unreaped: at - /// git's exit. The fetch is then found over budget after the leader's reap, - /// and that breach must not signal the group again (its id may be reused), - /// whether a pipe is still held or not. The download is still discarded. - #[cfg(unix)] - #[test] - fn the_group_is_signalled_once_and_never_after_the_leader_is_reaped() { - need_git!(); - for helper_holds_pipes in [true, false] { - let t = Tmp::new("nosignal"); - let (_, url) = hub(&t); - let pids = t.0.join("pids"); - let helper = if helper_holds_pipes { - format!("sleep 600 & echo $! > {};", pids.display()) - } else { - String::new() - }; - // The shim writes too much late, then exits; with a helper the pipes stay open. - let body = format!( - "case \" $* \" in *\" fetch \"*) sleep 1; head -c 2000000 /dev/zero > \"$GIT_DIR/objects/pack/pack-fake.pack\"; {helper} exit 0;; esac" - ); - let sh = shim(&t, "nosignal.sh", &body); - let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); - let (kill, killed) = recording_kill(&t); - b.tune.kill = kill; - b.tune.budget = Some(64 << 10); - b.tune.idle = Duration::from_secs(3); - // git has exited at the second look, so the breach comes after it. - b.tune.poll = Some(Duration::from_secs(3)); - let e = get_err(&b, "manifest"); - assert!(e.contains("65536 bytes"), "{e}"); - assert!( - dir_files(&b.cache.join("objects").join("pack")).is_empty(), - "the over-budget download must be discarded" - ); - let calls = std::fs::read_to_string(&killed).unwrap_or_default(); - assert_eq!( - calls.lines().count(), - 1, - "one signal while the leader is unreaped, none after: {calls:?}" - ); - assert!( - !calls.trim_end().ends_with("leader="), - "the leader must still exist (unreaped) when it is signalled: {calls:?}" - ); - if helper_holds_pipes { - let pid: u32 = std::fs::read_to_string(&pids) - .unwrap() - .trim() - .parse() - .unwrap(); - wait_gone(pid); - } - } - } - - /// The signalling decision: only while the leader is unreaped, when its pid - /// pins the group id. After the reap, never. - #[test] - fn the_group_is_signalled_only_while_the_leader_is_unreaped() { - assert!(may_signal_group(false), "an unreaped leader pins the id"); - assert!(!may_signal_group(true), "a reaped leader pins nothing"); - } - /// A helper that outlives git can cross the budget after git has exited and /// close both pipes before the shared deadline. The watchdog must still catch /// it, and must not call the fetch good. In the CLI any helper can do this; in @@ -4454,38 +4259,44 @@ mod tests { /// afterwards and the directory stops growing. The download is discarded only /// where it crossed the budget (the second case: git exits at once, a fast /// writer crosses the budget before the first poll that sees the exit). The - /// writer acknowledges its first write before git exits. With the backup - /// `kill` missing the leader's own `kill 0` does the same. + /// writer acknowledges its first write before git exits. The call ends with + /// the leader reaped, not left a zombie for the pipe closing on return. #[cfg(unix)] #[test] fn a_helper_that_closes_its_pipes_and_keeps_writing_is_dead_after_git_exits() { need_git!(); - for (budget, crosses, backup) in [ - (32u64 << 20, false, true), - (1 << 20, true, true), - (32 << 20, false, false), - (1 << 20, true, false), - ] { + for (budget, crosses) in [(32u64 << 20, false), (1 << 20, true)] { let t = Tmp::new("quietwriter"); let (_, url) = hub(&t); let pids = t.0.join("pids"); + let leader = t.0.join("leader"); let ack = t.0.join("ack"); let pause = if crosses { "0.01" } else { "0.05" }; let body = format!( - "case \" $* \" in *\" fetch \"*) ( exec >/dev/null 2>&1; head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/pack-late.pack\"; echo > {ack}; while :; do head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/pack-late.pack\"; sleep {pause}; done ) & echo $! > {p}; until [ -e {ack} ]; do sleep 0.01; done; exit 0;; esac", + "case \" $* \" in *\" fetch \"*) ps -o pgid= -p $$ | tr -d ' ' > {leader}; ( exec >/dev/null 2>&1; head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/pack-late.pack\"; echo > {ack}; while :; do head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/pack-late.pack\"; sleep {pause}; done ) & echo $! > {p}; until [ -e {ack} ]; do sleep 0.01; done; exit 0;; esac", p = pids.display(), + leader = leader.display(), ack = ack.display() ); let sh = shim(&t, "quiet.sh", &body); let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); - let (kill, killed) = recording_kill(&t); - b.tune.kill = if backup { kill } else { "/no/such/kill" }; b.tune.budget = Some(budget); if crosses { b.tune.poll = Some(Duration::from_millis(1500)); } let pack = b.cache.join("objects").join("pack"); let (out, over) = watched_fetch(&b); + // Straight after the call, before anything waits: the leader is gone, + // not a zombie. + let leader_pid = std::fs::read_to_string(&leader).unwrap().trim().to_string(); + let left = Command::new("ps") + .args(["-o", "stat=", "-p", &leader_pid]) + .output() + .unwrap(); + assert!( + String::from_utf8_lossy(&left.stdout).trim().is_empty(), + "the leader must be reaped when the call returns" + ); let helper: u32 = std::fs::read_to_string(&pids) .unwrap() .trim() @@ -4496,10 +4307,6 @@ mod tests { std::thread::sleep(Duration::from_millis(400)); assert_eq!(dir_bytes(&pack), after_call, "the writer kept going"); assert!(no_status_files(&b)); - // The group is ended by git's exit, not left to the pipe closing when - // the call returns: the backup signal went out exactly once. - let calls = std::fs::read_to_string(&killed).unwrap_or_default(); - assert_eq!(calls.lines().count(), usize::from(backup), "{calls:?}"); if crosses { assert!(over && out.is_err(), "crossed the budget: an error"); assert!(dir_files(&pack).is_empty(), "and the download is discarded"); @@ -4511,37 +4318,6 @@ mod tests { } } - /// The group dies on a mid-run breach with no external `kill` at all: closing - /// the leader's stdin is what ends it. - #[cfg(unix)] - #[test] - fn a_runaway_fetch_dies_without_a_kill_program() { - need_git!(); - let t = Tmp::new("nokillrun"); - let (_, url) = hub(&t); - let pids = t.0.join("pids"); - let body = format!( - "case \" $* \" in *\" fetch \"*) sleep 600 & echo $! > {}; while :; do head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/tmp_pack_x\"; sleep 0.01; done;; esac", - pids.display() - ); - let sh = shim(&t, "runaway2.sh", &body); - let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); - b.tune.kill = "/no/such/kill"; - b.tune.budget = Some(1 << 20); - let e = get_err(&b, "manifest"); - assert!(e.contains("1048576 bytes"), "{e}"); - let helper: u32 = std::fs::read_to_string(&pids) - .unwrap() - .trim() - .parse() - .unwrap(); - wait_gone(helper); - let tmp_pack = b.cache.join("objects").join("pack").join("tmp_pack_x"); - std::thread::sleep(Duration::from_millis(300)); - assert!(!tmp_pack.exists(), "the writing loop must be dead too"); - assert!(no_status_files(&b)); - } - /// A parent environment that changes how a POSIX shell behaves must not reach /// the leader: `SHELLOPTS=errexit:monitor` (the status line and the final /// `kill` would be skipped, the background job would leave the group), @@ -4595,41 +4371,6 @@ mod tests { assert!(no_status_files(&b)); } - /// A `kill` program that hangs cannot park a breach: it is waited for a - /// bounded time and killed, and the leader's own `kill 0` still ends the - /// group. - #[cfg(unix)] - #[test] - fn a_hanging_kill_program_is_bounded() { - need_git!(); - let t = Tmp::new("hangkill"); - let (_, url) = hub(&t); - let pids = t.0.join("pids"); - let body = format!( - "case \" $* \" in *\" fetch \"*) sleep 600 & echo $! > {}; while :; do head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/tmp_pack_x\"; sleep 0.01; done;; esac", - pids.display() - ); - let sh = shim(&t, "hangrun.sh", &body); - let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); - let hang = shim_script(&t, "hangkill.sh", "exec sleep 600\n"); - b.tune.kill = Box::leak(hang.into_boxed_str()); - b.tune.budget = Some(1 << 20); - let started = Instant::now(); - let e = get_err(&b, "manifest"); - assert!(e.contains("1048576 bytes"), "{e}"); - assert!( - started.elapsed() < Duration::from_secs(20), - "{:?}", - started.elapsed() - ); - let helper: u32 = std::fs::read_to_string(&pids) - .unwrap() - .trim() - .parse() - .unwrap(); - wait_gone(helper); - } - /// git's exit code reaches the caller through the leader (a failing fetch is /// `!success` with that code), and a git killed by a signal reads as 128 + n. /// The CLI reports the same codes, and no code for a signal. @@ -4726,50 +4467,41 @@ mod tests { /// A leader that dies on its own (here a shim that kills the supervisor and /// the subshell holding git's status line) never produces a status. That is - /// a bounded error through the no-progress deadline, and the group is still - /// killed from outside while the dead leader is unreaped. With no `kill` - /// program that backup is gone and the helper survives, the documented cost. + /// a bounded error through the no-progress deadline. The leader was the only + /// thing that kills the group, so the group's helpers survive: the + /// documented limit (the test kills the helper). #[cfg(unix)] #[test] fn a_leader_that_dies_early_is_a_bounded_error() { need_git!(); - for with_kill in [true, false] { - let t = Tmp::new("leaderdies"); - let (_, url) = hub(&t); - let pids = t.0.join("pids"); - let body = format!( - "case \" $* \" in *\" fetch \"*) l=$(ps -o pgid= -p $$ | tr -d ' '); kill -s KILL $PPID $l; sleep 600 & echo $! > {}; wait;; esac", - pids.display() - ); - let sh = shim(&t, "orphan.sh", &body); - let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); - b.tune.idle = Duration::from_millis(800); - if !with_kill { - b.tune.kill = "/no/such/kill"; - } - let started = Instant::now(); - let e = get_err(&b, "manifest"); - assert!(e.contains("no progress"), "{e}"); - assert!(started.elapsed() < Duration::from_secs(30)); - assert!(no_status_files(&b)); - let helper: u32 = std::fs::read_to_string(&pids) - .unwrap() - .trim() - .parse() - .unwrap(); - if with_kill { - wait_gone(helper); - } else { - let alive = Command::new("ps") - .args(["-o", "stat=", "-p", &helper.to_string()]) - .output() - .unwrap(); - let stat = String::from_utf8_lossy(&alive.stdout).trim().to_string(); - assert!(!stat.is_empty() && !stat.starts_with('Z'), "stat {stat:?}"); - kill_pid(helper); - wait_gone(helper); - } - } + let t = Tmp::new("leaderdies"); + let (_, url) = hub(&t); + let pids = t.0.join("pids"); + let body = format!( + "case \" $* \" in *\" fetch \"*) l=$(ps -o pgid= -p $$ | tr -d ' '); kill -s KILL $PPID $l; sleep 600 & echo $! > {}; wait;; esac", + pids.display() + ); + let sh = shim(&t, "orphan.sh", &body); + let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); + b.tune.idle = Duration::from_millis(800); + let started = Instant::now(); + let e = get_err(&b, "manifest"); + assert!(e.contains("no progress"), "{e}"); + assert!(started.elapsed() < Duration::from_secs(30)); + assert!(no_status_files(&b)); + let helper: u32 = std::fs::read_to_string(&pids) + .unwrap() + .trim() + .parse() + .unwrap(); + let alive = Command::new("ps") + .args(["-o", "stat=", "-p", &helper.to_string()]) + .output() + .unwrap(); + let stat = String::from_utf8_lossy(&alive.stdout).trim().to_string(); + assert!(!stat.is_empty() && !stat.starts_with('Z'), "stat {stat:?}"); + kill_pid(helper); + wait_gone(helper); } /// The status line is read strictly: one to three digits and a newline, at @@ -4850,44 +4582,6 @@ mod tests { assert!(e.contains("left its stderr open"), "{e}"); } - /// A missing `kill` program is reported once, and only costs the backup - /// signal: the leader still kills the group on its own, so a fetch that - /// stalls returns and its helper is dead. - #[cfg(unix)] - #[test] - fn a_missing_kill_program_warns_once_and_the_leader_still_kills_the_group() { - need_git!(); - let flag = std::sync::atomic::AtomicBool::new(false); - assert!(warn_once(&flag, "first"), "the first report prints"); - assert!(!warn_once(&flag, "second"), "later reports stay quiet"); - assert!( - !kill_group("/no/such/kill", 1), - "an unrunnable program is reported" - ); - - let t = Tmp::new("nokill"); - let (_, url) = hub(&t); - let pids = t.0.join("pids"); - let body = format!( - "case \" $* \" in *\" fetch \"*) sleep 600 & echo $! > {}; wait;; esac", - pids.display() - ); - let sh = shim(&t, "nokill.sh", &body); - let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); - b.tune.idle = Duration::from_millis(800); - b.tune.kill = "/no/such/kill"; - let started = Instant::now(); - let e = get_err(&b, "manifest"); - assert!(e.contains("no progress"), "{e}"); - assert!(started.elapsed() < Duration::from_secs(90)); - let helper: u32 = std::fs::read_to_string(&pids) - .unwrap() - .trim() - .parse() - .unwrap(); - wait_gone(helper); - } - /// A store whose `log/`, `checkpoints/` or `packs/*.idx` family passes the /// family budget must stay readable, or it could never be compacted. The /// family fetch is discarded, the requested key is read with a fetch of its diff --git a/docs/THREAT-MODEL.md b/docs/THREAT-MODEL.md index ef52fd5b..18442434 100644 --- a/docs/THREAT-MODEL.md +++ b/docs/THREAT-MODEL.md @@ -771,12 +771,9 @@ The bucket model above applies unchanged, with the Git repository as the bucket after it exits, and the measuring goes on while a helper still holds git's pipes and once more after they close. Every breach (budget, budget after exit, no progress, a pipe held open after exit) stops git, waits up to 5 s for helpers to go and the directory to settle, then - discards; in serve mode a normal exit takes the same stop and settle steps first. The - process group is signalled only while its leader is unreaped, when the leader's pid pins the - group id; after the reap it is never signalled, because the id may have been reused by an - unrelated process group of the same user and nothing observable shows otherwise (pinning the - pid without reaping needs `unsafe`). In serve mode the leader is a `/bin/sh` supervisor that - sc never waits for before the group is dead, so that is always the case until the final reap. + discards; in serve mode a normal exit takes the same stop and settle steps first. sc runs + no external `kill` and signals no group id, so a reused group id cannot be hit: in serve mode + the group is killed by its `/bin/sh` supervisor's own builtin when its stdin closes. The limits, plainly: - The bound is disk bytes per fetch. It does not cover git's memory while `index-pack` resolves a pack, and it does not cover the total size of the cache, which grows by what each @@ -795,16 +792,16 @@ The bucket model above applies unchanged, with the Git repository as the bucket a `/bin/sh` supervisor that leads its process group, holds a stdin pipe only sc has, and writes git's exit code to a status file in the private cache directory. When the fetch ends (a normal exit or any breach) sc closes that pipe and the supervisor kills the whole group - with its builtin `kill`, itself included; the system `kill` signals the group too, as a - backup while the supervisor is unreaped (without one the watchdog warns once). So `ssh`, - `git-remote-https`, `index-pack` and any helper that outlives git, a smart-HTTP v0/v1 - helper chain included, die and cannot write after the discard. The group also ends when - `sc serve` exits, because the supervisor's stdin closes (if the supervisor itself was - killed first only the backup `kill` remains, and a child spawned at the same moment can - briefly hold the pipe open on macOS). Outside serve mode (the unix CLI, Windows) the kill may not reach the + with its builtin `kill`, itself included, so `ssh`, `git-remote-https`, `index-pack` and + any helper that outlives git, a smart-HTTP v0/v1 helper chain included, die and cannot + write after the discard. The group also ends when `sc serve` exits, because the + supervisor's stdin closes (a child spawned at the same moment can briefly hold the pipe + open on macOS). If the supervisor itself was killed from outside first, nothing kills the + group: the fetch fails (no status, then the no-progress deadline) and helpers still in the + group may survive. Outside serve mode (the unix CLI, Windows) the kill may not reach the helper chain that writes the pack, which can keep writing after git is killed, and a helper - that outlives git is not killed: those cases are not disk-bounded. The call still returns, since detached pipe - readers keep a surviving helper from blocking it. + that outlives git is not killed: those cases are not disk-bounded. The call still returns, + since detached pipe readers keep a surviving helper from blocking it. - A process that leaves git's process group (`setsid`: an ssh `ControlPersist` master, a credential-cache daemon, a hostile process) escapes serve mode's kill. If it holds git's pipes the fetch fails after the shared deadline; if it closes them and writes into @@ -814,8 +811,8 @@ The bucket model above applies unchanged, with the Git repository as the bucket through `command --` with the `SC_GIT` program resolved by `/bin/sh` (a bare name through the PATH, a path as is). The supervisor's environment loses `SHELLOPTS`, `BASHOPTS`, `BASH_ENV`, `ENV` and every `BASH_FUNC_*`, so a hostile parent environment cannot change the shell's - behavior or override its builtins with exported functions. Each fetch costs an extra `sh`, a - `kill` spawn and about 20 ms. + behavior or override its builtins with exported functions. Each fetch costs an extra `sh` and + about 20 ms. - Only fetches are watched. `ls-remote` and push run unwatched: over HTTP `http.lowSpeedLimit`/`lowSpeedTime` bound them, over ssh the operator should set `ServerAliveInterval` for the daemon's ssh user. diff --git a/docs/adr/0060-git-carried-bucket-remotes.md b/docs/adr/0060-git-carried-bucket-remotes.md index 859f56c3..d8a5cbbb 100644 --- a/docs/adr/0060-git-carried-bucket-remotes.md +++ b/docs/adr/0060-git-carried-bucket-remotes.md @@ -272,26 +272,19 @@ under a watchdog: on purpose: with a `{ ...; } &` group bash execs the command in place and the status line never runs. It was checked on `/bin/sh`, `dash`, `bash` and `zsh` (exit code propagated, group killed at stdin close, also mid-run). -- sc never waits for the leader before the group is dead. It learns git's exit - from the status file (one to three ASCII digits and a newline, at most 255; - anything else, such as a partial write, counts as not yet), so the leader stays - unreaped and its pid keeps pinning the group id. A reaped leader would free the - id for reuse by an unrelated process group of the same user, and pinning a pid - without reaping it needs `waitid(WNOWAIT)`, which is `unsafe` and not allowed - here. A leader that dies early writes no status; the no-progress deadline ends - the fetch. Every end of a fetch, a normal exit or any breach, takes the same - steps: close the stdin pipe; signal the group through the system `kill` as a - backup while the leader is still unreaped (`may_signal_group`; it also reaches - helpers when the leader died before it saw the end of input); reap the leader, - within a bound; then settle and measure as below. After a successful fetch the - group is killed too, so every helper that stays in git's process group, a - smart-HTTP v0/v1 helper chain included, dies when git exits and cannot write - after the discard. After the reap the group is never signalled. `kill` is only - the backup: without it a warning is printed once and the leader's own `kill 0` - still ends the group, unless the leader died early. The backup is waited for at - most 2 s and then killed, and the leader gets 5 s to exit before it is killed - itself. Each serve-mode fetch costs an extra `sh`, a `kill` spawn and about - 20 ms of settle time. +- sc learns git's exit from the status file (one to three ASCII digits and a + newline, at most 255; anything else, such as a partial write, counts as not + yet), because the leader does not exit when git does: it waits for its stdin. A + leader that dies early writes no status; the no-progress deadline ends the + fetch. Every end of a fetch, a normal exit or any breach, takes the same steps: + close the stdin pipe, which makes the leader kill the group; give the leader 5 s + to exit, then kill it alone; reap it; then settle and measure as below. After a + successful fetch the group is killed too, so every helper that stays in git's + process group, a smart-HTTP v0/v1 helper chain included, dies when git exits and + cannot write after the discard. The leader's builtin `kill` is the only group + kill: sc runs no external `kill` program and signals no group id, so a reused + group id cannot be hit and the platform needs no `kill` on the PATH. Each + serve-mode fetch costs an extra `sh` and about 20 ms of settle time. - A `/bin/sh` exit status of 126 or 127 means `command` could not run git and gives the usual "cannot run git (...): install git or set SC_GIT" error, with git's own message. A missing `/bin/sh` is reported as that. A git killed by @@ -299,7 +292,10 @@ under a watchdog: leader says. - What escapes, and what is trusted. A process that leaves git's group with `setsid` (an ssh `ControlPersist` master, a credential-cache daemon) is not - killed. If it holds a pipe, the fetch is an error after the shared deadline, and + killed, and neither are the helpers still in the group when the leader died + before it killed the group (it was killed from outside, so the fetch ends by the + no-progress deadline or an early exit at the final wait): the fetch fails and + those helpers may survive. That is the one case without a second line of defense. If it holds a pipe, the fetch is an error after the shared deadline, and the discard follows; if it closes its pipes and writes into `objects/pack`, the watchdog bounds disk only while it measures. The status file is a channel that trusts local executables: any process of the same user can write it (the cache @@ -333,10 +329,9 @@ The exact limits: watchdog still frees the call. - The leader kills the group when its stdin closes, which happens when sc drops the pipe or exits, so a group normally ends with `sc serve`. If the leader itself is - killed first, only the backup `kill` remains, and a dead `sc serve` runs none. - On macOS a child spawned concurrently by another thread can briefly inherit the - stdin write end, which delays the end of input until it execs. A process that - left the group outlives all of this. + killed first, nothing kills the group. On macOS a child spawned concurrently by + another thread can briefly inherit the stdin write end, which delays the end of + input until it execs. A process that left the group outlives all of this. ### Private-branch opt-in From 82631376b13dd7295f810799da1005be54c2277e Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sun, 22 Nov 2026 11:47:45 +0200 Subject: [PATCH 6/9] fix(objio): require the leader to end by its own group kill, and bound its reap The status line can say 0 after the leader was killed from outside, with a helper that closed its pipes still writing. stop() now checks that the leader is still running right before closing its stdin, and then requires it to end by SIGKILL, its own `kill -s KILL 0`. A leader that had already exited, exits any other way, or is still there after the 5 s grace makes the fetch an error even when git succeeded, and the download is discarded. The slow leader is killed alone and waited for at most 2 s more, then left unreaped instead of blocking. Tests: a leader killed from outside after a successful fetch with a quiet writer, and a leader (through a new Tuning.leader seam) that exits normally or ignores the end of input. Two timing-fragile watchdog tests now put the pid file first and make the budget crossing deterministic. Docs describe the seen-to-end-by-its-own-kill rule instead of the reaping order. --- CLAUDE.md | 7 +- crates/objio/src/git.rs | 149 +++++++++++++++++--- docs/THREAT-MODEL.md | 7 +- docs/adr/0060-git-carried-bucket-remotes.md | 21 ++- 4 files changed, 153 insertions(+), 31 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0666c9e2..aca2cbb6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -273,9 +273,10 @@ transport-adjacent. The rest, imperatively: group with its builtin `kill`, so every helper that stays in git's process group, including one that outlives git, dies (the group also ends when `sc serve` exits, since the leader's stdin closes). That builtin is the only group kill: sc - runs no external `kill` and signals no group id. If the leader is killed from - outside first, nothing kills the group: the fetch fails and its helpers may - survive. 126 or 127 from `/bin/sh` is the "cannot run + runs no external `kill` and signals no group id. The leader must be seen running + when its stdin closes and to end by SIGKILL (its own group kill); otherwise + (killed from outside first, another exit, or still there after 5 s) the fetch is + an error even if git succeeded, and its helpers may survive. 126 or 127 from `/bin/sh` is the "cannot run git" error. A helper that leaves the group (`setsid`: an ssh `ControlPersist` master, a credential-cache daemon) escapes; the status file trusts local executables; `SC_GIT` is resolved by `/bin/sh` through `command --`, with diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index 9b0be0bf..d158cd62 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -340,6 +340,10 @@ struct Tuning { /// is scrubbed, standing in for a hostile parent environment (tests only). #[cfg(unix)] leader_env: &'static [(&'static str, &'static str)], + /// Replaces [`LEADER`] (tests only: a leader that does not end by its own + /// group kill). + #[cfg(unix)] + leader: Option<&'static str>, } impl Default for Tuning { @@ -351,6 +355,8 @@ impl Default for Tuning { poll: None, #[cfg(unix)] leader_env: &[], + #[cfg(unix)] + leader: None, } } } @@ -1001,7 +1007,11 @@ impl GitBucket { let mut cmd = match &status_file { Some(sf) => { let mut sh = Command::new("/bin/sh"); - sh.args(["-c", LEADER, "sc-fetch"]) + #[cfg(unix)] + let script = self.tune.leader.unwrap_or(LEADER); + #[cfg(not(unix))] + let script = LEADER; + sh.args(["-c", script, "sc-fetch"]) .arg(&sf.0) .arg(&self.git); let mut sh = self.hardened(sh, args, &[], true); @@ -1102,10 +1112,11 @@ impl GitBucket { self.terminate_and_discard(&mut child, &mut lead_in, &mut readers, &pack_dir, &before); return Err(breach.expect("only a breach leaves the loop without an exit")); }; + let mut leader_ended_group = true; if status_file.is_some() { // Git is done: whatever is left in its group outlived git, so the // group always ends here, the same way as on a breach. - self.terminate(&mut child, &mut lead_in, &mut readers, &pack_dir); + leader_ended_group = self.terminate(&mut child, &mut lead_in, &mut readers, &pack_dir); } // Git has exited and the leader is gone. A holder of the pipes that // left the group (or, in the CLI, a helper that outlived git) may still @@ -1139,6 +1150,16 @@ impl GitBucket { self.tune.idle ))); } + if !leader_ended_group { + // The status file may say 0, but the group was not killed by its + // leader, so a helper with closed pipes can still be writing. + self.discard_download(&pack_dir, &before); + return Err(Error::Backend( + "git's supervisor did not end its process group (it was killed or \ + stuck); the fetch was discarded and a helper may still be running" + .into(), + )); + } let (stdout, over) = readers.got_out.take().unwrap_or_default(); let stderr = readers.got_err.take().unwrap_or_default(); // `command` reports a git it cannot find or start as 127 or 126. @@ -1175,22 +1196,25 @@ impl GitBucket { pack_dir: &Path, before: &HashMap, ) { - self.terminate(child, lead_in, readers, pack_dir); + let _ = self.terminate(child, lead_in, readers, pack_dir); self.discard_download(pack_dir, before); } - /// Stop git (its whole group in serve mode), reap it, then keep watching - /// until the pipes have closed and the pack directory has stopped changing, - /// or a short settle deadline passes. + /// Stop git (its whole group in serve mode), then keep watching until the + /// pipes have closed and the pack directory has stopped changing, or a short + /// settle deadline passes. False when, in serve mode, the leader did not end + /// by its own group kill ([`Self::stop`]). fn terminate( &self, child: &mut Child, lead_in: &mut Option, readers: &mut Readers, pack_dir: &Path, - ) { - self.stop(child, lead_in); - let _ = child.wait(); + ) -> bool { + let ended = self.stop(child, lead_in); + if !self.leads() { + let _ = child.wait(); + } let settle = Instant::now() + self.tune.idle.min(SETTLE); let mut last = dir_bytes(pack_dir); loop { @@ -1202,30 +1226,47 @@ impl GitBucket { } last = now; } + ended } /// Stop a watched git. In serve mode (the child is the [`LEADER`]) closing /// its stdin makes it kill the whole group, itself included, with the shell /// builtin: that is the only group kill, with no group id and no external - /// program. The leader gets [`SETTLE`] to exit and is then killed alone, so a - /// leader that cannot run leaves its group behind. A leader that died on its - /// own before it killed the group (killed from outside) also leaves the - /// group's helpers running. Without a leader (the CLI) git is just killed. - fn stop(&self, child: &mut Child, lead_in: &mut Option) { + /// program. The safety argument is what is seen: the leader must still be + /// running when its stdin is dropped, and must then end by SIGKILL, its own + /// group kill. True only then. A leader that had already exited (killed from + /// outside, so its group's helpers were never killed), that exits any other + /// way, or that does not exit within [`SETTLE`] returns false; the latter is + /// killed alone and waited for at most 2 s more, then left unreaped. Without + /// a leader (the CLI) git is just killed. + fn stop(&self, child: &mut Child, lead_in: &mut Option) -> bool { #[cfg(not(unix))] let _ = &lead_in; // no process groups: only the child is killed #[cfg(unix)] if self.group { + use std::os::unix::process::ExitStatusExt; + let early = !matches!(child.try_wait(), Ok(None)); drop(lead_in.take()); + if early { + return false; + } let end = Instant::now() + SETTLE; while Instant::now() < end { - if !matches!(child.try_wait(), Ok(None)) { - return; + match child.try_wait() { + Ok(None) => std::thread::sleep(Duration::from_millis(2)), + Ok(Some(st)) => return st.signal() == Some(9), + Err(_) => return false, } + } + let _ = child.kill(); + let end = Instant::now() + Duration::from_secs(2); + while Instant::now() < end && matches!(child.try_wait(), Ok(None)) { std::thread::sleep(Duration::from_millis(2)); } + return false; } let _ = child.kill(); + true } /// Undo a fetch that went over budget: remove every file it added to @@ -4169,7 +4210,7 @@ mod tests { let (_, url) = hub(&t); let pids = t.0.join("pids"); let body = format!( - "case \" $* \" in *\" fetch \"*) sleep 0.4; head -c 2000000 /dev/zero > \"$GIT_DIR/objects/pack/pack-fake.pack\"; sleep 600 & echo $! > {}; exit 0;; esac", + "case \" $* \" in *\" fetch \"*) sleep 600 & echo $! > {}; sleep 0.4; head -c 2000000 /dev/zero > \"$GIT_DIR/objects/pack/pack-fake.pack\"; exit 0;; esac", pids.display() ); let sh = shim(&t, "finalover.sh", &body); @@ -4271,9 +4312,15 @@ mod tests { let pids = t.0.join("pids"); let leader = t.0.join("leader"); let ack = t.0.join("ack"); - let pause = if crosses { "0.01" } else { "0.05" }; + // Crossing is deterministic, not a race against the writer's speed: the + // first write alone is over the budget. + let (pause, first) = if crosses { + ("0.01", 2_000_000) + } else { + ("0.05", 65536) + }; let body = format!( - "case \" $* \" in *\" fetch \"*) ps -o pgid= -p $$ | tr -d ' ' > {leader}; ( exec >/dev/null 2>&1; head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/pack-late.pack\"; echo > {ack}; while :; do head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/pack-late.pack\"; sleep {pause}; done ) & echo $! > {p}; until [ -e {ack} ]; do sleep 0.01; done; exit 0;; esac", + "case \" $* \" in *\" fetch \"*) ps -o pgid= -p $$ | tr -d ' ' > {leader}; ( exec >/dev/null 2>&1; head -c {first} /dev/zero >> \"$GIT_DIR/objects/pack/pack-late.pack\"; echo > {ack}; while :; do head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/pack-late.pack\"; sleep {pause}; done ) & echo $! > {p}; until [ -e {ack} ]; do sleep 0.01; done; exit 0;; esac", p = pids.display(), leader = leader.display(), ack = ack.display() @@ -4504,6 +4551,70 @@ mod tests { wait_gone(helper); } + /// The status line can say 0 while the leader was killed from outside, so it + /// never ran its group kill and a helper with closed pipes keeps writing. That + /// fetch is an error, not a success (and the download is discarded). The + /// helper survives: the documented limit (the test kills it). + #[cfg(unix)] + #[test] + fn a_leader_killed_from_outside_makes_a_successful_fetch_an_error() { + need_git!(); + let t = Tmp::new("leaderkilled"); + let (_, url) = hub(&t); + let pids = t.0.join("pids"); + let ack = t.0.join("ack"); + let body = format!( + "case \" $* \" in *\" fetch \"*) ( exec >/dev/null 2>&1; while :; do head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/pack-late.pack\"; echo > {ack}; sleep 0.05; done ) & echo $! > {p}; until [ -e {ack} ]; do sleep 0.01; done; l=$(ps -o pgid= -p $$ | tr -d ' '); kill -s KILL $l; exit 0;; esac", + p = pids.display(), + ack = ack.display() + ); + let sh = shim(&t, "killedleader.sh", &body); + let b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); + let (out, over) = watched_fetch(&b); + let e = out.err().expect("not a success").to_string(); + assert!(!over && e.contains("supervisor"), "{e}"); + assert!(no_status_files(&b)); + let helper: u32 = std::fs::read_to_string(&pids) + .unwrap() + .trim() + .parse() + .unwrap(); + kill_pid(helper); + wait_gone(helper); + } + + /// After its stdin closes the leader must end by its own group kill, which is + /// a SIGKILL: one that exits normally, or ignores the end of input and has to + /// be killed after the grace period, makes the fetch an error even though git + /// succeeded. + #[cfg(unix)] + #[test] + fn a_leader_that_does_not_end_by_its_own_group_kill_is_an_error() { + need_git!(); + for (replacement, slow) in [("exit 0", false), ("exec sleep 600", true)] { + let t = Tmp::new("leaderother"); + let (_, url) = hub(&t); + let sh = shim(&t, "ok.sh", "case \" $* \" in *\" fetch \"*) exit 0;; esac"); + let mut b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); + b.tune.leader = Some(Box::leak( + LEADER + .replace("kill -s KILL 0", replacement) + .into_boxed_str(), + )); + let started = Instant::now(); + let e = watched_fetch(&b) + .0 + .err() + .expect("not a success") + .to_string(); + assert!(e.contains("supervisor"), "{e}"); + let took = started.elapsed(); + assert!(took < Duration::from_secs(30), "{took:?}"); + assert_eq!(took >= SETTLE, slow, "{replacement}: {took:?}"); + assert!(no_status_files(&b)); + } + } + /// The status line is read strictly: one to three digits and a newline, at /// most 255. Anything else is not yet a status. #[test] diff --git a/docs/THREAT-MODEL.md b/docs/THREAT-MODEL.md index 18442434..2bbe44cd 100644 --- a/docs/THREAT-MODEL.md +++ b/docs/THREAT-MODEL.md @@ -796,9 +796,10 @@ The bucket model above applies unchanged, with the Git repository as the bucket any helper that outlives git, a smart-HTTP v0/v1 helper chain included, die and cannot write after the discard. The group also ends when `sc serve` exits, because the supervisor's stdin closes (a child spawned at the same moment can briefly hold the pipe - open on macOS). If the supervisor itself was killed from outside first, nothing kills the - group: the fetch fails (no status, then the no-progress deadline) and helpers still in the - group may survive. Outside serve mode (the unix CLI, Windows) the kill may not reach the + open on macOS). The supervisor must be seen running when its stdin is closed and to end by + SIGKILL, its own group kill; if it was killed from outside first, or ends any other way, + nothing killed the group, so the fetch is an error even if git succeeded (the download is + discarded), and helpers still in the group may survive. Outside serve mode (the unix CLI, Windows) the kill may not reach the helper chain that writes the pack, which can keep writing after git is killed, and a helper that outlives git is not killed: those cases are not disk-bounded. The call still returns, since detached pipe readers keep a surviving helper from blocking it. diff --git a/docs/adr/0060-git-carried-bucket-remotes.md b/docs/adr/0060-git-carried-bucket-remotes.md index d8a5cbbb..a4cf70ea 100644 --- a/docs/adr/0060-git-carried-bucket-remotes.md +++ b/docs/adr/0060-git-carried-bucket-remotes.md @@ -277,13 +277,21 @@ under a watchdog: yet), because the leader does not exit when git does: it waits for its stdin. A leader that dies early writes no status; the no-progress deadline ends the fetch. Every end of a fetch, a normal exit or any breach, takes the same steps: - close the stdin pipe, which makes the leader kill the group; give the leader 5 s - to exit, then kill it alone; reap it; then settle and measure as below. After a - successful fetch the group is killed too, so every helper that stays in git's + check that the leader is still running (one that has already exited never ran + its group kill), close the stdin pipe, which makes the leader kill the group, and + require the leader to end by SIGKILL, its own group kill. A leader that had + already exited, that exits any other way, or that is still there after 5 s makes + the fetch an error even when the status file says 0, because a helper with closed + pipes may still be writing; the download is discarded. The slow leader is killed + alone and waited for at most 2 s more, then left unreaped rather than waited for. + Then sc settles and measures as below. After a successful fetch the group is + killed too, so every helper that stays in git's process group, a smart-HTTP v0/v1 helper chain included, dies when git exits and cannot write after the discard. The leader's builtin `kill` is the only group - kill: sc runs no external `kill` program and signals no group id, so a reused - group id cannot be hit and the platform needs no `kill` on the PATH. Each + kill: sc runs no external `kill` program and signals no group id, so the order + of reaping is not part of the safety argument (a reused group id cannot be hit) + and the platform needs no `kill` on the PATH. What is seen is the argument: the + leader must be seen to end by its own group kill. Each serve-mode fetch costs an extra `sh` and about 20 ms of settle time. - A `/bin/sh` exit status of 126 or 127 means `command` could not run git and gives the usual "cannot run git (...): install git or set SC_GIT" error, with @@ -329,7 +337,8 @@ The exact limits: watchdog still frees the call. - The leader kills the group when its stdin closes, which happens when sc drops the pipe or exits, so a group normally ends with `sc serve`. If the leader itself is - killed first, nothing kills the group. On macOS a child spawned concurrently by + killed first, nothing kills the group, which the fetch sees (the leader is gone + before its stdin is closed) and reports as an error. On macOS a child spawned concurrently by another thread can briefly inherit the stdin write end, which delays the end of input until it execs. A process that left the group outlives all of this. From d809b8ff19308be5a3adc2830b4ef76200cca2f5 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sun, 22 Nov 2026 11:49:50 +0200 Subject: [PATCH 7/9] test(objio): bound the surviving writer and stop it before asserting --- crates/objio/src/git.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index d158cd62..9d6f5457 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -4564,16 +4564,15 @@ mod tests { let pids = t.0.join("pids"); let ack = t.0.join("ack"); let body = format!( - "case \" $* \" in *\" fetch \"*) ( exec >/dev/null 2>&1; while :; do head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/pack-late.pack\"; echo > {ack}; sleep 0.05; done ) & echo $! > {p}; until [ -e {ack} ]; do sleep 0.01; done; l=$(ps -o pgid= -p $$ | tr -d ' '); kill -s KILL $l; exit 0;; esac", + "case \" $* \" in *\" fetch \"*) ( exec >/dev/null 2>&1; i=0; while [ $i -lt 600 ]; do head -c 65536 /dev/zero >> \"$GIT_DIR/objects/pack/pack-late.pack\"; echo > {ack}; sleep 0.05; i=$((i+1)); done ) & echo $! > {p}; until [ -e {ack} ]; do sleep 0.01; done; l=$(ps -o pgid= -p $$ | tr -d ' '); kill -s KILL $l; exit 0;; esac", p = pids.display(), ack = ack.display() ); let sh = shim(&t, "killedleader.sh", &body); let b = GitBucket::open_full(&url, sh, CHUNK, None, true).unwrap(); let (out, over) = watched_fetch(&b); - let e = out.err().expect("not a success").to_string(); - assert!(!over && e.contains("supervisor"), "{e}"); - assert!(no_status_files(&b)); + // The surviving writer goes first (it also stops on its own after ~30 s), + // so a failed assertion below never leaves it running. let helper: u32 = std::fs::read_to_string(&pids) .unwrap() .trim() @@ -4581,6 +4580,9 @@ mod tests { .unwrap(); kill_pid(helper); wait_gone(helper); + let e = out.err().expect("not a success").to_string(); + assert!(!over && e.contains("supervisor"), "{e}"); + assert!(no_status_files(&b)); } /// After its stdin closes the leader must end by its own group kill, which is From 3924876c8200ade118d64d68e5a3a8c1ec857a9d Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sun, 22 Nov 2026 11:51:36 +0200 Subject: [PATCH 8/9] docs: record the undetected outside-SIGKILL window of the fetch leader --- docs/THREAT-MODEL.md | 4 +++- docs/adr/0060-git-carried-bucket-remotes.md | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/THREAT-MODEL.md b/docs/THREAT-MODEL.md index 2bbe44cd..dbee49ac 100644 --- a/docs/THREAT-MODEL.md +++ b/docs/THREAT-MODEL.md @@ -799,7 +799,9 @@ The bucket model above applies unchanged, with the Git repository as the bucket open on macOS). The supervisor must be seen running when its stdin is closed and to end by SIGKILL, its own group kill; if it was killed from outside first, or ends any other way, nothing killed the group, so the fetch is an error even if git succeeded (the download is - discarded), and helpers still in the group may survive. Outside serve mode (the unix CLI, Windows) the kill may not reach the + discarded), and helpers still in the group may survive. An outside SIGKILL that lands + after that check and before the leader's own kill looks the same and is not detected; + only the serve user or root can send it, never the Git host. Outside serve mode (the unix CLI, Windows) the kill may not reach the helper chain that writes the pack, which can keep writing after git is killed, and a helper that outlives git is not killed: those cases are not disk-bounded. The call still returns, since detached pipe readers keep a surviving helper from blocking it. diff --git a/docs/adr/0060-git-carried-bucket-remotes.md b/docs/adr/0060-git-carried-bucket-remotes.md index a4cf70ea..91a48a83 100644 --- a/docs/adr/0060-git-carried-bucket-remotes.md +++ b/docs/adr/0060-git-carried-bucket-remotes.md @@ -303,7 +303,11 @@ under a watchdog: killed, and neither are the helpers still in the group when the leader died before it killed the group (it was killed from outside, so the fetch ends by the no-progress deadline or an early exit at the final wait): the fetch fails and - those helpers may survive. That is the one case without a second line of defense. If it holds a pipe, the fetch is an error after the shared deadline, and + those helpers may survive. One window is not detected: a SIGKILL from outside + that reaches the leader after sc saw it running and before its own group kill + ends it the same way, so the fetch can report success while such helpers keep + running. Only a process with signal rights over the leader (the serve user or + root) can send it, never the Git host. That is the one case without a second line of defense. If it holds a pipe, the fetch is an error after the shared deadline, and the discard follows; if it closes its pipes and writes into `objects/pack`, the watchdog bounds disk only while it measures. The status file is a channel that trusts local executables: any process of the same user can write it (the cache From 645bdeb5176a0d9a8600c98517a5b201c29378c2 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sun, 22 Nov 2026 12:06:33 +0200 Subject: [PATCH 9/9] docs: review wording on the fetch leader's limits --- crates/objio/src/git.rs | 9 +++++++-- docs/THREAT-MODEL.md | 9 ++++++--- docs/adr/0060-git-carried-bucket-remotes.md | 7 +++++-- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index 9d6f5457..0a4feedb 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -109,7 +109,9 @@ fn scrub_shell_env(cmd: &mut Command, extra: &[&str]) { } /// The leader's status file in the private cache directory, removed on drop so -/// every exit path of a fetch cleans it up. +/// every exit path of a fetch cleans it up. A leader killed from outside can leave +/// git running, and git's status line can then appear after the drop; the cache +/// removal takes it. #[cfg_attr(not(unix), allow(dead_code))] struct StatusFile(PathBuf); @@ -4370,7 +4372,10 @@ mod tests { /// `kill` would be skipped, the background job would leave the group), /// `BASH_ENV` and `ENV` startup files, and exported functions that override /// `command`, `echo`, `read` and `kill`. The fetch fails with exit 3: its code - /// is reported, the helper is dead and no startup file ran. + /// is reported, the helper is dead and no startup file ran. Only a bash + /// `/bin/sh` (macOS, and the macOS CI job) can fail it: dash ignores these + /// variables, and a bash started as `sh` reads neither startup file, so the + /// marker check is a guard, not a proof. #[cfg(unix)] #[test] fn a_hostile_shell_environment_does_not_change_the_leader() { diff --git a/docs/THREAT-MODEL.md b/docs/THREAT-MODEL.md index dbee49ac..7ad64910 100644 --- a/docs/THREAT-MODEL.md +++ b/docs/THREAT-MODEL.md @@ -795,11 +795,14 @@ The bucket model above applies unchanged, with the Git repository as the bucket with its builtin `kill`, itself included, so `ssh`, `git-remote-https`, `index-pack` and any helper that outlives git, a smart-HTTP v0/v1 helper chain included, die and cannot write after the discard. The group also ends when `sc serve` exits, because the - supervisor's stdin closes (a child spawned at the same moment can briefly hold the pipe - open on macOS). The supervisor must be seen running when its stdin is closed and to end by + supervisor's stdin closes. On macOS a child that another thread spawns at the same moment + can inherit the write end of that pipe and hold it for its whole life (std sets + close-on-exec there after creating the pipe); the supervisor then sees no end of input, + and the fetch fails safe after a 5 s grace with a supervisor error. The supervisor must be seen running when its stdin is closed and to end by SIGKILL, its own group kill; if it was killed from outside first, or ends any other way, nothing killed the group, so the fetch is an error even if git succeeded (the download is - discarded), and helpers still in the group may survive. An outside SIGKILL that lands + discarded), and helpers still in the group, git itself included if it is still running, + may survive. An outside SIGKILL that lands after that check and before the leader's own kill looks the same and is not detected; only the serve user or root can send it, never the Git host. Outside serve mode (the unix CLI, Windows) the kill may not reach the helper chain that writes the pack, which can keep writing after git is killed, and a helper diff --git a/docs/adr/0060-git-carried-bucket-remotes.md b/docs/adr/0060-git-carried-bucket-remotes.md index 91a48a83..ab43c4ab 100644 --- a/docs/adr/0060-git-carried-bucket-remotes.md +++ b/docs/adr/0060-git-carried-bucket-remotes.md @@ -303,11 +303,14 @@ under a watchdog: killed, and neither are the helpers still in the group when the leader died before it killed the group (it was killed from outside, so the fetch ends by the no-progress deadline or an early exit at the final wait): the fetch fails and - those helpers may survive. One window is not detected: a SIGKILL from outside + those helpers, and git itself if it is still running, may survive (its status + line can then appear in the cache after the fetch ended; it is removed with the + cache). One window is not detected: a SIGKILL from outside that reaches the leader after sc saw it running and before its own group kill ends it the same way, so the fetch can report success while such helpers keep running. Only a process with signal rights over the leader (the serve user or - root) can send it, never the Git host. That is the one case without a second line of defense. If it holds a pipe, the fetch is an error after the shared deadline, and + root) can send it, never the Git host. A process that left the group has no + second line of defense: if it holds a pipe, the fetch is an error after the shared deadline, and the discard follows; if it closes its pipes and writes into `objects/pack`, the watchdog bounds disk only while it measures. The status file is a channel that trusts local executables: any process of the same user can write it (the cache