From 8409bfb19c777a24c0a96116195992551322b225 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 11:18:36 +0200 Subject: [PATCH 01/15] feat(forge): accept refs/sc/* for sc+git:: stores The pre-receive policy now accepts create, update (including non-fast-forward) and delete for names under exactly `refs/sc/`, so a repository hosted on sc-forge can carry a native sc bucket store. `refs/sc`, `refs/scx/x` and every other namespace stay refused; heads and tags are unchanged. End-to-end tests drive the real receive path over smart HTTP with plain git: create with an empty lease, CAS update without `+`, stale lease rejected, exact-refspec fetch, delete, two racing CAS pushes with one winner, a hand-built receive-pack request that proves the server checks the old value under the ref lock, and a maintenance pass that keeps the live values and prunes the replaced one. ADR-0058 addendum, FORGE-OPERATIONS.md and THREAT-MODEL.md describe the namespace: writable by every repository writer, outside branch protection, advertised to readers, bounded by receive.maxInputSize. --- crates/forge/conformance/departures.json | 18 +- crates/forge/src/hook.rs | 7 +- crates/forge/src/policy.rs | 32 +- crates/forge/tests/e2e_git.rs | 341 ++++++++++++++++++ docs/FORGE-OPERATIONS.md | 12 + docs/THREAT-MODEL.md | 3 +- ...058-sc-forge-gitea-compatible-git-forge.md | 21 ++ 7 files changed, 418 insertions(+), 16 deletions(-) diff --git a/crates/forge/conformance/departures.json b/crates/forge/conformance/departures.json index 2e9ec688..9a088fc5 100644 --- a/crates/forge/conformance/departures.json +++ b/crates/forge/conformance/departures.json @@ -379,7 +379,7 @@ "refs/tags/" ] }, - "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/* and refs/tags/*: 'refusing to update : only refs/heads/* and refs/tags/* are writable'" + "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/*, refs/tags/* and refs/sc/*: 'refusing to update : only refs/heads/*, refs/tags/* and refs/sc/* are writable'" }, { "scenario": "S", @@ -393,7 +393,7 @@ "refs/tags/" ] }, - "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/* and refs/tags/*: 'refusing to update : only refs/heads/* and refs/tags/* are writable'" + "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/*, refs/tags/* and refs/sc/*: 'refusing to update : only refs/heads/*, refs/tags/* and refs/sc/* are writable'" }, { "scenario": "S", @@ -407,7 +407,7 @@ "refs/tags/" ] }, - "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/* and refs/tags/*: 'refusing to update : only refs/heads/* and refs/tags/* are writable'" + "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/*, refs/tags/* and refs/sc/*: 'refusing to update : only refs/heads/*, refs/tags/* and refs/sc/* are writable'" }, { "scenario": "S", @@ -421,7 +421,7 @@ "refs/tags/" ] }, - "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/* and refs/tags/*: 'refusing to update : only refs/heads/* and refs/tags/* are writable'" + "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/*, refs/tags/* and refs/sc/*: 'refusing to update : only refs/heads/*, refs/tags/* and refs/sc/* are writable'" }, { "scenario": "S", @@ -435,7 +435,7 @@ "refs/tags/" ] }, - "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/* and refs/tags/*: 'refusing to update : only refs/heads/* and refs/tags/* are writable'" + "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/*, refs/tags/* and refs/sc/*: 'refusing to update : only refs/heads/*, refs/tags/* and refs/sc/* are writable'" }, { "scenario": "S", @@ -449,7 +449,7 @@ "refs/tags/" ] }, - "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/* and refs/tags/*: 'refusing to update : only refs/heads/* and refs/tags/* are writable'" + "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/*, refs/tags/* and refs/sc/*: 'refusing to update : only refs/heads/*, refs/tags/* and refs/sc/* are writable'" }, { "scenario": "S", @@ -463,7 +463,7 @@ "refs/tags/" ] }, - "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/* and refs/tags/*: 'refusing to update : only refs/heads/* and refs/tags/* are writable'" + "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/*, refs/tags/* and refs/sc/*: 'refusing to update : only refs/heads/*, refs/tags/* and refs/sc/* are writable'" }, { "scenario": "S", @@ -477,7 +477,7 @@ "refs/tags/" ] }, - "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/* and refs/tags/*: 'refusing to update : only refs/heads/* and refs/tags/* are writable'" + "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/*, refs/tags/* and refs/sc/*: 'refusing to update : only refs/heads/*, refs/tags/* and refs/sc/* are writable'" }, { "scenario": "S", @@ -491,7 +491,7 @@ "refs/tags/" ] }, - "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/* and refs/tags/*: 'refusing to update : only refs/heads/* and refs/tags/* are writable'" + "reason": "sc-forge's pre-receive refuses every ref outside refs/heads/*, refs/tags/* and refs/sc/*: 'refusing to update : only refs/heads/*, refs/tags/* and refs/sc/* are writable'" }, { "scenario": "S", diff --git a/crates/forge/src/hook.rs b/crates/forge/src/hook.rs index aa7aee3c..88ce4253 100644 --- a/crates/forge/src/hook.rs +++ b/crates/forge/src/hook.rs @@ -343,17 +343,22 @@ mod tests { l(A, "refs/replace/x"), l(Z, "refs/heads/MAIN"), l(A, "refs/pull/1/head"), + l(A, "refs/sc/manifest"), + l(Z, "refs/sc/log/00000000000000000001"), + l(A, "refs/scx/x"), ], "main", true, &Rules::default(), &ff(true), ); - assert_eq!(out.len(), 3, "{out:?}"); + assert_eq!(out.len(), 4, "{out:?}"); assert!( out[0].contains("refs/replace/x") && out[1].contains("default branch") && out[2].contains("refs/pull/1/head") + && out[3].contains("refs/scx/x"), + "refs/sc/* is accepted, refs/scx/x is not: {out:?}" ); } diff --git a/crates/forge/src/policy.rs b/crates/forge/src/policy.rs index 8eee970b..403e72e3 100644 --- a/crates/forge/src/policy.rs +++ b/crates/forge/src/policy.rs @@ -19,8 +19,12 @@ pub fn same_branch_name(a: &str, b: &str) -> bool { } /// May a client push `name` (`old` -> `new`) to a repository whose default branch is -/// `default_branch`? Only `refs/heads/*` and `refs/tags/*` are writable; `refs/replace/*`, -/// `refs/forge/*`, `refs/pull/*`, `refs/notes/*` and anything else is refused. The default branch +/// `default_branch`? Only `refs/heads/*`, `refs/tags/*` and `refs/sc/*` are writable; +/// `refs/replace/*`, `refs/forge/*`, `refs/pull/*`, `refs/notes/*` and anything else is refused. +/// `refs/sc/*` (the trailing slash matters: `refs/sc` and `refs/scx/x` are refused) holds a native +/// sc bucket store (`sc+git::` remotes): every repository writer may create, update (including +/// non-fast-forward, the client uses `--force-with-lease`) and delete it, branch protection does +/// not apply, and it never touches the default branch. The default branch /// cannot be deleted, compared case-insensitively (on a case-insensitive filesystem `MAIN` aliases /// the loose ref `main`). pub fn check_update(name: &str, new: &str, default_branch: &str) -> Result<(), Refusal> { @@ -30,11 +34,11 @@ pub fn check_update(name: &str, new: &str, default_branch: &str) -> Result<(), R } return Ok(()); } - if name.starts_with("refs/tags/") { + if name.starts_with("refs/tags/") || name.starts_with("refs/sc/") { return Ok(()); } Err(format!( - "refusing to update {name}: only refs/heads/* and refs/tags/* are writable" + "refusing to update {name}: only refs/heads/*, refs/tags/* and refs/sc/* are writable" )) } @@ -142,6 +146,19 @@ mod tests { } } + #[test] + fn refs_sc_is_writable_for_create_update_and_delete() { + for r in [ + "refs/sc/manifest", + "refs/sc/log/00000000000000000001", + "refs/sc/packs/ab12.pack", + ] { + for new in [A, Z] { + assert!(check_update(r, new, "main").is_ok(), "{r} -> {new}"); + } + } + } + #[test] fn every_other_namespace_is_refused_with_the_recorded_text() { for r in [ @@ -156,13 +173,18 @@ mod tests { "refs/heads", "refs/headsx/main", "refs/tag/v1", + "refs/sc", + "refs/scx/x", + "refs/sc2/manifest", "FETCH_HEAD", "", ] { let e = check_update(r, A, "main").unwrap_err(); assert_eq!( e, - format!("refusing to update {r}: only refs/heads/* and refs/tags/* are writable"), + format!( + "refusing to update {r}: only refs/heads/*, refs/tags/* and refs/sc/* are writable" + ), "{r:?}" ); } diff --git a/crates/forge/tests/e2e_git.rs b/crates/forge/tests/e2e_git.rs index 0e801bb3..e87a38aa 100644 --- a/crates/forge/tests/e2e_git.rs +++ b/crates/forge/tests/e2e_git.rs @@ -22,6 +22,17 @@ impl Client { /// Run the client's git with a scrubbed environment; `header` is the Authorization value. fn git_in(&self, cwd: &Path, header: Option<&str>, args: &[&str]) -> (bool, String) { + self.git_env(cwd, header, &[], args) + } + + /// [`Client::git_in`] with extra environment variables (they override the defaults). + fn git_env( + &self, + cwd: &Path, + header: Option<&str>, + env: &[(&str, &str)], + args: &[&str], + ) -> (bool, String) { let mut cmd = Command::new("git"); cmd.env_clear() .env("PATH", "/usr/bin:/bin:/usr/local/bin:/opt/homebrew/bin") @@ -32,6 +43,7 @@ impl Client { .env("GIT_AUTHOR_EMAIL", "t@example.invalid") .env("GIT_COMMITTER_NAME", "T") .env("GIT_COMMITTER_EMAIL", "t@example.invalid") + .envs(env.iter().copied()) .current_dir(cwd); if let Some(h) = header { cmd.arg("-c") @@ -1328,3 +1340,332 @@ fn a_malformed_service_keeps_the_fixed_error_order() { "then the dumb listing is refused" ); } + +// ---- refs/sc/*: a native sc bucket store carried by a hosted Git repository ---- + +/// A parentless commit shaped like an `sc+git::` bucket value: fixed identity and date, a tree of +/// regular blobs `c0000`, `c0001`, ..., and a `sc-bucket new ` message. Built in `store` +/// with plain git (an isolated index), so the object exists locally for the push. +fn bucket_value(c: &Client, store: &Path, nonce: &str, chunks: &[&str]) -> String { + let idx = c.dir.join(format!("idx-{nonce}")); + let idx = idx.to_str().unwrap(); + let env = [ + ("GIT_INDEX_FILE", idx), + ("GIT_AUTHOR_NAME", "sc"), + ("GIT_AUTHOR_EMAIL", "sc@invalid"), + ("GIT_COMMITTER_NAME", "sc"), + ("GIT_COMMITTER_EMAIL", "sc@invalid"), + ("GIT_AUTHOR_DATE", "946684800 +0000"), + ("GIT_COMMITTER_DATE", "946684800 +0000"), + ]; + let run = |args: &[&str]| { + let (ok, text) = c.git_env(store, None, &env, args); + assert!(ok, "git {args:?}: {text}"); + text.trim().to_string() + }; + for (i, text) in chunks.iter().enumerate() { + let f = c.dir.join(format!("chunk-{nonce}-{i}")); + std::fs::write(&f, text).unwrap(); + let blob = run(&["hash-object", "-w", f.to_str().unwrap()]); + run(&[ + "update-index", + "--add", + "--cacheinfo", + &format!("100644,{blob},c{i:04}"), + ]); + } + let tree = run(&["write-tree"]); + run(&[ + "commit-tree", + &tree, + "-m", + &format!("sc-bucket new {nonce}"), + ]) +} + +/// `git push` of `oid` to `refname` guarded by a compare-and-swap on `expect` ("" = must not exist), +/// with no `+` in the refspec, as the sc client does it. +fn cas_push( + c: &Client, + store: &Path, + auth: &str, + url: &str, + refname: &str, + expect: &str, + oid: &str, +) -> (bool, String) { + c.git_in( + store, + Some(auth), + &[ + "push", + &format!("--force-with-lease={refname}:{expect}"), + url, + &format!("{oid}:{refname}"), + ], + ) +} + +fn remote_ref(c: &Client, dir: &Path, auth: &str, url: &str, refname: &str) -> Option { + let out = c.ok(dir, Some(auth), &["ls-remote", url, refname]); + out.split_whitespace().next().map(str::to_string) +} + +/// A server with one repository and a local store repository (no branches) that pushes to it. +fn store_world(tag: &str) -> (Srv, Client, String, String, PathBuf) { + let s = Srv::start(); + let c = Client::new(tag); + let url = repo(&s, "smart"); + let auth = token(&s.admin); + let store = c.dir.join("store"); + c.ok( + &c.dir, + None, + &["init", "-q", "-b", "main", store.to_str().unwrap()], + ); + (s, c, url, auth, store) +} + +/// Seed a `main` branch from a scratch repository so the repository has a HEAD. +fn seed_main(c: &Client, url: &str, auth: &str) { + let work = c.dir.join("seed"); + c.ok( + &c.dir, + None, + &["init", "-q", "-b", "main", work.to_str().unwrap()], + ); + put_commit(c, &work, "a.txt", "hello\n"); + c.ok(&work, Some(auth), &["push", "-q", url, "main"]); +} + +#[test] +fn refs_sc_create_cas_update_fetch_and_delete_round_trip_through_the_real_receive_path() { + let (_s, c, url, auth, store) = store_world("sc-rt"); + let m = "refs/sc/manifest"; + let v1 = bucket_value(&c, &store, "n1", &["one\n", "uno\n"]); + let v2 = bucket_value(&c, &store, "n2", &["two\n"]); + let v3 = bucket_value(&c, &store, "n3", &["three\n"]); + // create: an empty lease means "must not exist" + let (ok, t) = cas_push(&c, &store, &auth, &url, m, "", &v1); + assert!(ok, "create: {t}"); + assert_eq!(remote_ref(&c, &store, &auth, &url, m), Some(v1.clone())); + // creating again with the empty lease is refused and the ref is unchanged + let (ok, _) = cas_push(&c, &store, &auth, &url, m, "", &v2); + assert!(!ok, "create over an existing ref"); + assert_eq!(remote_ref(&c, &store, &auth, &url, m), Some(v1.clone())); + // update to an unrelated parentless commit (a non-fast-forward), lease on the old value + let (ok, t) = cas_push(&c, &store, &auth, &url, m, &v1, &v2); + assert!(ok, "cas update: {t}"); + assert_eq!(remote_ref(&c, &store, &auth, &url, m), Some(v2.clone())); + // a stale lease is refused and the ref stays at v2 + let (ok, t) = cas_push(&c, &store, &auth, &url, m, &v1, &v3); + assert!(!ok, "stale lease must be refused: {t}"); + assert_eq!(remote_ref(&c, &store, &auth, &url, m), Some(v2.clone())); + // a second key with a path-like name next to the first + let log = bucket_value(&c, &store, "n4", &["entry\n"]); + let k = "refs/sc/log/00000000000000000001"; + let (ok, t) = cas_push(&c, &store, &auth, &url, k, "", &log); + assert!(ok, "{t}"); + // fetch by exact refspec into a fresh repository + let fresh = c.dir.join("fresh"); + c.ok(&c.dir, None, &["init", "-q", fresh.to_str().unwrap()]); + c.ok( + &fresh, + Some(&auth), + &["fetch", "-q", &url, "+refs/sc/*:refs/sc/*"], + ); + assert_eq!(c.ok(&fresh, None, &["rev-parse", m]).trim(), v2); + assert_eq!(c.ok(&fresh, None, &["rev-parse", k]).trim(), log); + assert_eq!( + c.ok(&fresh, None, &["show", &format!("{m}:c0000")]), + "two\n" + ); + c.ok(&fresh, None, &["fsck", "--strict", "--no-dangling"]); + // refs/sc is advertised to readers, but a plain clone does not fetch it + seed_main(&c, &url, &auth); + let adv = c.ok(&store, Some(&auth), &["ls-remote", &url]); + assert!(adv.contains(m), "advertised to readers: {adv}"); + let clone = c.dir.join("clone"); + c.ok( + &c.dir, + Some(&auth), + &["clone", "-q", &url, clone.to_str().unwrap()], + ); + assert_eq!(c.ok(&clone, None, &["for-each-ref", "refs/sc"]), ""); + // delete with a lease: a stale one fails, the right one wins + let del = |expect: &str| { + c.git_in( + &store, + Some(&auth), + &[ + "push", + &format!("--force-with-lease={m}:{expect}"), + &url, + &format!(":{m}"), + ], + ) + }; + assert!(!del(&v1).0, "stale delete lease"); + assert_eq!(remote_ref(&c, &store, &auth, &url, m), Some(v2.clone())); + let (ok, t) = del(&v2); + assert!(ok, "delete: {t}"); + assert_eq!(remote_ref(&c, &store, &auth, &url, m), None); + // the namespace boundary is exactly `refs/sc/` + for r in ["refs/scx/x", "refs/sc", "refs/notes/x"] { + let (ok, t) = c.git_in(&store, Some(&auth), &["push", &url, &format!("{v2}:{r}")]); + assert!(!ok, "{r} must be refused: {t}"); + assert_eq!(remote_ref(&c, &store, &auth, &url, r), None, "{r}"); + } +} + +/// One receive-pack request by hand (`old` and `new` are what the client believes), so the +/// server's own compare-and-swap is exercised: the real client checks the advertisement first. +fn raw_update(s: &Srv, auth: &str, old: &str, new: &str, refname: &str) -> String { + let pkt = |t: &str| format!("{:04x}{t}", t.len() + 4); + let mut body = pkt(&format!( + "{old} {new} {refname}\0 report-status object-format=sha1\n" + )) + .into_bytes(); + body.extend_from_slice(b"0000"); + if new.bytes().any(|b| b != b'0') { + // the pushed object is already on the server: an empty pack carries it + let out = Command::new("git") + .args(["pack-objects", "--stdout"]) + .stdin(std::process::Stdio::null()) + .output() + .unwrap(); + body.extend_from_slice(&out.stdout); + } + let head = format!( + "POST /siteadmin/smart.git/git-receive-pack HTTP/1.1\r\nHost: x\r\nConnection: close\r\nAuthorization: {auth}\r\nContent-Type: application/x-git-receive-pack-request\r\nContent-Length: {}\r\n\r\n", + body.len() + ); + let r = s.raw(head.as_bytes(), &body); + assert_eq!(r.status, 200); + body_text(&r) +} + +#[test] +fn refs_sc_cas_is_enforced_by_the_server_under_the_ref_lock() { + let (s, c, url, auth, store) = store_world("sc-srv"); + let m = "refs/sc/manifest"; + let z = "0".repeat(40); + let v1 = bucket_value(&c, &store, "n1", &["one\n"]); + let v2 = bucket_value(&c, &store, "n2", &["two\n"]); + let v3 = bucket_value(&c, &store, "n3", &["three\n"]); + for (expect, oid) in [("", &v1), (v1.as_str(), &v2)] { + let (ok, t) = cas_push(&c, &store, &auth, &url, m, expect, oid); + assert!(ok, "{t}"); + } + // stale old value: refused for an update and for a delete, ref unchanged + let out = raw_update(&s, &auth, &v3, &v1, m); + assert!(out.contains(&format!("ng {m}")), "{out}"); + let out = raw_update(&s, &auth, &v1, &z, m); + assert!(out.contains(&format!("ng {m}")), "{out}"); + assert_eq!(remote_ref(&c, &store, &auth, &url, m), Some(v2.clone())); + // the same request with the right old value is accepted (the request shape is valid) + let out = raw_update(&s, &auth, &v2, &v1, m); + assert!(out.contains(&format!("ok {m}")), "{out}"); + assert_eq!(remote_ref(&c, &store, &auth, &url, m), Some(v1.clone())); + // a name outside the namespace is refused by the hook, whatever the old value + let out = raw_update(&s, &auth, &z, &v1, "refs/scx/x"); + assert!( + out.contains("refs/scx/x") && !out.contains("ok refs/scx/x"), + "{out}" + ); + assert_eq!(remote_ref(&c, &store, &auth, &url, "refs/scx/x"), None); +} + +#[test] +fn racing_refs_sc_cas_pushes_have_exactly_one_winner() { + let (_s, c, url, auth, store) = store_world("sc-race"); + let m = "refs/sc/manifest"; + let mut cur = bucket_value(&c, &store, "r0", &["zero\n"]); + let (ok, t) = cas_push(&c, &store, &auth, &url, m, "", &cur); + assert!(ok, "{t}"); + for round in 0..6 { + let a = bucket_value(&c, &store, &format!("r{round}a"), &["a\n"]); + let b = bucket_value(&c, &store, &format!("r{round}b"), &["b\n"]); + let (ra, rb) = std::thread::scope(|sc| { + let ta = sc.spawn(|| cas_push(&c, &store, &auth, &url, m, &cur, &a)); + let tb = sc.spawn(|| cas_push(&c, &store, &auth, &url, m, &cur, &b)); + (ta.join().unwrap(), tb.join().unwrap()) + }); + assert!( + ra.0 ^ rb.0, + "round {round}: exactly one wins: {ra:?} {rb:?}" + ); + let winner = if ra.0 { &a } else { &b }; + assert_eq!( + remote_ref(&c, &store, &auth, &url, m).as_ref(), + Some(winner), + "round {round}" + ); + cur = winner.clone(); + } +} + +#[test] +fn maintenance_keeps_the_live_refs_sc_values() { + use scl_forge::maintenance::Cfg; + use std::time::Duration; + let s = Srv::start_maint( + Cfg { + interval: Duration::ZERO, + min_free: 0, + gate_wait: Duration::from_secs(5), + prune_expire: Duration::from_secs(1), + ..Cfg::default() + }, + None, + ); + let c = Client::new("sc-maint"); + let url = repo(&s, "smart"); + let auth = token(&s.admin); + let store = c.dir.join("store"); + c.ok( + &c.dir, + None, + &["init", "-q", "-b", "main", store.to_str().unwrap()], + ); + let m = "refs/sc/manifest"; + let k = "refs/sc/packs/00ff.pack"; + let old = bucket_value(&c, &store, "m1", &["old\n"]); + let live = bucket_value(&c, &store, "m2", &["live\n", "live2\n"]); + let pack = bucket_value(&c, &store, "m3", &["packbytes\n"]); + for (r, expect, oid) in [(m, "", &old), (m, old.as_str(), &live), (k, "", &pack)] { + let (ok, t) = cas_push(&c, &store, &auth, &url, r, expect, oid); + assert!(ok, "{t}"); + } + std::thread::sleep(Duration::from_millis(2200)); + let before = s.app.maint.passes(); + let r = s.send("POST", "/api/v1/admin/cron/git_gc_repos", Some(&auth), None); + assert_eq!(r.status, 204, "{}", r.s_text()); + let end = std::time::Instant::now() + Duration::from_secs(60); + while s.app.maint.passes() <= before { + assert!(std::time::Instant::now() < end, "the pass never finished"); + std::thread::sleep(Duration::from_millis(25)); + } + assert_eq!(remote_ref(&c, &store, &auth, &url, m), Some(live.clone())); + assert_eq!(remote_ref(&c, &store, &auth, &url, k), Some(pack.clone())); + assert!(s.git_try(1, &["cat-file", "-e", &live]).is_some()); + assert!(s + .git_try(1, &["fsck", "--strict", "--no-dangling"]) + .is_some()); + assert!( + s.git_try(1, &["cat-file", "-e", &old]).is_none(), + "the replaced value is pruned (fetchable by id only until then)" + ); + // what survived still fetches and reads back + let fresh = c.dir.join("fresh"); + c.ok(&c.dir, None, &["init", "-q", fresh.to_str().unwrap()]); + c.ok( + &fresh, + Some(&auth), + &["fetch", "-q", &url, "+refs/sc/*:refs/sc/*"], + ); + assert_eq!( + c.ok(&fresh, None, &["show", &format!("{m}:c0001")]), + "live2\n" + ); +} diff --git a/docs/FORGE-OPERATIONS.md b/docs/FORGE-OPERATIONS.md index 2b0afb64..3bb2c52b 100644 --- a/docs/FORGE-OPERATIONS.md +++ b/docs/FORGE-OPERATIONS.md @@ -95,6 +95,18 @@ Aborted merges, refused writes and force-pushed history leave unreferenced objec - A pass also removes stale ref locks, commit-graph locks and quarantine directories of killed git processes, only when nothing can own them. +## Native sc stores in `refs/sc/*` + +A repository can carry an sc bucket store (`sc+git::` remotes keep each key as a parentless commit at `refs/sc/`). The server +accepts pushes to exactly the `refs/sc/` prefix; other namespaces outside `refs/heads/*` and `refs/tags/*` stay refused. + +- Every repository writer can create, overwrite and delete `refs/sc/*`. There is no per-repository switch. +- Branch protection does not apply to it. +- It is advertised to every reader. Plain `git clone` and `git fetch` skip it unless asked for the exact refspec. +- A deleted or replaced value stays fetchable by object id until a maintenance pass prunes it (`--gc-prune-expire`). The live + values are reachable from their refs, so gc and prune keep them. +- `receive.maxInputSize` (`--max-pack-size`, default 2 GiB) bounds one pushed value. + ## Resource limits (cgroups) The server bounds its own git children (timeouts, `RLIMIT_CPU`/`RLIMIT_FSIZE`, output caps, `--git-memory-limit` as `RLIMIT_DATA` on diff --git a/docs/THREAT-MODEL.md b/docs/THREAT-MODEL.md index 861fb7d2..a2bd22cf 100644 --- a/docs/THREAT-MODEL.md +++ b/docs/THREAT-MODEL.md @@ -1074,7 +1074,8 @@ unless said otherwise). `objects/info/alternates` and every other file are 404. Objects of a deleted branch stay readable by anyone with read access to *that* repository (`uploadpack.allowAnySHA1InWant=true` is what makes fetch by id work; it is not a cross-tenant read, since each repository is a separate object store and no alternates exist). -- **Ref namespace.** Only `refs/heads/*` and `refs/tags/*` are writable: `refs/replace/*`, `refs/pull/*`, `refs/notes/*`, +- **Ref namespace.** Only `refs/heads/*`, `refs/tags/*` and `refs/sc/*` (a native sc store, writable by every repository + writer, outside branch protection; see ADR-0058's addendum) are writable: `refs/replace/*`, `refs/pull/*`, `refs/notes/*`, `refs/changes/*` and everything else are refused by the `pre-receive` hook (`the_ref_policy_is_enforced_for_every_unwritable_namespace_and_the_default_branch`); `refs/forge/` (reserved for the server) is hidden from both services (`transfer.hideRefs`) and cannot be fetched by name diff --git a/docs/adr/0058-sc-forge-gitea-compatible-git-forge.md b/docs/adr/0058-sc-forge-gitea-compatible-git-forge.md index 8c074467..4413b55b 100644 --- a/docs/adr/0058-sc-forge-gitea-compatible-git-forge.md +++ b/docs/adr/0058-sc-forge-gitea-compatible-git-forge.md @@ -168,3 +168,24 @@ Acceptance does not approve a production migration. The local fixture used the consumer's Redis-free middleware mode, so production rate limiting was not qualified. SSH, Actions and the other v1 exclusions above remain excluded; implementation follow-ons remain tracked in #316. + +## Addendum: `refs/sc/*` carries a native sc bucket store + +The `pre-receive` policy accepts exactly the `refs/sc/` prefix (trailing slash; `refs/sc` +and `refs/scx/x` stay refused) for create, update and delete, in addition to `refs/heads/*` +and `refs/tags/*`. A `sc+git::` remote stores each bucket key as a parentless commit at +`refs/sc/`, so a repository hosted here can hold an sc store. Updates may be +non-fast-forward: the client compare-and-swaps with `--force-with-lease`, and git checks the +old value under the ref lock. + +- Every repository writer may write `refs/sc/*`. There is no per-repository opt-in; a writer + can already push arbitrary content to any branch, so this adds no new kind of access. +- Branch protection does not cover it. +- `refs/sc/*` is advertised to every reader. A plain `git clone` or `git fetch` does not fetch it + by default; a reader who asks for the exact refspec gets it. +- A deleted or replaced value stays fetchable by object id until the gc prune stage removes it + (`--gc-prune-expire`). +- `receive.maxInputSize` (`--max-pack-size`, default 2 GiB) bounds one pushed value. + +Tests: `policy::tests`, `hook::tests`, and in `crates/forge/tests/e2e_git.rs` the +`refs_sc_*`, `racing_refs_sc_*` and `maintenance_keeps_the_live_refs_sc_values` cases. From ca41ce769d3502865c4450d2a19dbe67fd703752 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 11:23:30 +0200 Subject: [PATCH 02/15] feat(objio): GitBucket, a bucket stored as refs/sc/* in any Git repository Adds the objio backend for the sc+git:: carrier (P39). Each bucket key lives at refs/sc/ as a parentless commit whose tree holds the value as chunk blobs (32 MiB each, 2 GiB per value). Create-only writes push with an empty lease and CAS writes with the expected oid as the lease, never with a + refspec; every rejection is classified by re-reading the exact ref with ls-remote, so a hook refusal is an error and never a lost race. The backend drives the system git binary (SC_GIT overrides), with a scrubbed GIT_* environment, a protocol allowlist, no hooks or signing, a fixed carrier identity and a private temp cache repository removed on drop. Cold reads of log/, checkpoints/ and packs/*.idx fetch the whole family in one call. Tests cover the contract suite, racing CAS and create, stale and malformed tags, a rejecting pre-receive hook, decoy refs, chunking, corrupt value shapes, URL and key validation, hostile user git config, cache cleanup and a counting git shim that bounds network calls independent of entry count. --- crates/objio/src/git.rs | 1728 +++++++++++++++++++++++++++++++++++++++ crates/objio/src/lib.rs | 8 +- 2 files changed, 1734 insertions(+), 2 deletions(-) create mode 100644 crates/objio/src/git.rs diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs new file mode 100644 index 00000000..060b8ca6 --- /dev/null +++ b/crates/objio/src/git.rs @@ -0,0 +1,1728 @@ +//! `GitBucket`: a `Bucket` stored as `refs/sc/*` inside any Git repository +//! (the `sc+git::` carrier, P39). +//! +//! Each bucket key `K` lives at the ref `refs/sc/K`. The ref points at a +//! parentless commit whose tree holds the value as chunk blobs `c0000`, +//! `c0001`, ... of at most [`CHUNK`] bytes each; the commit id is the tag. +//! Atomicity comes from `receive-pack` checking the sent old-oid under its ref +//! lock: create-only writes push with an empty lease, CAS writes with the +//! expected oid as the lease, and no `+` refspec is ever used (a `+` makes +//! `--force-with-lease` ineffective). +//! +//! Git is driven through the system `git` binary (`SC_GIT` overrides it), never +//! through `gix`, which stays quarantined in `gitio`. A per-instance temporary +//! bare repository caches fetched objects and is removed on `Drop`; a SIGKILL +//! can leave `sc-git-bucket--` behind in the temp directory. +//! +//! Every rejected write is classified by re-reading the exact ref with +//! `ls-remote`, never by the host's wording, so a hook refusal can never look +//! like a lost race. + +use crate::{validate_key, Bucket, Error, Fetched, ObjectMeta, Result}; +use std::collections::HashMap; +use std::io::{Read, Write}; +use std::path::PathBuf; +use std::process::{Command, Stdio}; +use std::sync::{Mutex, MutexGuard}; +use std::time::{Duration, Instant, SystemTime}; + +/// Largest chunk blob a value is split into (GitHub's hard per-blob limit is +/// 100 MB); also the largest chunk accepted on read. +const CHUNK: usize = 32 << 20; +/// Largest value written or read: GitHub's per-push cap and sc-forge's +/// `receive.maxInputSize`. Values are buffered in RAM like on S3. +const MAX_VALUE: u64 = 2 << 30; +/// Most chunk entries a value tree may hold (names are `c0000`..`c9999`). +const MAX_CHUNKS: usize = 10_000; +/// How long a tag this instance itself observed or wrote is trusted for a +/// conditional read before the network is asked again. +const FRESH: Duration = Duration::from_secs(2); +/// Cap on captured `ls-remote` output (a hostile host cannot fill RAM). +const LS_REMOTE_CAP: usize = 64 << 20; +/// Cap on captured output of every other small git call. +const SMALL_CAP: usize = 1 << 20; +/// Namespace prefix of every bucket ref. +const NS: &str = "refs/sc/"; +/// Refspecs per `fetch` call in `list_meta` (keeps the command line short). +const FETCH_BATCH: usize = 256; + +/// `-c` overrides on every git call: transport allowlist, no hooks, no signing, +/// no background maintenance, bounded stalls. +const GIT_CONFIG: &[&str] = &[ + "protocol.allow=never", + "protocol.file.allow=always", + "protocol.ssh.allow=always", + "protocol.https.allow=always", + "protocol.http.allow=always", + "core.hooksPath=/dev/null", + "gc.auto=0", + "maintenance.auto=false", + "push.gpgSign=false", + "commit.gpgSign=false", + "tag.gpgSign=false", + "core.useReplaceRefs=false", + "http.lowSpeedLimit=1000", + "http.lowSpeedTime=60", +]; + +/// `GIT_*` variables that survive into child git processes: the user's +/// transport and credential setup. Everything else (`GIT_DIR` family, config +/// injection, traces) is removed so the process environment cannot redirect +/// the cache repository or change config. +const KEEP_GIT_ENV: &[&str] = &[ + "GIT_SSH", + "GIT_SSH_COMMAND", + "GIT_SSH_VARIANT", + "GIT_ASKPASS", + "GIT_CONFIG_GLOBAL", + "GIT_CONFIG_SYSTEM", + "GIT_CONFIG_NOSYSTEM", + "GIT_EXEC_PATH", + "GIT_PROXY_COMMAND", + "GIT_SSL_NO_VERIFY", + "GIT_SSL_CAINFO", + "GIT_SSL_CAPATH", + "GIT_SSL_CERT", + "GIT_SSL_KEY", + "GIT_SSL_VERSION", +]; + +/// Fixed carrier identity so a clean account with no `user.name` can write. +const IDENTITY_ENV: &[(&str, &str)] = &[ + ("GIT_AUTHOR_NAME", "sc"), + ("GIT_AUTHOR_EMAIL", "sc@invalid"), + ("GIT_COMMITTER_NAME", "sc"), + ("GIT_COMMITTER_EMAIL", "sc@invalid"), +]; + +fn keep_env(name: &str) -> bool { + !name.starts_with("GIT_") || KEEP_GIT_ENV.contains(&name) +} + +/// A key family GitBucket knows the WAL layout of: the first cache miss in one +/// fetches the whole family in a single `git fetch` instead of one round trip +/// per entry. +struct Family { + dir: &'static str, + suffix: &'static str, +} + +const FAMILIES: [Family; 3] = [ + Family { + dir: "log/", + suffix: "", + }, + Family { + dir: "checkpoints/", + suffix: "", + }, + Family { + dir: "packs/", + suffix: ".idx", + }, +]; + +impl Family { + fn matches(&self, key: &str) -> bool { + key.len() > self.dir.len() + self.suffix.len() + && key.starts_with(self.dir) + && key.ends_with(self.suffix) + } + fn glob(&self) -> String { + format!("{NS}{}*{}", self.dir, self.suffix) + } +} + +fn family_of(key: &str) -> Option<&'static Family> { + FAMILIES.iter().find(|f| f.matches(key)) +} + +/// What this instance knows about a key's current remote value. +struct Known { + oid: String, + /// Written by `put_new`: the value never changes, so reads are served from + /// the local cache with no network call. + create_only: bool, + /// When the oid was last observed or written by this instance. + at: Instant, +} + +/// Captured result of one git invocation. +struct Out { + success: bool, + code: Option, + stdout: Vec, + stderr: String, +} + +/// Outcome of a push of one ref. +enum Pushed { + Done, + /// The host refused or the push failed; carries the captured message. + Refused(String), +} + +/// A bucket stored as `refs/sc/*` in a Git repository reachable at `url`. +/// Intent: let a plain Git host carry a native sc store (P39) without a new +/// server. See the module docs for the value encoding and write protocol. +pub struct GitBucket { + url: String, + git: String, + cache: PathBuf, + /// Chunk size used for writes (tests shrink it). + chunk: usize, + state: Mutex>, +} + +fn is_oid(s: &str) -> bool { + s.len() == 40 && s.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) +} + +/// A usable CAS tag: a nonzero full hex oid of the cache's (SHA-1) format. +fn is_tag(s: &str) -> bool { + is_oid(s) && s.bytes().any(|b| b != b'0') +} + +/// Strict ref-safe key check on top of the shared `validate_key`: git ref +/// format rules, so a key can never smuggle ref syntax into a refspec. +fn check_key(key: &str) -> Result<()> { + validate_key(key)?; + if key.len() > 1024 + || key.contains("..") + || !key + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'/' | b'-')) + || key + .split('/') + .any(|c| c.starts_with('.') || c.ends_with('.') || c.ends_with(".lock")) + { + return Err(Error::BadKey(format!("{key:?}"))); + } + Ok(()) +} + +/// Prefix check for `list`/`list_meta`: the key charset, so no glob character +/// reaches an `ls-remote` pattern. Empty is allowed. +fn check_prefix(prefix: &str) -> Result<()> { + if prefix.len() > 1024 + || prefix.starts_with('/') + || prefix.contains("..") + || prefix.contains("//") + || !prefix + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'/' | b'-')) + { + return Err(Error::BadKey(format!("{prefix:?}"))); + } + Ok(()) +} + +/// Replace URL userinfo (`user@`) with `***@` for error text. +fn redact(s: &str) -> String { + if let Some((scheme, rest)) = s.split_once("://") { + let end = rest.find('/').unwrap_or(rest.len()); + if let Some((_, host)) = rest[..end].rsplit_once('@') { + return format!("{scheme}://***@{host}{}", &rest[end..]); + } + } + s.to_string() +} + +/// Refuse a value over the per-value cap before building anything. +fn check_len(len: u64) -> Result<()> { + if len > MAX_VALUE { + return Err(Error::Backend(format!( + "value of {len} bytes exceeds the {MAX_VALUE}-byte git bucket cap" + ))); + } + Ok(()) +} + +fn corrupt(why: impl std::fmt::Display) -> Error { + Error::Backend(format!("corrupt git bucket value: {why}")) +} + +fn host_ok(host: &str) -> bool { + !host.is_empty() && !host.starts_with('-') && !host.contains(' ') +} + +impl GitBucket { + /// Validate an inner Git URL (the part after `sc+git::`) without touching + /// the network. Accepts `https://`, `http://`, `ssh://`, `file:///abs`, + /// scp-form `user@host:path` and absolute local paths; refuses everything + /// else (leading `-`, control characters, `::` transport helpers, `git://`, + /// relative paths, and a password in the URL: use git's credential helper). + pub fn validate_url(url: &str) -> Result<()> { + let bad = |why: &str| { + Err(Error::Backend(format!( + "unsupported git url ({why}): {:?}", + redact(url) + ))) + }; + if url.is_empty() || url.len() > 4096 { + return bad("empty or too long"); + } + if url.starts_with('-') { + return bad("starts with '-'"); + } + if url.chars().any(|c| c.is_control()) { + return bad("control character"); + } + if url.contains("::") { + return bad("'::' transport helper"); + } + if let Some((scheme, rest)) = url.split_once("://") { + match scheme { + "file" => { + if !rest.starts_with('/') { + return bad("file:// needs an absolute path (file:///...)"); + } + } + "https" | "http" | "ssh" => { + let end = rest.find('/').unwrap_or(rest.len()); + let (authority, path) = rest.split_at(end); + if path.len() < 2 { + return bad("missing repository path"); + } + let host = match authority.rsplit_once('@') { + Some((userinfo, host)) => { + if userinfo.contains(':') { + return bad("password in url; use a credential helper"); + } + host + } + None => authority, + }; + if !host_ok(host) { + return bad("bad host"); + } + } + _ => return bad("scheme not allowed"), + } + return Ok(()); + } + if std::path::Path::new(url).is_absolute() { + return Ok(()); + } + // scp form: user@host:path + if let Some((left, path)) = url.split_once(':') { + if let Some((user, host)) = left.split_once('@') { + let user_ok = !user.is_empty() + && user + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')); + let host_chars = host.bytes().all(|b| { + b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'[' | b']' | b':') + }); + if user_ok && host_ok(host) && host_chars && !left.contains('/') && !path.is_empty() + { + return Ok(()); + } + } + } + bad("relative path or unrecognized form") + } + + /// Open a bucket stored as `refs/sc/*` in the Git repository at `url` + /// (validates first). Spawns `$SC_GIT` (default `git`). Creates a private + /// temporary cache repository (local only); no network call happens until + /// the first method call. + pub fn open(url: &str) -> Result { + let git = std::env::var("SC_GIT").unwrap_or_else(|_| "git".to_string()); + Self::open_with(url, git, CHUNK) + } + + fn open_with(url: &str, git: String, chunk: usize) -> Result { + Self::validate_url(url)?; + let b = GitBucket { + url: url.to_string(), + git, + cache: new_cache_dir()?, + chunk: chunk.max(1), + state: Mutex::new(HashMap::new()), + }; + // An early `?` drops `b`, which removes the cache dir. + b.run_ok( + &["init", "--bare", "--quiet", "--template=", "."], + None, + &[], + SMALL_CAP, + )?; + Ok(b) + } + + fn lock(&self) -> MutexGuard<'_, HashMap> { + self.state.lock().unwrap_or_else(|p| p.into_inner()) + } + + /// Run git in the cache repository with the hardened environment. + fn run( + &self, + args: &[&str], + stdin: Option<&[u8]>, + env: &[(&str, &str)], + max_out: usize, + ) -> Result { + let mut cmd = Command::new(&self.git); + for c in GIT_CONFIG { + cmd.arg("-c").arg(c); + } + cmd.args(args).current_dir(&self.cache); + for (k, _) in std::env::vars_os() { + if !keep_env(&k.to_string_lossy()) { + cmd.env_remove(&k); + } + } + cmd.env("GIT_DIR", &self.cache) + .env("LC_ALL", "C") + .env("GIT_TERMINAL_PROMPT", "0"); + for (k, v) in env { + cmd.env(k, v); + } + cmd.stdin(if stdin.is_some() { + Stdio::piped() + } else { + Stdio::null() + }) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let mut child = cmd.spawn().map_err(|e| { + Error::Backend(format!( + "cannot run git ({}): {e} (install git or set SC_GIT)", + self.git + )) + })?; + let sin = child.stdin.take(); + let mut sout = child.stdout.take().expect("piped"); + let mut serr = child.stderr.take().expect("piped"); + let (stdout, stderr, over) = std::thread::scope(|s| { + if let (Some(mut w), Some(data)) = (sin, stdin) { + // A write error means the child already exited; its status says why. + s.spawn(move || { + let _ = w.write_all(data); + }); + } + let eh = s.spawn(move || { + let mut buf = Vec::new(); + let _ = (&mut serr).take(SMALL_CAP as u64).read_to_end(&mut buf); + let _ = std::io::copy(&mut serr, &mut std::io::sink()); + buf + }); + let mut out = Vec::new(); + let _ = (&mut sout).take(max_out as u64 + 1).read_to_end(&mut out); + let over = out.len() > max_out; + if over { + let _ = child.kill(); + } + (out, eh.join().unwrap_or_default(), over) + }); + let status = child.wait()?; + if over { + return Err(Error::Backend(format!( + "git output exceeded {max_out} bytes (hostile or corrupt remote)" + ))); + } + Ok(Out { + success: status.success(), + code: status.code(), + stdout, + stderr: self.scrub(&String::from_utf8_lossy(&stderr)), + }) + } + + fn run_ok( + &self, + args: &[&str], + stdin: Option<&[u8]>, + env: &[(&str, &str)], + max_out: usize, + ) -> Result { + let out = self.run(args, stdin, env, max_out)?; + if out.success { + Ok(out) + } else { + Err(self.fail(args.first().copied().unwrap_or("git"), &out)) + } + } + + /// Error text safe to print: URL userinfo redacted, control bytes (ESC + /// from a hostile hook message) dropped, length bounded. + fn scrub(&self, text: &str) -> String { + let t = text.replace(&self.url, &redact(&self.url)); + t.chars() + .filter(|c| !c.is_control() || *c == '\n') + .take(4096) + .collect::() + .trim() + .to_string() + } + + fn fail(&self, what: &str, out: &Out) -> Error { + Error::Backend(format!( + "git {what} failed (exit {}): {}", + out.code.map_or("signal".to_string(), |c| c.to_string()), + out.stderr + )) + } + + fn out_oid(out: &Out) -> Result { + let s = String::from_utf8_lossy(&out.stdout).trim().to_string(); + if is_oid(&s) { + Ok(s) + } else { + Err(Error::Backend(format!( + "git returned a bad object id: {s:?}" + ))) + } + } + + /// `ls-remote` lines `(oid, refname)` for `pattern`. Patterns tail-match, so + /// callers must compare full refnames themselves. + fn ls_remote(&self, pattern: &str) -> Result> { + let out = self.run( + &[ + "ls-remote", + "--refs", + "--end-of-options", + &self.url, + pattern, + ], + None, + &[], + LS_REMOTE_CAP, + )?; + if !out.success { + return Err(self.fail("ls-remote", &out)); + } + let mut v = Vec::new(); + for line in String::from_utf8_lossy(&out.stdout).lines() { + match line.split_once('\t') { + Some((oid, name)) if is_oid(oid) => v.push((oid.to_string(), name.to_string())), + _ => return Err(Error::Backend(format!("bad ls-remote line: {line:?}"))), + } + } + Ok(v) + } + + /// The remote oid of exactly `refs/sc/`. + fn remote_ref(&self, key: &str) -> Result> { + let full = format!("{NS}{key}"); + Ok(self + .ls_remote(&full)? + .into_iter() + .find(|(_, name)| *name == full) + .map(|(oid, _)| oid)) + } + + /// Remote `(key, oid)` pairs whose key starts with the raw `prefix`, sorted. + fn remote_list(&self, prefix: &str) -> Result> { + let mut v: Vec<(String, String)> = self + .ls_remote(&format!("{NS}{prefix}*"))? + .into_iter() + .filter_map(|(oid, name)| { + let key = name.strip_prefix(NS)?.to_string(); + (key.starts_with(prefix) && check_key(&key).is_ok()).then_some((key, oid)) + }) + .collect(); + v.sort(); + Ok(v) + } + + /// Fetch `refspecs`. `family` = a glob fetch of a whole key family: it + /// prunes stale local refs and omits `--depth=1`, because a shallow fetch + /// that matches no ref exits 1 with no error text, which cannot be told + /// from a failure. Exact-ref fetches stay shallow so a hostile commit's + /// parents are never downloaded. + fn fetch(&self, refspecs: &[String], family: bool) -> Result { + let mut a = vec![ + "fetch", + "--no-tags", + "--no-write-fetch-head", + "--no-recurse-submodules", + ]; + a.push(if family { "--prune" } else { "--depth=1" }); + a.extend(["--end-of-options", &self.url]); + a.extend(refspecs.iter().map(String::as_str)); + self.run(&a, None, &[], SMALL_CAP) + } + + /// Push `oid` to `refs/sc/` (or delete it when `oid` is `None`) under + /// a lease: `Some("")` = must not exist, `Some(oid)` = must equal `oid`. + /// Never a `+` refspec. + fn push(&self, key: &str, oid: Option<&str>, lease: Option<&str>) -> Result { + let full = format!("{NS}{key}"); + let refspec = format!("{}:{full}", oid.unwrap_or("")); + let lease_arg = lease.map(|l| format!("--force-with-lease={full}:{l}")); + let mut a = vec!["push", "--porcelain", "--no-verify"]; + if let Some(l) = &lease_arg { + a.push(l); + } + a.extend(["--end-of-options", &self.url, &refspec]); + let out = self.run(&a, None, &[], SMALL_CAP)?; + let landed = out.success + && (oid.is_none() + || String::from_utf8_lossy(&out.stdout).lines().any(|line| { + let mut f = line.split('\t'); + let flag = f.next().unwrap_or(""); + let to = f.next().and_then(|r| r.rsplit_once(':')).map(|(_, t)| t); + to == Some(full.as_str()) && matches!(flag, " " | "+" | "*") + })); + if landed { + Ok(Pushed::Done) + } else { + let first = out + .stdout + .split(|b| *b == b'\n') + .find(|l| l.first() == Some(&b'!')) + .map(|l| self.scrub(&String::from_utf8_lossy(l))) + .unwrap_or_default(); + Ok(Pushed::Refused(format!( + "git push failed (exit {}): {} {}", + out.code.map_or("signal".to_string(), |c| c.to_string()), + out.stderr, + first + ))) + } + } + + /// Build the value commit locally: chunk blobs, tree, parentless commit + /// with a unique nonce so a fresh write can never be "up to date". + fn make_commit(&self, bytes: &[u8], kind: &str) -> Result { + check_len(bytes.len() as u64)?; + let mut tree = String::new(); + for (i, c) in bytes.chunks(self.chunk).enumerate() { + let o = self.run_ok( + &["hash-object", "-w", "--no-filters", "--stdin"], + Some(c), + &[], + SMALL_CAP, + )?; + tree.push_str(&format!("100644 blob {}\tc{i:04}\n", Self::out_oid(&o)?)); + } + let t = Self::out_oid(&self.run_ok(&["mktree"], Some(tree.as_bytes()), &[], SMALL_CAP)?)?; + let msg = format!("sc-bucket {kind} {}", nonce(&self.cache)); + let c = self.run_ok( + &["commit-tree", "--no-gpg-sign", "-m", &msg, &t], + None, + IDENTITY_ENV, + SMALL_CAP, + )?; + Self::out_oid(&c) + } + + /// Tree id and create-only flag of a cached value commit; refuses parents. + fn commit_info(&self, oid: &str) -> Result<(String, bool)> { + let out = self.run_ok(&["cat-file", "commit", oid], None, &[], SMALL_CAP)?; + let text = String::from_utf8_lossy(&out.stdout); + let (head, msg) = text.split_once("\n\n").unwrap_or((&text, "")); + let mut tree = None; + for line in head.lines() { + if line.starts_with("parent ") { + return Err(corrupt("commit has a parent")); + } + if let Some(t) = line.strip_prefix("tree ") { + tree.get_or_insert(t.to_string()); + } + } + match tree { + Some(t) if is_oid(&t) => Ok((t, msg.starts_with("sc-bucket new "))), + _ => Err(corrupt("commit has no tree")), + } + } + + /// Validate a cached value's shape and sizes without reading any content. + fn inspect(&self, oid: &str) -> Result<(bool, Vec<(String, u64)>)> { + let (tree, create_only) = self.commit_info(oid)?; + let raw = self + .run_ok(&["cat-file", "tree", &tree], None, &[], SMALL_CAP)? + .stdout; + let mut oids = Vec::new(); + let mut pos = 0; + while pos < raw.len() { + let sp = raw[pos..].iter().position(|b| *b == b' '); + let nul = raw[pos..].iter().position(|b| *b == 0); + let (Some(sp), Some(nul)) = (sp, nul) else { + return Err(corrupt("malformed tree")); + }; + if sp > nul || raw.len() < pos + nul + 21 { + return Err(corrupt("malformed tree")); + } + let mode = &raw[pos..pos + sp]; + let name = &raw[pos + sp + 1..pos + nul]; + let i = oids.len(); + if i >= MAX_CHUNKS { + return Err(corrupt("too many chunks")); + } + if mode != b"100644" || name != format!("c{i:04}").as_bytes() { + return Err(corrupt("tree is not contiguous regular chunk blobs")); + } + oids.push(hex::encode(&raw[pos + nul + 1..pos + nul + 21])); + pos += nul + 21; + } + let input: String = oids.iter().map(|o| format!("{o}\n")).collect(); + let out = self.run_ok( + &["cat-file", "--batch-check"], + Some(input.as_bytes()), + &[], + SMALL_CAP * 4, + )?; + let text = String::from_utf8_lossy(&out.stdout); + let mut chunks = Vec::new(); + let mut total = 0u64; + for (o, line) in oids.iter().zip(text.lines()) { + let f: Vec<&str> = line.split(' ').collect(); + let size: u64 = match f.as_slice() { + [_, "blob", size] => size.parse().map_err(|_| corrupt("bad chunk size"))?, + _ => return Err(corrupt("chunk is not a present blob")), + }; + if size > CHUNK as u64 { + return Err(corrupt(format!("chunk of {size} bytes exceeds {CHUNK}"))); + } + total += size; + if total > MAX_VALUE { + return Err(corrupt(format!("value exceeds the {MAX_VALUE}-byte cap"))); + } + chunks.push((o.clone(), size)); + } + if chunks.len() != oids.len() { + return Err(corrupt("chunk listing incomplete")); + } + Ok((create_only, chunks)) + } + + fn read_chunks(&self, chunks: &[(String, u64)]) -> Result> { + let total: u64 = chunks.iter().map(|c| c.1).sum(); + let mut bytes = Vec::with_capacity(total as usize); + for (o, size) in chunks { + let out = self.run_ok(&["cat-file", "blob", o], None, &[], CHUNK + 1)?; + if out.stdout.len() as u64 != *size { + return Err(corrupt("chunk size changed")); + } + bytes.extend_from_slice(&out.stdout); + } + Ok(bytes) + } + + /// Answer a read for `oid` already present in the cache; records it as known. + fn finish_read( + &self, + st: &mut HashMap, + key: &str, + oid: &str, + cached: Option<&str>, + ) -> Result { + let (create_only, chunks) = self.inspect(oid)?; + st.insert( + key.to_string(), + Known { + oid: oid.to_string(), + create_only, + at: Instant::now(), + }, + ); + if cached == Some(oid) { + return Ok(Fetched::Unchanged); + } + Ok(Fetched::New { + bytes: self.read_chunks(&chunks)?, + tag: oid.to_string(), + }) + } + + /// Local refs under `prefix` as `(key, oid, objecttype)`. + fn local_refs(&self, prefix: &str) -> Result> { + let out = self.run_ok( + &[ + "for-each-ref", + "--format=%(objectname) %(objecttype) %(refname)", + prefix, + ], + None, + &[], + LS_REMOTE_CAP, + )?; + Ok(String::from_utf8_lossy(&out.stdout) + .lines() + .filter_map(|l| { + let mut f = l.splitn(3, ' '); + let (oid, ty, name) = (f.next()?, f.next()?, f.next()?); + Some((name.strip_prefix(NS)?.to_string(), oid.into(), ty.into())) + }) + .collect()) + } + + fn get_family( + &self, + st: &mut HashMap, + key: &str, + cached: Option<&str>, + fam: &Family, + ) -> Result { + let pat = fam.glob(); + let out = self.fetch(&[format!("+{pat}:{pat}")], true)?; + if !out.success { + return Err(self.fail("fetch", &out)); + } + let local: Vec<_> = self + .local_refs(&format!("{NS}{}", fam.dir))? + .into_iter() + .filter(|(k, _, _)| fam.matches(k)) + .collect(); + st.retain(|k, _| !fam.matches(k) || local.iter().any(|(l, _, _)| l == k)); + for (k, oid, ty) in &local { + if k == key && ty != "commit" { + return Err(corrupt("ref does not point at a commit")); + } + if ty != "commit" { + continue; + } + match st.get_mut(k) { + Some(known) if known.oid == *oid => known.at = Instant::now(), + _ => { + let (_, create_only) = self.commit_info(oid)?; + st.insert( + k.clone(), + Known { + oid: oid.clone(), + create_only, + at: Instant::now(), + }, + ); + } + } + } + match st.get(key) { + None => Ok(Fetched::Absent), + Some(k) => { + let oid = k.oid.clone(); + self.answer(&oid, cached) + } + } + } + + fn answer(&self, oid: &str, cached: Option<&str>) -> Result { + if cached == Some(oid) { + return Ok(Fetched::Unchanged); + } + let (_, chunks) = self.inspect(oid)?; + Ok(Fetched::New { + bytes: self.read_chunks(&chunks)?, + tag: oid.to_string(), + }) + } + + fn put(&self, key: &str, bytes: &[u8], kind: &str, lease: &str) -> Result> { + let oid = self.make_commit(bytes, kind)?; + match self.push(key, Some(&oid), Some(lease))? { + Pushed::Done => { + self.lock().insert( + key.to_string(), + Known { + oid: oid.clone(), + create_only: kind == "new", + at: Instant::now(), + }, + ); + Ok(Ok(oid)) + } + Pushed::Refused(msg) => Ok(Err(Error::Backend(msg))), + } + } +} + +impl Drop for GitBucket { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.cache); + } +} + +/// Fresh private temp directory `sc-git-bucket--` (mode 0700). +fn new_cache_dir() -> Result { + for attempt in 0u64..16 { + let name = format!( + "sc-git-bucket-{}-{}", + std::process::id(), + &nonce_seed(attempt)[..16] + ); + let p = std::env::temp_dir().join(name); + let mut b = std::fs::DirBuilder::new(); + #[cfg(unix)] + std::os::unix::fs::DirBuilderExt::mode(&mut b, 0o700); + match b.create(&p) { + Ok(()) => return Ok(p), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(e.into()), + } + } + Err(Error::Backend( + "cannot create a unique cache directory".into(), + )) +} + +fn nonce_seed(extra: u64) -> String { + use std::sync::atomic::{AtomicU64, Ordering}; + static COUNTER: AtomicU64 = AtomicU64::new(0); + let nanos = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()); + let mut h = blake3::Hasher::new(); + h.update(&nanos.to_le_bytes()); + h.update(&COUNTER.fetch_add(1, Ordering::Relaxed).to_le_bytes()); + h.update(&std::process::id().to_le_bytes()); + h.update(&extra.to_le_bytes()); + // Address entropy separates threads racing within one clock tick. + let local = 0u8; + h.update(&(std::ptr::addr_of!(local) as usize).to_le_bytes()); + hex::encode(h.finalize().as_bytes()) +} + +fn nonce(cache: &std::path::Path) -> String { + let mut h = blake3::Hasher::new(); + h.update(nonce_seed(0).as_bytes()); + h.update(cache.as_os_str().as_encoded_bytes()); + hex::encode(&h.finalize().as_bytes()[..16]) +} + +impl Bucket for GitBucket { + fn get(&self, key: &str, cached_tag: Option<&str>) -> Result { + check_key(key)?; + let mut st = self.lock(); + if let Some(k) = st.get(key) { + if k.create_only { + let oid = k.oid.clone(); + return self.answer(&oid, cached_tag); + } + if cached_tag == Some(k.oid.as_str()) && k.at.elapsed() < FRESH { + return Ok(Fetched::Unchanged); + } + } + if let Some(fam) = family_of(key) { + return self.get_family(&mut st, key, cached_tag, fam); + } + let Some(remote) = self.remote_ref(key)? else { + st.remove(key); + return Ok(Fetched::Absent); + }; + if cached_tag == Some(remote.as_str()) { + st.entry(key.to_string()) + .and_modify(|k| k.at = Instant::now()) + .or_insert(Known { + oid: remote, + create_only: false, + at: Instant::now(), + }); + return Ok(Fetched::Unchanged); + } + let full = format!("{NS}{key}"); + let out = self.fetch(&[format!("+{full}:{full}")], false)?; + if !out.success { + // The ref may have vanished between ls-remote and fetch; only the + // exact full refname decides (patterns tail-match). + return match self.remote_ref(key)? { + None => { + st.remove(key); + Ok(Fetched::Absent) + } + Some(_) => Err(self.fail("fetch", &out)), + }; + } + let rp = self.run_ok( + &[ + "rev-parse", + "--verify", + "--quiet", + &format!("{full}^{{commit}}"), + ], + None, + &[], + SMALL_CAP, + ); + let oid = match rp { + Ok(o) => Self::out_oid(&o)?, + Err(_) => return Err(corrupt("ref does not point at a commit")), + }; + self.finish_read(&mut st, key, &oid, cached_tag) + } + + fn put_new(&self, key: &str, bytes: &[u8]) -> Result { + check_key(key)?; + match self.put(key, bytes, "new", "")? { + Ok(_) => Ok(true), + Err(refusal) => match self.remote_ref(key) { + Ok(Some(_)) => Ok(false), + _ => Err(refusal), + }, + } + } + + fn put_if_tag( + &self, + key: &str, + bytes: &[u8], + expected_tag: Option<&str>, + ) -> Result> { + check_key(key)?; + let lease = match expected_tag { + None => "", + Some(t) if is_tag(t) => t, + Some(_) => return Ok(None), + }; + match self.put(key, bytes, "cas", lease)? { + Ok(oid) => Ok(Some(oid)), + Err(refusal) => match self.remote_ref(key) { + // Someone else holds a different value (or deleted it): lost the race. + Ok(Some(cur)) if cur != lease => Ok(None), + Ok(None) if expected_tag.is_some() => Ok(None), + // The ref still holds what we expected (or a create found nothing): + // a genuine refusal (hook, permissions), never a lost race. + _ => Err(refusal), + }, + } + } + + fn list(&self, prefix: &str) -> Result> { + check_prefix(prefix)?; + Ok(self + .remote_list(prefix)? + .into_iter() + .map(|(k, _)| k) + .collect()) + } + + // shortcut: downloads every listed value to read its size; only compaction + // (refused on git remotes in v1) and tests call it. Upgrade to a filtered + // fetch if listing large stores becomes slow. + fn list_meta(&self, prefix: &str) -> Result> { + check_prefix(prefix)?; + let listed = self.remote_list(prefix)?; + let mut st = self.lock(); + let missing: Vec = listed + .iter() + .filter(|(k, oid)| st.get(k).is_none_or(|known| known.oid != *oid)) + .map(|(k, _)| format!("+{NS}{k}:{NS}{k}")) + .collect(); + for group in missing.chunks(FETCH_BATCH) { + let out = self.fetch(group, false)?; + if !out.success { + return Err(self.fail("fetch", &out)); + } + } + let now = SystemTime::now(); + let mut metas = Vec::new(); + for (key, oid) in listed { + let (create_only, chunks) = self.inspect(&oid)?; + st.insert( + key.clone(), + Known { + oid, + create_only, + at: Instant::now(), + }, + ); + metas.push(ObjectMeta { + key, + size: chunks.iter().map(|c| c.1).sum(), + // Committer time is writer-controlled, so the observation time + // stands in: compaction never prunes on it (ADR-0060). + modified: now, + }); + } + Ok(metas) + } + + fn exists(&self, key: &str) -> Result { + check_key(key)?; + Ok(self.remote_ref(key)?.is_some()) + } + + fn delete(&self, key: &str) -> Result<()> { + check_key(key)?; + let mut st = self.lock(); + st.remove(key); + match self.push(key, None, None)? { + Pushed::Done => Ok(()), + Pushed::Refused(msg) => match self.remote_ref(key) { + Ok(None) => Ok(()), + _ => Err(Error::Backend(msg)), + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::contract_suite; + use std::path::Path; + use std::process::Command; + use std::sync::atomic::{AtomicU64, Ordering}; + + fn have_git() -> bool { + Command::new("git") + .arg("--version") + .output() + .is_ok_and(|o| o.status.success()) + } + + macro_rules! need_git { + () => { + if !have_git() { + eprintln!("git not available; skipping"); + return; + } + }; + } + + /// A scratch directory removed on drop, asserting it is gone. + struct Tmp(PathBuf); + + impl Tmp { + fn new(tag: &str) -> Tmp { + static N: AtomicU64 = AtomicU64::new(0); + let p = std::env::temp_dir().join(format!( + "scl-objio-git-{}-{tag}-{}", + std::process::id(), + N.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&p).unwrap(); + Tmp(p) + } + } + + impl Drop for Tmp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + if !std::thread::panicking() { + assert!(!self.0.exists(), "scratch dir must be removed"); + } + } + } + + /// Run real git (isolated config, fixed identity); returns trimmed stdout. + fn git(dir: &Path, args: &[&str], stdin: &str) -> String { + let mut c = Command::new("git"); + c.args(args) + .current_dir(dir) + .env_remove("GIT_DIR") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_AUTHOR_NAME", "t") + .env("GIT_AUTHOR_EMAIL", "t@invalid") + .env("GIT_COMMITTER_NAME", "t") + .env("GIT_COMMITTER_EMAIL", "t@invalid") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let mut child = c.spawn().unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(stdin.as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + /// A bare hub repository; returns (path, file:// url). + fn hub(t: &Tmp) -> (PathBuf, String) { + let p = t.0.join("hub.git"); + git(&t.0, &["init", "--bare", "-q", p.to_str().unwrap()], ""); + let url = format!("file://{}", p.display()); + (p, url) + } + + fn open(url: &str) -> GitBucket { + GitBucket::open(url).unwrap() + } + + #[cfg(unix)] + fn shim(t: &Tmp, name: &str, body: &str) -> String { + use std::os::unix::fs::PermissionsExt; + let p = t.0.join(name); + std::fs::write(&p, format!("#!/bin/sh\n{body}\nexec git \"$@\"\n")).unwrap(); + std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap(); + p.to_str().unwrap().to_string() + } + + fn open_git(url: &str, git: &str) -> GitBucket { + GitBucket::open_with(url, git.to_string(), CHUNK).unwrap() + } + + fn tag_of(b: &GitBucket, key: &str) -> String { + let Fetched::New { tag, .. } = b.get(key, None).unwrap() else { + panic!("expected New") + }; + tag + } + + #[test] + fn git_bucket_passes_contract() { + need_git!(); + let t = Tmp::new("contract"); + let (_, url) = hub(&t); + contract_suite(&open(&url)); + } + + #[test] + fn racing_cas_has_exactly_one_winner() { + need_git!(); + let t = Tmp::new("racecas"); + let (_, url) = hub(&t); + for round in 0..4 { + let key = format!("manifest{round}"); + let base = open(&url); + let tag = base.put_if_tag(&key, b"v0", None).unwrap().unwrap(); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let hs: Vec<_> = (0..2) + .map(|i| { + let (url, key, tag, barrier) = + (url.clone(), key.clone(), tag.clone(), barrier.clone()); + std::thread::spawn(move || { + let b = GitBucket::open(&url).unwrap(); + barrier.wait(); + b.put_if_tag(&key, format!("w{i}").as_bytes(), Some(&tag)) + .unwrap() + }) + }) + .collect(); + let wins = hs + .into_iter() + .map(|h| h.join().unwrap()) + .filter(Option::is_some) + .count(); + assert_eq!(wins, 1, "round {round}"); + } + } + + #[test] + fn racing_create_has_exactly_one_winner() { + need_git!(); + let t = Tmp::new("racenew"); + let (_, url) = hub(&t); + for round in 0..4 { + let key = format!("log/{round}"); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let hs: Vec<_> = (0..2) + .map(|i| { + let (url, key, barrier) = (url.clone(), key.clone(), barrier.clone()); + std::thread::spawn(move || { + let b = GitBucket::open(&url).unwrap(); + barrier.wait(); + b.put_new(&key, format!("w{i}").as_bytes()).unwrap() + }) + }) + .collect(); + let wins = hs + .into_iter() + .map(|h| h.join().unwrap()) + .filter(|w| *w) + .count(); + assert_eq!(wins, 1, "round {round}"); + } + } + + #[test] + fn stale_and_malformed_tags_lose() { + need_git!(); + let t = Tmp::new("tags"); + let (_, url) = hub(&t); + let b = open(&url); + b.put_if_tag("manifest", b"v1", None).unwrap().unwrap(); + let stale = "1".repeat(40); + assert!(b + .put_if_tag("manifest", b"x", Some(&stale)) + .unwrap() + .is_none()); + assert!(b + .put_if_tag("manifest", b"x", Some("zz-not-hex")) + .unwrap() + .is_none()); + assert!(b + .put_if_tag("manifest", b"x", Some(&"0".repeat(40))) + .unwrap() + .is_none()); + assert!(b + .put_if_tag("manifest", b"x", Some(&"A".repeat(40))) + .unwrap() + .is_none()); + let Fetched::New { bytes, .. } = b.get("manifest", None).unwrap() else { + panic!() + }; + assert_eq!(bytes, b"v1"); + // a deleted key with a stale tag also loses rather than erroring + b.delete("manifest").unwrap(); + assert!(b + .put_if_tag("manifest", b"x", Some(&stale)) + .unwrap() + .is_none()); + } + + #[cfg(unix)] + #[test] + fn rejecting_hook_is_an_error_not_a_lost_race() { + use std::os::unix::fs::PermissionsExt; + need_git!(); + let t = Tmp::new("hook"); + let (hubdir, url) = hub(&t); + let b = open(&url); + let tag = b.put_if_tag("manifest", b"v1", None).unwrap().unwrap(); + let hook = hubdir.join("hooks/pre-receive"); + std::fs::write( + &hook, + "#!/bin/sh\nprintf 'nope \\033[31mred\\n' >&2\nexit 1\n", + ) + .unwrap(); + std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap(); + let start = Instant::now(); + let e1 = b.put_new("log/1", b"x").unwrap_err().to_string(); + let e2 = b + .put_if_tag("manifest", b"v2", Some(&tag)) + .unwrap_err() + .to_string(); + assert!(start.elapsed() < Duration::from_secs(30)); + assert!(e1.contains("nope"), "{e1}"); + assert!( + !e1.contains('\u{1b}') && !e2.contains('\u{1b}'), + "ESC leaked" + ); + // the refused write left the store untouched + assert!(!b.exists("log/1").unwrap()); + assert_eq!(tag_of(&open(&url), "manifest"), tag); + // delete is refused too, and is not mistaken for idempotent success + assert!(b.delete("manifest").is_err()); + } + + #[test] + fn decoy_branch_does_not_confuse_reads() { + need_git!(); + let t = Tmp::new("decoy"); + let (hubdir, url) = hub(&t); + let b = open(&url); + b.put_new("other", b"o").unwrap(); + let oid = git(&hubdir, &["rev-parse", "refs/sc/other"], ""); + git( + &hubdir, + &["update-ref", "refs/heads/x/refs/sc/manifest", &oid], + "", + ); + git( + &hubdir, + &["update-ref", "refs/heads/x/refs/sc/log/1", &oid], + "", + ); + let r = open(&url); + assert!(matches!(r.get("manifest", None).unwrap(), Fetched::Absent)); + assert!(matches!(r.get("log/1", None).unwrap(), Fetched::Absent)); + assert!(!r.exists("manifest").unwrap()); + assert_eq!(r.list("").unwrap(), vec!["other".to_string()]); + assert_eq!(r.list("log/").unwrap(), Vec::::new()); + assert!(r.list_meta("").unwrap().len() == 1); + // a real manifest next to the decoy is the one read + assert!(r.put_new("manifest", b"real").unwrap()); + let Fetched::New { bytes, .. } = open(&url).get("manifest", None).unwrap() else { + panic!() + }; + assert_eq!(bytes, b"real"); + } + + #[test] + fn values_larger_than_a_chunk_round_trip() { + need_git!(); + let t = Tmp::new("chunks"); + let (hubdir, url) = hub(&t); + let small = GitBucket::open_with(&url, "git".into(), 4).unwrap(); + let big: Vec = (0..26u8).collect(); // 7 chunks of at most 4 bytes + assert!(small.put_new("packs/big.pack", &big).unwrap()); + assert!(small.put_new("exact", &big[..8]).unwrap()); // exact multiple + assert!(small.put_new("empty", b"").unwrap()); + let tree = git(&hubdir, &["ls-tree", "refs/sc/packs/big.pack"], ""); + assert_eq!(tree.lines().count(), 7); + let r = open(&url); // fresh cache: exercises the fetch path + for (k, v) in [ + ("packs/big.pack", &big[..]), + ("exact", &big[..8]), + ("empty", &b""[..]), + ] { + let Fetched::New { bytes, .. } = r.get(k, None).unwrap() else { + panic!("{k}") + }; + assert_eq!(bytes, v, "{k}"); + } + let metas = r.list_meta("").unwrap(); + let sizes: Vec<(&str, u64)> = metas.iter().map(|m| (m.key.as_str(), m.size)).collect(); + assert_eq!( + sizes, + vec![("empty", 0), ("exact", 8), ("packs/big.pack", 26)] + ); + } + + /// Plant a value commit straight into the hub, bypassing GitBucket. + fn plant(hubdir: &Path, key: &str, parent: bool, entries: &[(&str, &str, &str)]) { + let blob = git(hubdir, &["hash-object", "-w", "--stdin"], "data"); + let empty = git(hubdir, &["mktree"], ""); + let lines: String = entries + .iter() + .map(|(mode, ty, name)| { + let oid = if *ty == "tree" { &empty } else { &blob }; + format!("{mode} {ty} {oid}\t{name}\n") + }) + .collect(); + let tree = git(hubdir, &["mktree"], &lines); + let mut args = vec!["commit-tree", &tree, "-m", "planted"]; + let base; + if parent { + base = git(hubdir, &["commit-tree", &empty, "-m", "base"], ""); + args.extend(["-p", &base]); + } + let c = git(hubdir, &args, ""); + git(hubdir, &["update-ref", &format!("refs/sc/{key}"), &c], ""); + } + + #[test] + fn corrupt_value_shapes_are_refused() { + need_git!(); + let t = Tmp::new("corrupt"); + let (hubdir, url) = hub(&t); + let ok = [("100644", "blob", "c0000")]; + for (key, parent, entries) in [ + ("k-parent", true, &ok[..]), + ("k-subtree", false, &[("040000", "tree", "c0000")][..]), + ( + "k-gap", + false, + &[("100644", "blob", "c0000"), ("100644", "blob", "c0002")][..], + ), + ("k-name", false, &[("100644", "blob", "evil")][..]), + ("k-exec", false, &[("100755", "blob", "c0000")][..]), + // family keys take the batched path + ("log/1", true, &ok[..]), + ( + "log/2", + false, + &[("100644", "blob", "c0000"), ("100644", "blob", "c0002")][..], + ), + ] { + plant(&hubdir, key, parent, entries); + let err = open(&url).get(key, None); + assert!( + matches!(&err, Err(Error::Backend(m)) if m.contains("corrupt")), + "{key}: {:?}", + err.err() + ); + } + // sanity: a well-formed planted value reads fine + plant(&hubdir, "k-ok", false, &ok); + assert!(matches!( + open(&url).get("k-ok", None).unwrap(), + Fetched::New { .. } + )); + } + + #[cfg(unix)] + #[test] + fn ref_vanishing_between_ls_remote_and_fetch_is_absent() { + need_git!(); + let t = Tmp::new("vanish"); + let (hubdir, url) = hub(&t); + open(&url).put_new("k", b"v").unwrap(); + // The shim deletes the ref in the hub just before the fetch runs. + let body = format!( + "for a in \"$@\"; do if [ \"$a\" = fetch ]; then git --git-dir={} update-ref -d refs/sc/k; fi; done", + hubdir.display() + ); + let sh = shim(&t, "vanish.sh", &body); + let r = open_git(&url, &sh); + assert!(matches!(r.get("k", None).unwrap(), Fetched::Absent)); + assert!(!r.exists("k").unwrap()); + } + + #[test] + fn url_validation() { + for bad in [ + "", + "-oProxyCommand=x", + "--upload-pack=x", + "https://h/x\n", + "https://h/x\r", + "https://h/x\0", + "https://h/x\u{1b}", + "ext::sh -c id", + "https://h/a::b", + "git://host/x.git", + "ftp://host/x", + "relative/path", + "./x", + "host:path", + "https://user:pass@host/x", + "ssh://user:pw@host/x", + "https://-evil/x", + "ssh://git@-evil/x", + "git@-evil:x", + "https://host", + "file://host/x", + "file://rel", + "git@host:", + "@host:x", + "us er@host:x", + "foo/bar@host:x", + ] { + assert!(GitBucket::validate_url(bad).is_err(), "{bad:?}"); + } + for good in [ + "https://host/x.git", + "http://127.0.0.1:8080/x.git", + "https://user@host/x.git", + "ssh://git@host/x", + "ssh://git@host:2222/org/x.git", + "git@host:org/x.git", + "git@github.com:org/repo.git", + "file:///abs/path/x.git", + "/abs/path/x.git", + "/abs/with space/x.git", + ] { + GitBucket::validate_url(good).unwrap_or_else(|e| panic!("{good:?}: {e}")); + } + // a password never appears in the error text + let e = GitBucket::validate_url("https://u:hunter2@host/x") + .unwrap_err() + .to_string(); + assert!(!e.contains("hunter2"), "{e}"); + assert_eq!(redact("https://u:pw@h/x"), "https://***@h/x"); + } + + #[test] + fn keys_are_validated_on_every_method() { + need_git!(); + let t = Tmp::new("keys"); + let (_, url) = hub(&t); + let b = open(&url); + for bad in [ + "a.lock", ".x", "a..b", "*", "a b", "a/.x", "a.", "x.lock/y", "a?b", "a[b", "a:b", + "a~b", "a^b", "a\\b", "", "/a", "a//b", "é", "a@{b", + ] { + let is_bad = |r: Result<()>| matches!(r, Err(Error::BadKey(_))); + assert!(is_bad(b.get(bad, None).map(drop)), "get {bad:?}"); + assert!(is_bad(b.put_new(bad, b"x").map(drop)), "put_new {bad:?}"); + assert!( + is_bad(b.put_if_tag(bad, b"x", None).map(drop)), + "put_if_tag {bad:?}" + ); + assert!(is_bad(b.exists(bad).map(drop)), "exists {bad:?}"); + assert!(is_bad(b.delete(bad)), "delete {bad:?}"); + } + for bad in ["*", "a b", "a?", "/a", "a..b", "log//"] { + assert!(matches!(b.list(bad), Err(Error::BadKey(_))), "list {bad:?}"); + assert!( + matches!(b.list_meta(bad), Err(Error::BadKey(_))), + "list_meta {bad:?}" + ); + } + assert!(b.list("").unwrap().is_empty()); + } + + #[test] + fn list_is_a_raw_string_prefix() { + need_git!(); + let t = Tmp::new("list"); + let (_, url) = hub(&t); + let b = open(&url); + for k in ["log/1", "logger/x", "manifest", "packs/a.pack"] { + b.put_new(k, b"x").unwrap(); + } + assert_eq!(b.list("log").unwrap(), vec!["log/1", "logger/x"]); + assert_eq!(b.list("log/").unwrap(), vec!["log/1"]); + assert_eq!(b.list("").unwrap().len(), 4); + } + + #[cfg(unix)] + #[test] + fn puts_survive_hostile_user_git_config() { + need_git!(); + let t = Tmp::new("cfg"); + let (_, url) = hub(&t); + // no config at all, so no identity either + let none = shim(&t, "none.sh", "export GIT_CONFIG_GLOBAL=/dev/null"); + let b = open_git(&url, &none); + assert!(b.put_new("a", b"1").unwrap()); + // signing on and an unusable gpg program, a hooks dir, and a bad identity + let cfg = t.0.join("gitconfig"); + std::fs::write( + &cfg, + "[commit]\n\tgpgSign = true\n[tag]\n\tgpgSign = true\n[push]\n\tgpgSign = true\n[gpg]\n\tprogram = /nonexistent/gpg\n[user]\n\tname = X\n\temail = x@x\n[core]\n\thooksPath = /nonexistent/hooks\n", + ) + .unwrap(); + let signing = shim( + &t, + "sign.sh", + &format!("export GIT_CONFIG_GLOBAL={}", cfg.display()), + ); + let b = open_git(&url, &signing); + assert!(b.put_new("b", b"2").unwrap()); + let tag = b.put_if_tag("m", b"1", None).unwrap().unwrap(); + assert!(b.put_if_tag("m", b"2", Some(&tag)).unwrap().is_some()); + assert_eq!(b.list("").unwrap(), vec!["a", "b", "m"]); + } + + #[test] + fn env_scrub_keeps_transport_and_drops_redirects() { + for keep in [ + "PATH", + "HOME", + "SSH_AUTH_SOCK", + "GIT_SSH_COMMAND", + "GIT_CONFIG_GLOBAL", + ] { + assert!(keep_env(keep), "{keep}"); + } + for drop in [ + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + "GIT_INDEX_FILE", + "GIT_CONFIG_PARAMETERS", + "GIT_CONFIG_COUNT", + "GIT_CONFIG_KEY_0", + "GIT_NAMESPACE", + "GIT_COMMON_DIR", + "GIT_PREFIX", + "GIT_ALLOW_PROTOCOL", + "GIT_TRACE", + ] { + assert!(!keep_env(drop), "{drop}"); + } + } + + #[test] + fn cache_dir_is_private_and_removed_on_drop() { + need_git!(); + let t = Tmp::new("drop"); + let (_, url) = hub(&t); + let b = open(&url); + b.put_new("k", b"v").unwrap(); + let cache = b.cache.clone(); + assert!(cache.is_dir()); + assert!(cache + .file_name() + .unwrap() + .to_string_lossy() + .starts_with(&format!("sc-git-bucket-{}-", std::process::id()))); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + cache.metadata().unwrap().permissions().mode() & 0o777, + 0o700 + ); + } + drop(b); + assert!(!cache.exists()); + // a missing git binary is a clear error at open + let e = GitBucket::open_with(&url, "/no/such/git".into(), CHUNK).err(); + assert!(e.is_some_and(|e| e.to_string().contains("SC_GIT"))); + } + + #[test] + fn plain_local_path_urls_work() { + need_git!(); + let t = Tmp::new("path"); + let (hubdir, _) = hub(&t); + let b = open(hubdir.to_str().unwrap()); + contract_suite(&b); + } + + #[test] + fn value_cap_is_enforced() { + assert!(check_len(MAX_VALUE).is_ok()); + let e = check_len(MAX_VALUE + 1).unwrap_err().to_string(); + assert!(e.contains("cap"), "{e}"); + } + + #[cfg(unix)] + #[test] + fn cold_reads_cost_a_bounded_number_of_network_calls() { + need_git!(); + let mut measured = Vec::new(); + for n in [4usize, 24] { + let t = Tmp::new("count"); + let (_, url) = hub(&t); + let w = open(&url); + for i in 1..=n { + assert!(w.put_new(&format!("log/{i:020}"), b"entry").unwrap()); + assert!(w.put_new(&format!("packs/h{i}.idx"), b"idx").unwrap()); + } + let m = w.put_if_tag("manifest", b"m", None).unwrap().unwrap(); + let log = t.0.join("calls.log"); + let sh = shim( + &t, + "count.sh", + &format!( + "s=; k=0; for x in \"$@\"; do if [ $k = 1 ]; then k=0; continue; fi; if [ \"$x\" = -c ]; then k=1; continue; fi; s=$x; break; done; echo \"$s\" >> {}", + log.display() + ), + ); + let calls = || -> usize { + std::fs::read_to_string(&log) + .unwrap_or_default() + .lines() + .filter(|l| matches!(*l, "fetch" | "ls-remote" | "push")) + .count() + }; + let r = open_git(&url, &sh); + // cold read of the manifest, every log entry and every idx + assert!(matches!( + r.get("manifest", None).unwrap(), + Fetched::New { .. } + )); + for i in 1..=n { + assert!(matches!( + r.get(&format!("log/{i:020}"), None).unwrap(), + Fetched::New { .. } + )); + assert!(matches!( + r.get(&format!("packs/h{i}.idx"), None).unwrap(), + Fetched::New { .. } + )); + } + // probing the first absent log slot + assert!(matches!( + r.get(&format!("log/{:020}", n + 1), None).unwrap(), + Fetched::Absent + )); + let cold = calls(); + // a create-only key just written is served locally, a freshly + // written CAS tag is trusted inside the window: zero network calls + let before = calls(); + assert!(matches!( + w.get("manifest", Some(&m)).unwrap(), + Fetched::Unchanged + )); + assert_eq!(calls(), before); + // incremental: one new log entry + idx written elsewhere, then polled + assert!(w.put_new(&format!("log/{:020}", n + 1), b"new").unwrap()); + assert!(w.put_new("packs/hnew.idx", b"idx").unwrap()); + let before = calls(); + assert!(matches!( + r.get(&format!("log/{:020}", n + 1), None).unwrap(), + Fetched::New { .. } + )); + assert!(matches!( + r.get("packs/hnew.idx", None).unwrap(), + Fetched::New { .. } + )); + let incremental = calls() - before; + measured.push((n, cold, incremental)); + } + eprintln!("git network calls (n, cold, incremental): {measured:?}"); + assert_eq!(measured[0].1, measured[1].1, "cold cost independent of N"); + assert_eq!(measured[0].2, measured[1].2); + assert!(measured[1].1 <= 6, "{measured:?}"); + assert!(measured[1].2 <= 4, "{measured:?}"); + } +} diff --git a/crates/objio/src/lib.rs b/crates/objio/src/lib.rs index e9dfefcf..8d4ab35b 100644 --- a/crates/objio/src/lib.rs +++ b/crates/objio/src/lib.rs @@ -1,11 +1,15 @@ -//! Object-storage access for sc bucket remotes (P36). +//! Object-storage access for sc bucket remotes (P36, P39). //! //! Quarantine rule: object-store SDKs live here and only here — the rest of //! the workspace sees the [`Bucket`] trait. Leaf crate: depends on no other -//! workspace crate (like `tlsio`). +//! workspace crate (like `tlsio`). The git backend ([`GitBucket`], the +//! `sc+git::` carrier) spawns the system `git` binary (`SC_GIT` overrides it); +//! it does not use `gix`, which stays quarantined in `gitio`. mod dir; pub use dir::DirBucket; +mod git; +pub use git::GitBucket; mod s3; pub use s3::S3Bucket; From ee144360618b9b148c0f8fdebf76c3dd7719df99 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 11:28:15 +0200 Subject: [PATCH 03/15] fix(objio): keep family fetches shallow so a hostile host cannot send history A glob fetch of a key family now uses --depth=1 like exact fetches. A shallow glob that matches nothing exits 1, so a failed family fetch lists the family once: empty means absent, anything else is a real failure. --- crates/objio/src/git.rs | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/crates/objio/src/git.rs b/crates/objio/src/git.rs index 060b8ca6..a7865735 100644 --- a/crates/objio/src/git.rs +++ b/crates/objio/src/git.rs @@ -528,11 +528,10 @@ impl GitBucket { Ok(v) } - /// Fetch `refspecs`. `family` = a glob fetch of a whole key family: it - /// prunes stale local refs and omits `--depth=1`, because a shallow fetch - /// that matches no ref exits 1 with no error text, which cannot be told - /// from a failure. Exact-ref fetches stay shallow so a hostile commit's - /// parents are never downloaded. + /// Fetch `refspecs`, always shallow so a hostile commit's parents are never + /// downloaded. `family` = a glob fetch of a whole key family, which also + /// prunes stale local refs. A shallow glob fetch that matches no ref exits 1 + /// with no error text; [`Self::get_family`] tells that apart by listing. fn fetch(&self, refspecs: &[String], family: bool) -> Result { let mut a = vec![ "fetch", @@ -540,7 +539,10 @@ impl GitBucket { "--no-write-fetch-head", "--no-recurse-submodules", ]; - a.push(if family { "--prune" } else { "--depth=1" }); + a.push("--depth=1"); + if family { + a.push("--prune"); + } a.extend(["--end-of-options", &self.url]); a.extend(refspecs.iter().map(String::as_str)); self.run(&a, None, &[], SMALL_CAP) @@ -761,7 +763,17 @@ impl GitBucket { let pat = fam.glob(); let out = self.fetch(&[format!("+{pat}:{pat}")], true)?; if !out.success { - return Err(self.fail("fetch", &out)); + // A shallow glob fetch that matches nothing fails too: an empty + // family is absence, anything else is a real failure. + if self + .remote_list(fam.dir)? + .iter() + .any(|(k, _)| fam.matches(k)) + { + return Err(self.fail("fetch", &out)); + } + st.retain(|k, _| !fam.matches(k)); + return Ok(Fetched::Absent); } let local: Vec<_> = self .local_refs(&format!("{NS}{}", fam.dir))? From 85cf18d01811e50b9d37e058bf0599e8788adaa2 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 11:28:25 +0200 Subject: [PATCH 04/15] feat(repo,cli): sc+git:: bucket remotes with a per-remote private-branch opt-in Route `sc+git::` to the bucket transport (GitBucket stores the WAL as refs/sc/* in any Git repository). `sc+git:` is a reserved prefix: only the exact `sc+git::` form parses, and a near miss is refused instead of being saved or read as a local path or sent to the P18 Git bridge. - one `is_bucket_url` replaces the duplicated scheme checks - `remote add --git` and `clone --git` refuse sc+git:: - per-remote `allow_private_branches` in .sc/config (absent = false, omitted on write while false); `remote add`/`clone --allow-private-branches` set it - `Repo::push` refuses a private branch on a gated remote before any I/O, naming the flag and what becomes readable - `remote add`/`clone` print a one-line exposure notice - `remote compact` and `serve --store` refuse sc+git:: (deferred) - bucket errors are escaped for the terminal and no longer expose the raw inner error as an anyhow source - end-to-end tests over a bare Git repo are #[ignore]d until the real GitBucket lands --- crates/cli/src/main.rs | 102 +++- crates/cli/tests/git_bucket.rs | 511 ++++++++++++++++++ crates/repo/src/bucket_transport.rs | 97 +++- .../src/bucket_transport/git_remote_tests.rs | 201 +++++++ crates/repo/src/error.rs | 42 +- crates/repo/src/lib.rs | 3 +- crates/repo/src/remote.rs | 46 ++ crates/repo/src/stdio_transport.rs | 8 +- crates/repo/src/sync.rs | 32 ++ crates/repo/src/wire.rs | 3 +- 10 files changed, 1019 insertions(+), 26 deletions(-) create mode 100644 crates/cli/tests/git_bucket.rs create mode 100644 crates/repo/src/bucket_transport/git_remote_tests.rs diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index c18fe37e..4ae49328 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -296,8 +296,8 @@ enum Cmd { #[command(subcommand)] op: WsOp, }, - /// Clone a repo (local path, `ssh://`, `sc+http(s)://`, `sc+wal://`/`sc+s3://` - /// or Git URL) into a new directory. + /// Clone a repo (local path, `ssh://`, `sc+http(s)://`, `sc+wal://`/`sc+s3://`, + /// `sc+git::` or Git URL) into a new directory. Clone { src: String, dst: PathBuf, @@ -332,6 +332,12 @@ enum Cmd { /// Does nothing for git remotes, which carry no transcripts. #[arg(long)] no_transcripts: bool, + /// Let `origin` receive private branches (`sc+git::` only): pushing one + /// makes its name, recipient ids and sizes readable by everyone who + /// can read the Git repository, and the ciphertext stays there after + /// any later revoke. + #[arg(long)] + allow_private_branches: bool, }, /// Serve a repo to a remote `sc` client, either over stdin/stdout /// (invoked by `ssh` for ssh:// remotes) or over TCP (`sc+http://` @@ -407,8 +413,8 @@ enum Cmd { /// --tls-cert (enforced in run_serve, not by clap). #[arg(long)] tls_key: Option, - /// Serve a bucket WAL remote (`sc+wal://…` or `sc+s3://…`) instead of - /// this repo's own object store (P36c). `` remains the serve + /// Serve a bucket WAL remote (`sc+wal://…` or `sc+s3://…`; `sc+git::` + /// is not supported yet) instead of this repo's own object store (P36c). `` remains the serve /// home: its `.sc/` still provides access tokens, the TLS identity, /// and scratch space — but all served content lives in the bucket, /// making this instance disposable. @@ -886,12 +892,19 @@ enum RemoteOp { name: String, url: String, /// Treat the remote URL as a Git repository (translated via gitio). + /// Not for `sc+git::` URLs, which carry a native sc store. #[arg(long)] git: bool, + /// Let this remote receive private branches (`sc+git::` only): pushing + /// one makes its name, recipient ids and sizes readable by everyone + /// who can read the Git repository, and the ciphertext stays there + /// after any later revoke. + #[arg(long)] + allow_private_branches: bool, }, /// List configured remotes. List, - /// Compact a bucket remote (`sc+wal://`, `sc+s3://`): merge small packs, + /// Compact a bucket remote (`sc+wal://`, `sc+s3://`; not `sc+git::` yet): merge small packs, /// then delete log entries, checkpoints and packs no reader or writer can /// still need (ADR-0057). Needs bucket list and delete rights. Compact { @@ -1224,7 +1237,16 @@ fn real_main() -> Result<()> { filter, lazy, no_transcripts, - } => run_clone(src, dst, git, filter, lazy, no_transcripts), + allow_private_branches, + } => run_clone( + src, + dst, + git, + filter, + lazy, + no_transcripts, + allow_private_branches, + ), Cmd::Cat { target } => run_cat(&target), Cmd::Serve { sub, @@ -4012,7 +4034,9 @@ fn resolve_ids_to_pubkeys( /// so persist the absolute path. URLs (`ssh://`, `sc+http://`, ...) and paths /// that do not resolve are returned unchanged. fn absolute_local_source(src: String) -> String { - if src.contains("://") { + // Bucket URLs, including `sc+git::` (scp-form carriers have no `://`) and + // its near misses, are never paths. + if src.contains("://") || scl_repo::is_bucket_url(&src) { return src; } std::fs::canonicalize(&src) @@ -4020,6 +4044,10 @@ fn absolute_local_source(src: String) -> String { .unwrap_or(src) } +/// Printed once by `remote add` and `clone` for an `sc+git::` remote. +const GIT_BUCKET_NOTICE: &str = "note: sc+git:: stores public content and sealed metadata \ +(branch names, recipient ids, sizes) where everyone who can read the Git repository can read them"; + fn run_clone( src: String, dst: PathBuf, @@ -4027,12 +4055,26 @@ fn run_clone( filter: Vec, lazy: bool, no_transcripts: bool, + allow_private_branches: bool, ) -> Result<()> { + // `sc+git::` carries a native sc store inside a Git repository: it is a + // bucket remote, never the P18 mirror bridge, whatever its inner URL looks + // like. + let git_bucket = scl_repo::is_git_bucket_url(&src); + if git_bucket && git { + anyhow::bail!( + "--git cannot be used with {src}: sc+git:: carries a native sc store, not a Git history" + ); + } + if allow_private_branches && !git_bucket { + anyhow::bail!("--allow-private-branches applies only to sc+git:: URLs"); + } // Auto-detect unambiguous git URL forms (https/http, scp-style, file://): // those can never be sc-native, so no flag is needed. Bare `ssh://` is // ambiguous — it means an sc-native remote (ADR-0022) unless `--git` // forces the mirror-bridge path (ADR-0028; user-adjudicated). - let git_shaped = scl_gitio::bridge::is_network_git_url(&src) && !src.starts_with("ssh://"); + let git_shaped = + !git_bucket && scl_gitio::bridge::is_network_git_url(&src) && !src.starts_with("ssh://"); if git || git_shaped { if !filter.is_empty() { anyhow::bail!("partial clone is not supported over git remotes"); @@ -4046,6 +4088,12 @@ fn run_clone( if lazy { scl_repo::promisor::set_lazy(repo.layout(), true)?; } + if allow_private_branches { + repo.remote_allow_private_branches("origin")?; + } + if git_bucket { + eprintln!("{GIT_BUCKET_NOTICE}"); + } let n = repo.branches()?.len(); if let Some(promisor) = repo.promisor()? { println!( @@ -4207,6 +4255,7 @@ fn run_serve( }; scl_repo::wire::validate_max_conn_bytes(conn_bytes, max_pack, scl_core::max_object_size())?; if let Some(url) = &store { + scl_repo::refuse_git_store(url)?; scl_repo::BucketUrl::parse(url)?; // fail fast on malformed URLs, before any bind } match (stdio, http) { @@ -4419,7 +4468,21 @@ fn git_remote_effective_path( fn run_remote(op: RemoteOp) -> Result<()> { let repo = open_repo()?; match op { - RemoteOp::Add { name, url, git } => { + RemoteOp::Add { + name, + url, + git, + allow_private_branches, + } => { + let git_bucket = scl_repo::is_git_bucket_url(&url); + if git_bucket && git { + anyhow::bail!( + "--git cannot be used with {url}: sc+git:: carries a native sc store, not a Git history" + ); + } + if allow_private_branches && !git_bucket { + anyhow::bail!("--allow-private-branches applies only to sc+git:: URLs"); + } if git { if !scl_gitio::bridge::is_network_git_url(&url) && !std::path::Path::new(&url).join("HEAD").exists() @@ -4442,11 +4505,14 @@ fn run_remote(op: RemoteOp) -> Result<()> { if url.starts_with("sc+http:") || url.starts_with("sc+https:") { scl_repo::ScHttpUrl::parse(&url)?; // fail fast on malformed URLs } - if url.starts_with("sc+wal://") || url.starts_with("sc+s3://") { + if scl_repo::is_bucket_url(&url) { scl_repo::BucketUrl::parse(&url)?; // fail fast on malformed URLs } - repo.remote_add(&name, &url)?; + repo.remote_add_opts(&name, &url, allow_private_branches)?; println!("added remote {name} -> {url}"); + if git_bucket { + eprintln!("{GIT_BUCKET_NOTICE}"); + } } } RemoteOp::Compact { @@ -4468,7 +4534,8 @@ fn run_remote(op: RemoteOp) -> Result<()> { .find(|(n, _)| *n == name) .map(|(_, u)| u) .ok_or_else(|| anyhow::anyhow!("no such remote: {name}"))?; - if !(url.starts_with("sc+wal://") || url.starts_with("sc+s3://")) { + scl_repo::refuse_git_compact(&url)?; + if !scl_repo::is_bucket_url(&url) { anyhow::bail!("remote '{name}' is not a bucket remote ({url}); nothing to compact"); } let transport = scl_repo::BucketTransport::open(&url)?; @@ -5480,6 +5547,7 @@ mod tests { Vec::new(), false, false, + false, ) .unwrap(); let repo = scl_repo::Repo::open(&dst).unwrap(); @@ -5502,7 +5570,14 @@ mod tests { // once the user cd's into it, so the recorded origin must be absolute. let abs = absolute_local_source(".".to_string()); assert!(std::path::Path::new(&abs).is_absolute(), "got {abs}"); - for url in ["ssh://host/srv/repo", "sc+http://127.0.0.1:1/x"] { + // `sc+git::` carriers (scp-form has no `://`) and near misses are never paths. + for url in [ + "ssh://host/srv/repo", + "sc+http://127.0.0.1:1/x", + "sc+git::git@h:o/r.git", + "sc+git::/abs/hub.git", + "sc+git:/x", + ] { assert_eq!(absolute_local_source(url.to_string()), url); } assert_eq!( @@ -5531,6 +5606,7 @@ mod tests { Vec::new(), false, false, + false, ) .unwrap_err(); std::env::remove_var("SC_SSH"); diff --git a/crates/cli/tests/git_bucket.rs b/crates/cli/tests/git_bucket.rs new file mode 100644 index 00000000..5afe014a --- /dev/null +++ b/crates/cli/tests/git_bucket.rs @@ -0,0 +1,511 @@ +//! `sc+git::` bucket remotes through the real binary: URL routing, +//! the private-branch opt-in, the deferred-feature refusals, and one end-to-end +//! scenario (private branch + protected path + secret over a Git carrier). +//! +//! The scenario runs twice from one body: over `sc+wal://` as a control that +//! proves the script itself (always on), and over `sc+git::file://` +//! (`#[ignore = "needs real GitBucket"]` until the objio implementation lands). + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +fn tmp(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("scl-cli-gitbucket-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +/// Run `sc` in `dir` with a private `HOME` (so no real `~/.sc/identity` is +/// picked up) and, optionally, `SC_IDENTITY`. +fn sc_as(dir: &Path, identity: Option<&Path>, args: &[&str]) -> Output { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_sc")); + cmd.args(args) + .current_dir(dir) + .env("HOME", dir) + .env("SC_AUTHOR", "tester") + .env_remove("SC_IDENTITY"); + if let Some(id) = identity { + cmd.env("SC_IDENTITY", id); + } + cmd.output().expect("sc runs") +} + +fn sc(dir: &Path, args: &[&str]) -> Output { + sc_as(dir, None, args) +} + +fn ok(out: Output) -> String { + assert!( + out.status.success(), + "stdout: {}\nstderr: {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).into_owned() +} + +fn err_text(out: &Output) -> String { + assert!(!out.status.success(), "expected failure: {out:?}"); + String::from_utf8_lossy(&out.stderr).into_owned() +} + +fn keygen(dir: &Path, name: &str) -> (PathBuf, String) { + let idfile = dir.join(format!("{name}.id")); + let out = ok(sc(dir, &["keygen", "--out", idfile.to_str().unwrap()])); + let pk = out + .lines() + .find(|l| l.contains("public key")) + .and_then(|l| l.split_whitespace().find(|w| w.starts_with("scl-pk-"))) + .expect("public key in keygen output") + .to_string(); + (idfile, pk) +} + +fn init_repo(dir: &Path) { + std::fs::create_dir_all(dir).unwrap(); + ok(sc(dir, &["init"])); +} + +// ---- always-on: URL routing and refusals (no GitBucket I/O) ---------------- + +/// `sc+git:` is a reserved prefix. A near miss is refused by `remote add` and +/// `clone`, never saved and never read as a local path or a Git URL. +#[test] +fn sc_git_near_misses_are_refused_and_never_saved_or_cloned() { + let a = tmp("nearmiss"); + init_repo(&a); + for url in [ + "sc+git:/x", + "sc+git://x", + "sc+git:x", + "sc+git:git@h:o/r.git", + ] { + let out = sc(&a, &["remote", "add", "x", url]); + let e = err_text(&out); + assert!(e.contains("sc+git::"), "{url}: {e}"); + } + let listed = ok(sc(&a, &["remote", "list"])); + assert_eq!(listed.trim(), "", "nothing was saved"); + + // clone: refused before the destination is created + let dst = a.join("dst"); + let out = sc(&a, &["clone", "sc+git:/x", dst.to_str().unwrap()]); + let e = err_text(&out); + assert!(e.contains("sc+git::"), "{e}"); + assert!(!dst.exists()); + std::fs::remove_dir_all(&a).unwrap(); +} + +#[test] +fn sc_git_is_never_a_git_bridge_remote_and_the_opt_in_is_sc_git_only() { + let a = tmp("flags"); + init_repo(&a); + let carrier = "sc+git::file:///nonexistent/carrier.git"; + let e = err_text(&sc(&a, &["remote", "add", "x", carrier, "--git"])); + assert!(e.contains("--git cannot be used"), "{e}"); + let e = err_text(&sc( + &a, + &[ + "remote", + "add", + "x", + "https://example.com/r.git", + "--allow-private-branches", + ], + )); + assert!(e.contains("only to sc+git::"), "{e}"); + let dst = a.join("dst"); + let e = err_text(&sc(&a, &["clone", "--git", carrier, dst.to_str().unwrap()])); + assert!(e.contains("--git cannot be used"), "{e}"); + let e = err_text(&sc( + &a, + &[ + "clone", + "/some/path", + dst.to_str().unwrap(), + "--allow-private-branches", + ], + )); + assert!(e.contains("only to sc+git::"), "{e}"); + assert!(!dst.exists()); + assert_eq!(ok(sc(&a, &["remote", "list"])).trim(), ""); + std::fs::remove_dir_all(&a).unwrap(); +} + +/// Deferred in v1: `sc remote compact` and `sc serve --store` refuse an +/// `sc+git::` store with a clear message, before any I/O. +#[test] +fn compact_and_serve_store_refuse_sc_git() { + let a = tmp("deferred"); + init_repo(&a); + // Written by hand so the test does not depend on GitBucket validation. + std::fs::write( + a.join(".sc/config"), + "[remote.origin]\nurl = \"sc+git::file:///nonexistent/carrier.git\"\n", + ) + .unwrap(); + let e = err_text(&sc(&a, &["remote", "compact", "origin", "--dry-run"])); + assert!(e.contains("does not support sc+git::"), "{e}"); + + let home = tmp("deferred-home"); + init_repo(&home); + let e = err_text(&sc( + &a, + &[ + "serve", + "--stdio", + "--store", + "sc+git::file:///nonexistent/carrier.git", + home.to_str().unwrap(), + ], + )); + assert!(e.contains("does not support sc+git::"), "{e}"); + for d in [&a, &home] { + std::fs::remove_dir_all(d).unwrap(); + } +} + +/// The private-branch gate through the binary, with a config written by hand +/// (an old-style entry with no `allow_private_branches` key parses as false). +/// The carrier does not exist: refusing proves it happens before any upload. +#[test] +fn pushing_a_private_branch_to_a_git_carrier_needs_the_opt_in() { + let a = tmp("gate"); + let keys = tmp("gate-keys"); + init_repo(&a); + let (id, pk) = keygen(&keys, "alice"); + std::fs::write( + a.join(".sc/recipients.toml"), + format!("[recipients]\nalice = \"{pk}\"\n"), + ) + .unwrap(); + std::fs::write(a.join("f.txt"), "f\n").unwrap(); + ok(sc(&a, &["commit", "-m", "base"])); + let idp = id.to_str().unwrap(); + ok(sc( + &a, + &[ + "branch", + "hotfix", + "--private", + "--to", + "alice", + "--identity", + idp, + ], + )); + ok(sc(&a, &["switch", "hotfix", "--identity", idp])); + std::fs::write(a.join("embargo.txt"), "secret fix\n").unwrap(); + ok(sc(&a, &["commit", "-m", "embargoed", "--identity", idp])); + + let carrier = a.join("never-created.git"); + std::fs::write( + a.join(".sc/config"), + format!( + "[remote.origin]\nurl = \"sc+git::file://{}\"\n", + carrier.display() + ), + ) + .unwrap(); + let e = err_text(&sc(&a, &["push", "origin"])); + for needle in [ + "--allow-private-branches", + "readable by everyone who can read the Git repository", + "after any later revoke", + ] { + assert!(e.contains(needle), "missing {needle:?}: {e}"); + } + assert!(!carrier.exists(), "nothing was uploaded"); + for d in [&a, &keys] { + std::fs::remove_dir_all(d).unwrap(); + } +} + +// ---- end-to-end scenario ---------------------------------------------------- + +struct Fixture { + root: PathBuf, + keys: PathBuf, + id: PathBuf, + /// The carrier (`sc+wal://` directory or bare Git repo) and its sc URL. + carrier: PathBuf, + url: String, + a: PathBuf, +} + +fn git_carrier(tag: &str) -> Fixture { + let mut fx = fixture(tag, "git"); + let st = Command::new("git") + .args(["init", "--bare", "-q", fx.carrier.to_str().unwrap()]) + .status() + .unwrap(); + assert!(st.success()); + fx.url = format!("sc+git::file://{}", fx.carrier.display()); + fx +} + +fn fixture(tag: &str, kind: &str) -> Fixture { + let root = tmp(&format!("e2e-{kind}-{tag}")); + let keys = tmp(&format!("e2e-{kind}-{tag}-keys")); + let (id, pk) = keygen(&keys, "alice"); + let carrier = root.join("carrier"); + let a = root.join("a"); + init_repo(&a); + std::fs::write( + a.join(".sc/recipients.toml"), + format!("[recipients]\nalice = \"{pk}\"\n"), + ) + .unwrap(); + let idp = id.to_str().unwrap(); + ok(sc(&a, &["protect", "config/", "--to", "alice"])); + std::fs::create_dir_all(a.join("config")).unwrap(); + std::fs::write(a.join("config/prod.env"), "DB=prod-secret-marker\n").unwrap(); + std::fs::write(a.join("README.md"), "public readme\n").unwrap(); + ok(sc( + &a, + &[ + "secret", + "add", + "API_TOKEN", + "--to", + "alice", + "--value", + "s3cret", + ], + )); + ok(sc_as(&a, Some(&id), &["commit", "-m", "main base"])); + ok(sc( + &a, + &[ + "branch", + "hotfix", + "--private", + "--to", + "alice", + "--identity", + idp, + ], + )); + ok(sc(&a, &["switch", "hotfix", "--identity", idp])); + std::fs::write(a.join("embargo.txt"), "embargoed fix\n").unwrap(); + ok(sc( + &a, + &["commit", "-m", "embargoed message", "--identity", idp], + )); + ok(sc(&a, &["switch", "main", "--identity", idp])); + Fixture { + url: format!("sc+wal://{}", carrier.display()), + root, + keys, + id, + carrier, + a, + } +} + +fn clone_into( + fx: &Fixture, + name: &str, + identity: Option<&Path>, + extra: &[&str], +) -> (PathBuf, Output) { + let dst = fx.root.join(name); + let mut args = vec!["clone", fx.url.as_str(), dst.to_str().unwrap()]; + args.extend_from_slice(extra); + let out = sc_as(&fx.root, identity, &args); + (dst, out) +} + +/// The shared body. `gated` is true for `sc+git::`, where a private branch +/// needs the per-remote opt-in. +fn scenario(fx: Fixture, gated: bool) { + let idp = fx.id.clone(); + let idp = idp.to_str().unwrap(); + let a = &fx.a; + + let notice = |out: &Output| { + String::from_utf8_lossy(&out.stderr).contains("everyone who can read the Git repository") + }; + + let out = sc(a, &["remote", "add", "origin", &fx.url]); + assert_eq!(notice(&out), gated, "one notice for sc+git:: only: {out:?}"); + ok(out); + ok(sc(a, &["push", "origin"])); // public main + + ok(sc(a, &["switch", "hotfix", "--identity", idp])); + if gated { + let before = refs_on_carrier(&fx.carrier); + assert!(!before.is_empty(), "main's push stored refs/sc/*"); + let e = err_text(&sc(a, &["push", "origin"])); + assert!(e.contains("--allow-private-branches"), "{e}"); + assert_eq!( + refs_on_carrier(&fx.carrier), + before, + "refusal uploaded nothing" + ); + let out = sc( + a, + &[ + "remote", + "add", + "opted", + &fx.url, + "--allow-private-branches", + ], + ); + ok(out); + ok(sc(a, &["push", "opted"])); + let cfg = std::fs::read_to_string(a.join(".sc/config")).unwrap(); + assert!(cfg.contains("allow_private_branches = true"), "{cfg}"); + } else { + ok(sc(a, &["push", "origin"])); + } + ok(sc(a, &["switch", "main", "--identity", idp])); + + // Keyless clone: public content and the branch marker only. + let (keyless, out) = clone_into(&fx, "keyless", None, &[]); + assert_eq!(notice(&out), gated); + ok(out); + let listing = ok(sc(&keyless, &["branch", "list"])); + assert!(listing.contains("hotfix (private, no access)"), "{listing}"); + assert_eq!( + std::fs::read_to_string(keyless.join("README.md")).unwrap(), + "public readme\n" + ); + assert!( + !keyless.join("config/prod.env").exists(), + "protected path is absent" + ); + assert!(!keyless.join("embargo.txt").exists()); + let cfg = std::fs::read_to_string(keyless.join(".sc/config")).unwrap(); + assert!(!cfg.contains("allow_private_branches"), "{cfg}"); + assert!(ok(sc(&keyless, &["secret", "list"])).contains("API_TOKEN")); + + // Keyed clone: everything reads. + let extra: &[&str] = if gated { + &["--allow-private-branches"] + } else { + &[] + }; + let (keyed, out) = clone_into(&fx, "keyed", Some(&fx.id), extra); + ok(out); + // Clone materializes HEAD without a key; a keyed switch lays the protected file out. + ok(sc_as( + &keyed, + Some(&fx.id), + &["switch", "main", "--identity", idp], + )); + assert_eq!( + std::fs::read_to_string(keyed.join("config/prod.env")).unwrap(), + "DB=prod-secret-marker\n" + ); + ok(sc_as( + &keyed, + Some(&fx.id), + &[ + "run", + "--identity", + idp, + "--", + "sh", + "-c", + "test \"$API_TOKEN\" = s3cret", + ], + )); + let cfg = std::fs::read_to_string(keyed.join(".sc/config")).unwrap(); + assert_eq!( + cfg.contains("allow_private_branches = true"), + gated, + "{cfg}" + ); + ok(sc_as( + &keyed, + Some(&fx.id), + &["switch", "hotfix", "--identity", idp], + )); + assert_eq!( + std::fs::read_to_string(keyed.join("embargo.txt")).unwrap(), + "embargoed fix\n" + ); + let log = ok(sc_as(&keyed, Some(&fx.id), &["log", "--identity", idp])); + assert!(log.contains("embargoed message"), "{log}"); + + // A partial clone of a bucket remote is refused, as for sc+wal://. + let (partial, out) = clone_into(&fx, "partial", None, &["--filter", "config/"]); + let e = err_text(&out); + assert!(!e.is_empty() && !partial.exists(), "{e}"); + + std::fs::remove_dir_all(&fx.root).unwrap(); + std::fs::remove_dir_all(&fx.keys).unwrap(); + assert!(!fx.root.exists()); +} + +fn refs_on_carrier(carrier: &Path) -> String { + let out = Command::new("git") + .args(["-C", carrier.to_str().unwrap(), "for-each-ref", "refs/sc/"]) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).into_owned() +} + +/// Control: the same script over the existing `sc+wal://` backend. +#[test] +fn scenario_over_a_wal_bucket_is_the_control() { + scenario(fixture("control", "wal"), false); +} + +#[test] +#[ignore = "needs real GitBucket"] +fn private_branch_protected_path_and_secret_round_trip_over_a_git_carrier() { + scenario(git_carrier("roundtrip"), true); +} + +/// Carrier URL forms through the CLI: file://, scp-form, ssh://, https://. No +/// network is touched: `remote add` validates the URL and writes the config. +#[test] +#[ignore = "needs real GitBucket"] +fn remote_add_accepts_scp_form_and_file_carrier_urls_and_notices_once() { + let a = tmp("forms"); + init_repo(&a); + for (i, inner) in [ + "file:///srv/hub.git", + "git@github.com:org/repo.git", + "ssh://git@host/org/repo.git", + "https://example.com/org/repo.git", + "/abs/path/hub.git", + ] + .iter() + .enumerate() + { + let url = format!("sc+git::{inner}"); + let out = sc(&a, &["remote", "add", &format!("r{i}"), &url]); + let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); + ok(out); + assert_eq!( + stderr + .matches("everyone who can read the Git repository") + .count(), + 1, + "{url}: {stderr}" + ); + } + let listed = ok(sc(&a, &["remote", "list"])); + assert!( + listed.contains("sc+git::git@github.com:org/repo.git\t[sc]"), + "{listed}" + ); + for bad in [ + "relative/path", + "-oProxyCommand=x", + "https://u:pw@h/r.git", + "", + ] { + let url = format!("sc+git::{bad}"); + assert!( + !sc(&a, &["remote", "add", "bad", &url]).status.success(), + "{url}" + ); + } + std::fs::remove_dir_all(&a).unwrap(); +} diff --git a/crates/repo/src/bucket_transport.rs b/crates/repo/src/bucket_transport.rs index c5b27b5c..c0cfb556 100644 --- a/crates/repo/src/bucket_transport.rs +++ b/crates/repo/src/bucket_transport.rs @@ -15,6 +15,8 @@ use std::collections::BTreeMap; use std::io::Write; mod compact; +#[cfg(test)] +mod git_remote_tests; pub use compact::{CompactOptions, CompactReport}; /// How many times a read restarts from a fresh manifest after an object the @@ -158,13 +160,78 @@ pub enum BucketScheme { Wal, /// `sc+s3://` — an S3-compatible object store. S3, + /// `sc+git::` — `refs/sc/*` inside any Git repository. + Git, +} + +/// Whether `url` belongs to the bucket transport: `sc+wal://`, `sc+s3://`, or +/// anything under the reserved `sc+git:` prefix. The prefix is reserved on +/// purpose: a near miss such as `sc+git:/x` is routed here, where +/// [`BucketUrl::parse`] refuses it, instead of falling through to a local path +/// or the Git mirror bridge. +pub fn is_bucket_url(url: &str) -> bool { + url.starts_with("sc+wal://") || url.starts_with("sc+s3://") || is_git_bucket_url(url) +} + +/// Whether `url` is under the reserved `sc+git:` prefix (valid or not). +pub fn is_git_bucket_url(url: &str) -> bool { + url.starts_with("sc+git:") } -/// A parsed `sc+wal://` or `sc+s3:///` remote URL. +/// The private-branch gate for `sc+git::` remotes (ADR-0044 reason 4, reversed +/// only under this opt-in): a private branch published to a Git host exposes +/// its name, recipient ids and sizes to every reader of the repository, and the +/// ciphertext stays there for good. Pure, so it runs before any bucket I/O. +/// Remotes of any other kind are unaffected. +pub fn check_private_push_allowed(url: &str, allowed: bool, branch: &str) -> Result<()> { + if allowed || !is_git_bucket_url(url) { + return Ok(()); + } + Err(Error::InvalidArgument(format!( + "branch {branch} is private and this remote is a Git repository ({url}): pushing it \ + makes the branch name, recipient ids and sizes readable by everyone who can read the \ + Git repository, and the ciphertext stays there after any later revoke. Nothing was \ + uploaded. To accept that, run `sc remote add --allow-private-branches` \ + (or `sc clone --allow-private-branches`) for this remote" + ))) +} + +/// `sc serve --store sc+git::…` is refused (deferred): a daemon cannot answer +/// ssh or credential prompts, and it would sit between clients and the +/// client-side private-branch gate. Checked on the prefix, before any parse. +pub fn refuse_git_store(url: &str) -> Result<()> { + if is_git_bucket_url(url) { + return Err(Error::InvalidArgument( + "sc serve --store does not support sc+git:: remotes yet (deferred); \ + serve an sc+wal:// or sc+s3:// store, or push to the Git repository with `sc push`" + .into(), + )); + } + Ok(()) +} + +/// `sc remote compact` on an `sc+git::` remote is refused (deferred): the +/// compactor has no trustworthy age source for refs in a Git repository, so +/// it could not tell a stale pack from one an in-flight push just uploaded. +pub fn refuse_git_compact(url: &str) -> Result<()> { + if is_git_bucket_url(url) { + return Err(Error::InvalidArgument( + "sc remote compact does not support sc+git:: remotes yet (deferred): a Git \ + repository gives the compactor no trustworthy age for stored packs" + .into(), + )); + } + Ok(()) +} + +/// A parsed `sc+wal://`, `sc+s3:///` or +/// `sc+git::` remote URL. /// `sc+wal` treats everything after the scheme as a directory path (so /// `sc+wal:///abs/path` and `sc+wal://rel/path` both work); `sc+s3` splits /// the first path component off as the bucket name and keeps the remainder -/// (possibly empty) as the key prefix. +/// (possibly empty) as the key prefix; `sc+git` keeps everything after `::` +/// verbatim as the Git URL in `bucket` (the objio backend validates it). +#[derive(Debug)] pub struct BucketUrl { pub scheme: BucketScheme, pub bucket: String, @@ -177,16 +244,33 @@ impl BucketUrl { /// embedded CR/LF, which would otherwise smuggle extra "lines" into /// anything that later logs or shells out with the raw URL. pub fn parse(url: &str) -> Result { + if url.chars().any(|c| c == '\r' || c == '\n') { + return Err(Error::InvalidArgument(format!("bad bucket url: {url}"))); + } + if let Some(r) = url.strip_prefix("sc+git:") { + let inner = r.strip_prefix(':').ok_or_else(|| { + Error::InvalidArgument(format!( + "bad bucket url: {url} (the form is sc+git::, with two colons)" + )) + })?; + scl_objio::GitBucket::validate_url(inner) + .map_err(|e| Error::InvalidArgument(format!("bad bucket url: {url}: {e}")))?; + return Ok(BucketUrl { + scheme: BucketScheme::Git, + bucket: inner.to_string(), + prefix: String::new(), + }); + } let (scheme, rest) = if let Some(r) = url.strip_prefix("sc+wal://") { (BucketScheme::Wal, r) } else if let Some(r) = url.strip_prefix("sc+s3://") { (BucketScheme::S3, r) } else { return Err(Error::InvalidArgument(format!( - "not an sc+wal:// or sc+s3:// url: {url}" + "not an sc+wal://, sc+s3:// or sc+git:: url: {url}" ))); }; - if rest.is_empty() || rest.chars().any(|c| c == '\r' || c == '\n') { + if rest.is_empty() { return Err(Error::InvalidArgument(format!("bad bucket url: {url}"))); } Ok(match scheme { @@ -206,6 +290,7 @@ impl BucketUrl { prefix: prefix.trim_matches('/').to_string(), } } + BucketScheme::Git => unreachable!("handled above"), }) } } @@ -213,7 +298,8 @@ impl BucketUrl { impl BucketTransport { /// Open the right bucket backend for a bucket URL (`sc+wal://` opens a /// local-directory test/demo backend, `sc+s3://` opens the real - /// S3-compatible backend), then wrap it as a `Transport` the same way + /// S3-compatible backend, `sc+git::` stores `refs/sc/*` in a Git + /// repository), then wrap it as a `Transport` the same way /// [`BucketTransport::from_bucket`] does. pub fn open(url: &str) -> Result { let parsed = BucketUrl::parse(url)?; @@ -222,6 +308,7 @@ impl BucketTransport { BucketScheme::S3 => { Box::new(scl_objio::S3Bucket::open(&parsed.bucket, &parsed.prefix)?) } + BucketScheme::Git => Box::new(scl_objio::GitBucket::open(&parsed.bucket)?), }; BucketTransport::from_bucket(bucket) } diff --git a/crates/repo/src/bucket_transport/git_remote_tests.rs b/crates/repo/src/bucket_transport/git_remote_tests.rs new file mode 100644 index 00000000..914eb063 --- /dev/null +++ b/crates/repo/src/bucket_transport/git_remote_tests.rs @@ -0,0 +1,201 @@ +//! `sc+git::` remote plumbing: URL parsing and the reserved prefix, the +//! private-branch gate, and the refusals (`serve --store`, `remote compact`). +//! Nothing here reaches `GitBucket`'s real I/O except the test marked +//! `#[ignore = "needs real GitBucket"]`, which needs the objio implementation. + +use super::*; +use crate::error::Error; +use crate::repo::Repo; + +fn root(tag: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("scl-bt-gitremote-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +fn invalid(r: Result) -> String { + match r { + Err(Error::InvalidArgument(m)) => m, + other => panic!("expected InvalidArgument, got {other:?}"), + } +} + +/// `sc+git:` is a reserved prefix: only `sc+git::` is accepted, and a +/// near miss is refused before the inner validator runs. +#[test] +fn sc_git_near_misses_are_refused_at_parse() { + for url in [ + "sc+git:", + "sc+git:/x", + "sc+git:x", + "sc+git://x", + "sc+git:file:///x", + "sc+git::file:///x\n", + "sc+git::file:///x\r", + ] { + let msg = invalid(BucketUrl::parse(url)); + assert!(msg.contains("bad bucket url"), "{url}: {msg}"); + } + // Not under the prefix at all: the generic refusal names all three schemes. + assert!(invalid(BucketUrl::parse("sc+gitx::y")).contains("sc+git::")); +} + +#[test] +fn is_bucket_url_recognises_every_bucket_scheme_and_the_reserved_prefix() { + for url in [ + "sc+wal:///tmp/b", + "sc+s3://bucket/p", + "sc+git::file:///srv/r.git", + "sc+git::git@github.com:org/repo.git", + // reserved prefix: routed to the bucket transport so parse refuses it + "sc+git:/x", + "sc+git://x", + ] { + assert!(is_bucket_url(url), "{url}"); + } + for url in [ + "", + "/srv/repo", + "ssh://host/repo", + "sc+http://h/r", + "sc+https://h/r", + "https://github.com/o/r.git", + "git@github.com:o/r.git", + "sc+wal:/x", + "sc+gitx::y", + ] { + assert!(!is_bucket_url(url), "{url}"); + } + assert!(is_git_bucket_url("sc+git:/x") && !is_git_bucket_url("sc+wal:///x")); +} + +/// A near miss must not be opened as a local `.sc/` path by `open_transport`. +#[test] +fn a_near_miss_never_opens_as_a_local_path() { + let d = root("nearmiss"); + // A directory that a path reading of the URL would find. + std::fs::create_dir_all(d.join(".sc")).unwrap(); + let url = format!("sc+git:{}", d.display()); + let Err(err) = crate::stdio_transport::open_transport(&url) else { + panic!("a near miss is refused"); + }; + assert!(matches!(err, Error::InvalidArgument(_)), "{err}"); + std::fs::remove_dir_all(&d).unwrap(); +} + +#[test] +fn the_private_branch_gate_names_the_flag_and_the_exposure() { + let url = "sc+git::git@github.com:org/repo.git"; + let msg = invalid(check_private_push_allowed(url, false, "hotfix")); + for needle in [ + "--allow-private-branches", + "hotfix", + "branch name, recipient ids and sizes", + "readable by everyone who can read the Git repository", + "after any later revoke", + "Nothing was uploaded", + ] { + assert!(msg.contains(needle), "missing {needle:?}: {msg}"); + } + assert!(check_private_push_allowed(url, true, "hotfix").is_ok()); + // Other remote kinds are unchanged. + for other in ["sc+wal:///tmp/b", "sc+s3://b/p", "ssh://h/r", "/srv/repo"] { + assert!(check_private_push_allowed(other, false, "hotfix").is_ok()); + } +} + +/// The gate runs in `Repo::push` before any bucket I/O: the carrier here does +/// not exist, and a push that got past the gate would try to open it. +#[test] +fn push_of_a_private_branch_to_a_git_carrier_refuses_before_any_io() { + let d = root("gate-push"); + let repo = Repo::init(&d).unwrap(); + std::fs::write(d.join("a.txt"), "a\n").unwrap(); + repo.commit("me", "base").unwrap(); + let (sk, pk) = scl_crypto::generate_keypair(); + repo.branch_private("emb", &sk, std::slice::from_ref(&pk), &[]) + .unwrap(); + repo.switch_with_identity("emb", Some(&sk)).unwrap(); + assert!(repo.head_private().unwrap().is_some()); + let url = format!("sc+git::file://{}/never-created.git", d.display()); + repo.remote_add("origin", &url).unwrap(); + + let msg = invalid(repo.push("origin")); + assert!(msg.contains("--allow-private-branches"), "{msg}"); + assert!(!d.join("never-created.git").exists()); + + // The opt-in is per remote and persisted: a remote without it is still + // gated, and the flag survives a config reload. + repo.remote_add_opts("opted", &url, true).unwrap(); + repo.remote_add("plain", &url).unwrap(); + let cfg = crate::remote::RemoteConfig::load(repo.layout()).unwrap(); + assert!(cfg.allow_private_branches("opted")); + assert!(!cfg.allow_private_branches("plain")); + assert!(invalid(repo.push("plain")).contains("--allow-private-branches")); + drop(repo); + std::fs::remove_dir_all(&d).unwrap(); +} + +#[test] +fn serve_store_and_compact_refuse_git_carriers_before_parsing() { + let url = "sc+git::file:///srv/r.git"; + assert!(invalid(refuse_git_store(url)).contains("sc serve --store")); + assert!(invalid(refuse_git_compact(url)).contains("remote compact")); + // The reserved prefix is refused too, so a near miss never reaches a parser. + assert!(refuse_git_store("sc+git:/x").is_err()); + assert!(refuse_git_store("sc+wal:///tmp/b").is_ok()); + assert!(refuse_git_compact("sc+s3://b/p").is_ok()); + + // The wire layer enforces it for both `--stdio` and `--http` serving, and + // the client sees why at the handshake. + let home = root("serve-store"); + std::fs::create_dir_all(home.join(".sc")).unwrap(); + let (client_read, mut server_write) = std::io::pipe().unwrap(); + let (mut server_read, client_write) = std::io::pipe().unwrap(); + let home_for_server = home.clone(); + let srv = std::thread::spawn(move || { + crate::wire::serve_bucket_with_policy( + url, + &home_for_server, + &mut server_read, + &mut server_write, + crate::wire::WirePolicy::default(), + ) + }); + let Err(err) = crate::stdio_transport::WireClient::handshake(client_read, client_write) else { + panic!("the session is refused"); + }; + assert!(err.to_string().contains("sc serve --store"), "{err}"); + srv.join().unwrap().unwrap(); + std::fs::remove_dir_all(&home).unwrap(); +} + +#[test] +#[ignore = "needs real GitBucket"] +fn sc_git_urls_parse_with_the_inner_url_verbatim() { + for inner in [ + "file:///srv/r.git", + "git@github.com:org/repo.git", + "https://example.com/org/repo.git", + "ssh://git@host/org/repo.git", + "/abs/local/repo.git", + ] { + let u = BucketUrl::parse(&format!("sc+git::{inner}")).unwrap(); + assert_eq!(u.scheme, BucketScheme::Git); + assert_eq!(u.bucket, inner); + assert_eq!(u.prefix, ""); + } + // The backend's own validator decides what an inner URL may be. + for inner in [ + "", + "-oProxyCommand=x", + "relative/path", + "https://u:pw@h/r.git", + ] { + assert!( + BucketUrl::parse(&format!("sc+git::{inner}")).is_err(), + "{inner}" + ); + } +} diff --git a/crates/repo/src/error.rs b/crates/repo/src/error.rs index 3bdaf757..754c5278 100644 --- a/crates/repo/src/error.rs +++ b/crates/repo/src/error.rs @@ -205,10 +205,20 @@ pub enum Error { /// P36a: the bucket WAL is untrusted input; decode/consistency failures /// are their own variant so callers can distinguish "bucket corrupt or /// newer-format" from transport errors. - #[error("bucket wal: {0}")] + #[error("bucket wal: {}", scl_core::sanitize_display_line(.0))] Wal(String), - #[error("bucket: {0}")] - ObjIo(#[from] scl_objio::Error), + /// Bucket backend errors carry text from the store or the Git host (a + /// `sc+git::` hook message, a server's reply), so the display is escaped + /// and the inner error is deliberately not a `source`, which anyhow would + /// print raw under "Caused by". + #[error("bucket: {}", scl_core::sanitize_display_line(&.0.to_string()))] + ObjIo(scl_objio::Error), +} + +impl From for Error { + fn from(e: scl_objio::Error) -> Self { + Error::ObjIo(e) + } } pub type Result = std::result::Result; @@ -259,3 +269,29 @@ mod tests { assert!(matches!(Error::Remote(raw.into()), Error::Remote(m) if m == raw)); } } + +#[cfg(test)] +mod bucket_error_display_tests { + use super::*; + + /// Text a Git host or bucket controls (a hook message, an error body) must + /// not reach the terminal with its escape bytes, in the message or in an + /// anyhow "Caused by" chain (no `source`, so there is none). + #[test] + fn bucket_errors_escape_control_bytes_and_have_no_raw_source() { + let evil = "\x1b[2J\x1b]0;pwned\x07remote: declined"; + for e in [ + Error::from(scl_objio::Error::Backend(evil.to_string())), + Error::from(scl_objio::Error::BadKey(evil.to_string())), + Error::Wal(evil.to_string()), + ] { + let shown = e.to_string(); + assert!( + !shown.contains('\x1b') && !shown.contains('\x07'), + "{shown:?}" + ); + assert!(shown.contains("declined")); + assert!(std::error::Error::source(&e).is_none()); + } + } +} diff --git a/crates/repo/src/lib.rs b/crates/repo/src/lib.rs index 84d7da75..90e43639 100644 --- a/crates/repo/src/lib.rs +++ b/crates/repo/src/lib.rs @@ -78,7 +78,8 @@ pub(crate) mod ws; mod ws_refresh; pub use bucket_transport::{ - BucketScheme, BucketTransport, BucketUrl, CompactOptions, CompactReport, + check_private_push_allowed, is_bucket_url, is_git_bucket_url, refuse_git_compact, + refuse_git_store, BucketScheme, BucketTransport, BucketUrl, CompactOptions, CompactReport, }; pub use conflicts::{ActiveOp, ConflictKind, ConflictVersions, ResolveSide, Side}; pub use error::{Error, Result}; diff --git a/crates/repo/src/remote.rs b/crates/repo/src/remote.rs index a2a49eaa..53b800dd 100644 --- a/crates/repo/src/remote.rs +++ b/crates/repo/src/remote.rs @@ -27,6 +27,13 @@ pub struct RemoteEntry { pub url: String, #[serde(default)] pub kind: RemoteKind, + /// Opt-in to pushing private branches to an `sc+git::` remote (the branch + /// name, recipient ids and sizes become readable by every reader of the + /// Git repository). Absent in configs written before it existed, which + /// parse as `false`; omitted on write while `false`, so those configs + /// stay byte-identical. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub allow_private_branches: bool, } impl RemoteConfig { @@ -58,6 +65,7 @@ impl RemoteConfig { RemoteEntry { url: url.to_string(), kind: RemoteKind::default(), + allow_private_branches: false, }, ); Ok(()) @@ -73,6 +81,7 @@ impl RemoteConfig { RemoteEntry { url: url.to_string(), kind, + allow_private_branches: false, }, ); Ok(()) @@ -88,6 +97,23 @@ impl RemoteConfig { self.remote.get(name).map(|r| r.kind) } + /// Whether `name` may receive private branches (`false` when unset or + /// when there is no such remote). + pub fn allow_private_branches(&self, name: &str) -> bool { + self.remote + .get(name) + .is_some_and(|r| r.allow_private_branches) + } + + /// Record the private-branch opt-in for an existing remote. + pub fn set_allow_private_branches(&mut self, name: &str, allow: bool) -> Result<()> { + self.remote + .get_mut(name) + .ok_or_else(|| Error::NoSuchRemote(name.to_string()))? + .allow_private_branches = allow; + Ok(()) + } + /// The configured remote names, sorted (the backing map is ordered). pub fn names(&self) -> Vec { self.remote.keys().cloned().collect() @@ -122,6 +148,26 @@ mod tests { std::fs::remove_dir_all(&layout.root).unwrap(); } + #[test] + fn allow_private_branches_roundtrips_and_old_configs_parse_false() { + let layout = tmp_layout("allowpriv"); + // A config from before the field existed. + std::fs::write(layout.config_path(), "[remote.origin]\nurl = \"/path/a\"\n").unwrap(); + let mut cfg = RemoteConfig::load(&layout).unwrap(); + assert!(!cfg.allow_private_branches("origin")); + assert!(!cfg.allow_private_branches("missing")); + // Not written while false: an old binary and a diff see no change. + cfg.save(&layout).unwrap(); + let text = std::fs::read_to_string(layout.config_path()).unwrap(); + assert!(!text.contains("allow_private_branches"), "{text}"); + cfg.set_allow_private_branches("origin", true).unwrap(); + assert!(cfg.set_allow_private_branches("missing", true).is_err()); + cfg.save(&layout).unwrap(); + let loaded = RemoteConfig::load(&layout).unwrap(); + assert!(loaded.allow_private_branches("origin")); + std::fs::remove_dir_all(&layout.root).unwrap(); + } + #[test] fn legacy_config_without_kind_loads_as_sc() { let layout = tmp_layout("legacy"); diff --git a/crates/repo/src/stdio_transport.rs b/crates/repo/src/stdio_transport.rs index 19c51c2b..b46b70a0 100644 --- a/crates/repo/src/stdio_transport.rs +++ b/crates/repo/src/stdio_transport.rs @@ -501,8 +501,10 @@ pub(crate) fn ssh_command(url: &SshUrl) -> Command { /// Open the right [`Transport`] for a remote URL: `ssh://` spawns the wire /// client over a child process's stdio, `sc+http://`/`sc+https://` dial it -/// directly over TCP (TLS-wrapped for the latter, P32); anything else is a -/// local `.sc/` path. +/// directly over TCP (TLS-wrapped for the latter, P32), `sc+wal://`, +/// `sc+s3://` and `sc+git::` open a bucket remote (a malformed `sc+git:` near +/// miss is refused there, never read as a path); anything else is a local +/// `.sc/` path. pub fn open_transport(url: &str) -> Result> { if url.starts_with("ssh://") { let parsed = SshUrl::parse(url)?; @@ -512,7 +514,7 @@ pub fn open_transport(url: &str) -> Result> { Ok(Box::new(crate::http_transport::HttpTransport::connect( &parsed, )?)) - } else if url.starts_with("sc+wal://") || url.starts_with("sc+s3://") { + } else if crate::bucket_transport::is_bucket_url(url) { Ok(Box::new(crate::bucket_transport::BucketTransport::open( url, )?)) diff --git a/crates/repo/src/sync.rs b/crates/repo/src/sync.rs index dbf13822..a9198545 100644 --- a/crates/repo/src/sync.rs +++ b/crates/repo/src/sync.rs @@ -143,9 +143,30 @@ impl Repo { /// under `refs/remotes/`, so it is validated like a branch name to keep a /// hostile name (e.g. `../heads`) from escaping into `refs/heads/`. pub fn remote_add(&self, name: &str, url: &str) -> Result<()> { + self.remote_add_opts(name, url, false) + } + + /// [`Repo::remote_add`] that also records the private-branch opt-in for an + /// `sc+git::` remote (see [`check_private_push_allowed`]) in the same + /// config write. + pub fn remote_add_opts( + &self, + name: &str, + url: &str, + allow_private_branches: bool, + ) -> Result<()> { validate_branch_name(name)?; let mut cfg = RemoteConfig::load(&self.layout)?; cfg.add(name, url)?; + cfg.set_allow_private_branches(name, allow_private_branches)?; + cfg.save(&self.layout) + } + + /// Record the private-branch opt-in on an existing remote (`sc clone + /// --allow-private-branches` sets it on the new `origin`). + pub fn remote_allow_private_branches(&self, name: &str) -> Result<()> { + let mut cfg = RemoteConfig::load(&self.layout)?; + cfg.set_allow_private_branches(name, true)?; cfg.save(&self.layout) } @@ -653,6 +674,17 @@ impl Repo { let url = cfg .url(remote) .ok_or_else(|| Error::NoSuchRemote(remote.to_string()))?; + // Before any bucket I/O: a private branch only goes to a Git-hosted + // store when the remote opted in (ADR-0060). + if crate::bucket_transport::is_git_bucket_url(url) { + if let Some((branch, _, _)) = self.head_private()? { + crate::bucket_transport::check_private_push_allowed( + url, + cfg.allow_private_branches(remote), + &branch, + )?; + } + } let transport = open_transport(url)?; let branch = refs::current_branch(&self.layout)?; let local_tip = self.head_tip()?.ok_or(Error::Unborn)?; diff --git a/crates/repo/src/wire.rs b/crates/repo/src/wire.rs index b2888c1b..bf51ad8c 100644 --- a/crates/repo/src/wire.rs +++ b/crates/repo/src/wire.rs @@ -1119,12 +1119,13 @@ pub fn serve_with_policy( /// (see [`TempServeDir::create_in`]) — the spool dir is created under its /// `.sc/tmp/`, not the process-wide temp dir. fn open_bucket_serve_transport(store_url: &str, home: &std::path::Path) -> Result { + crate::bucket_transport::refuse_git_store(store_url)?; let transport = BucketTransport::open(store_url)?; let tmp = TempServeDir::create_in(home)?; Ok(ServeTransport::Bucket { transport, tmp }) } -/// Serve a bucket WAL (`sc+wal://`/`sc+s3://`) over the same wire protocol +/// Serve a bucket WAL (`sc+wal://`/`sc+s3://`; `sc+git::` is refused) over the same wire protocol /// `serve_with_policy` speaks for a local repo (P36c): same handshake, same /// `PROTOCOL_VERSION`, same read-only gate and pack-spool caps — every verb /// except the pack pair goes through [`Transport`] identically either way. From f8ffb566601e082f82be39dd36d11d36a6c24018 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 11:30:02 +0200 Subject: [PATCH 05/15] test: run the sc+git:: end-to-end tests against the real GitBucket --- crates/cli/tests/git_bucket.rs | 5 +---- crates/repo/src/bucket_transport/git_remote_tests.rs | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/crates/cli/tests/git_bucket.rs b/crates/cli/tests/git_bucket.rs index 5afe014a..8445c665 100644 --- a/crates/cli/tests/git_bucket.rs +++ b/crates/cli/tests/git_bucket.rs @@ -3,8 +3,7 @@ //! scenario (private branch + protected path + secret over a Git carrier). //! //! The scenario runs twice from one body: over `sc+wal://` as a control that -//! proves the script itself (always on), and over `sc+git::file://` -//! (`#[ignore = "needs real GitBucket"]` until the objio implementation lands). +//! proves the script itself, and over `sc+git::file://`. use std::path::{Path, PathBuf}; use std::process::{Command, Output}; @@ -456,7 +455,6 @@ fn scenario_over_a_wal_bucket_is_the_control() { } #[test] -#[ignore = "needs real GitBucket"] fn private_branch_protected_path_and_secret_round_trip_over_a_git_carrier() { scenario(git_carrier("roundtrip"), true); } @@ -464,7 +462,6 @@ fn private_branch_protected_path_and_secret_round_trip_over_a_git_carrier() { /// Carrier URL forms through the CLI: file://, scp-form, ssh://, https://. No /// network is touched: `remote add` validates the URL and writes the config. #[test] -#[ignore = "needs real GitBucket"] fn remote_add_accepts_scp_form_and_file_carrier_urls_and_notices_once() { let a = tmp("forms"); init_repo(&a); diff --git a/crates/repo/src/bucket_transport/git_remote_tests.rs b/crates/repo/src/bucket_transport/git_remote_tests.rs index 914eb063..898dfa89 100644 --- a/crates/repo/src/bucket_transport/git_remote_tests.rs +++ b/crates/repo/src/bucket_transport/git_remote_tests.rs @@ -1,7 +1,6 @@ //! `sc+git::` remote plumbing: URL parsing and the reserved prefix, the //! private-branch gate, and the refusals (`serve --store`, `remote compact`). -//! Nothing here reaches `GitBucket`'s real I/O except the test marked -//! `#[ignore = "needs real GitBucket"]`, which needs the objio implementation. +//! Nothing here reaches `GitBucket`'s real I/O except the URL-parse test. use super::*; use crate::error::Error; @@ -172,7 +171,6 @@ fn serve_store_and_compact_refuse_git_carriers_before_parsing() { } #[test] -#[ignore = "needs real GitBucket"] fn sc_git_urls_parse_with_the_inner_url_verbatim() { for inner in [ "file:///srv/r.git", From 5d0c0bdf0d4d786cb1f7b149dd8b9786de2d91e6 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 11:33:41 +0200 Subject: [PATCH 06/15] docs: ADR-0060 and P39 docs for sc+git:: remotes ADR-0060 records the refs/sc/* mapping, lease CAS, private-branch opt-in, deferred compaction and serve --store, and the dependency-rule amendment. CLAUDE.md, ARCHITECTURE, THREAT-MODEL, ROADMAP, README and the sc-cli-reference skill carry the capability row, boundaries and follow-ons. --- .claude/skills/sc-cli-reference/SKILL.md | 32 +++ ARCHITECTURE.md | 46 +++- CLAUDE.md | 27 ++- README.md | 22 +- ROADMAP.md | 27 +++ docs/THREAT-MODEL.md | 52 ++++ docs/adr/0060-git-carried-bucket-remotes.md | 254 ++++++++++++++++++++ docs/adr/README.md | 7 +- 8 files changed, 452 insertions(+), 15 deletions(-) create mode 100644 docs/adr/0060-git-carried-bucket-remotes.md diff --git a/.claude/skills/sc-cli-reference/SKILL.md b/.claude/skills/sc-cli-reference/SKILL.md index 3b0123a6..dc6ca190 100644 --- a/.claude/skills/sc-cli-reference/SKILL.md +++ b/.claude/skills/sc-cli-reference/SKILL.md @@ -407,6 +407,30 @@ cargo run --bin sc -- remote add sc+s3:/// # same, on a # content stays ciphertext); `clone --filter` is # refused; signatures, transcripts and drop # tombstones travel like on any native remote (#229) +cargo run --bin sc -- remote add sc+git:: [--allow-private-branches] + # bucket remote stored INSIDE a Git repository as + # refs/sc/* (P39, ADR-0060); is https://, + # http://, ssh://, file:///abs, scp-form + # (git@host:o/r.git) or an absolute path. `sc+git:` + # is a reserved prefix (only `sc+git::` parses), + # `--git` is an error on it, no password in the URL + # (use git's credential helper / ssh agent), SC_GIT + # overrides the git binary. `clone sc+git:: + # [--allow-private-branches]` takes the same URL and + # sets the opt-in on `origin`. Pushing a PRIVATE + # branch needs --allow-private-branches (its name, + # recipient ids and sizes become readable by everyone + # who can read the repo, ciphertext stays after a + # revoke); it refuses before any upload without it. + # Protected paths and secrets need no opt-in. A plain + # `git clone` of the repo never fetches refs/sc/*; + # `git ls-remote` lists them; the same repo can also + # hold a `--git` mirror of public branches. Anyone + # with push to the repo holds bucket put + delete; + # deleting a ref is not erasure. `remote compact` + # and `serve --store` refuse it in v1; clone + # --filter refused as for any bucket. Untested + # against real GitHub cargo run --bin sc -- remote compact [--dry-run] [--prune-expire 24h] [--allow-short-grace] # bucket remotes only (#216, ADR-0057): merge small # packs behind one manifest CAS, then delete log @@ -429,6 +453,14 @@ cargo run --bin sc -- serve --http --store sc+wal://|sc+s3:// # all content lives in the bucket (instance is # disposable); still needs `.sc/` for tokens, # the TLS identity and scratch space +bash demo/run_git_carrier_demo.sh # sc+git:: proof (P39): a bare Git repo as the host; + # protected path + secret + private branch pushed + # (private branch refused without the opt-in), plain + # `git clone` sees no refs/sc, keyless clone sees + # `(private, no access)` and no plaintext, keyed + # clone reads all, public main also mirrored as a + # normal Git branch; second leg against sc-forge; + # no sc-git-bucket-* temp leftovers bash demo/run_http_remote_demo.sh # sc-native HTTP transport proof (P26): real # loopback TCP (no shim, unlike ssh://), clone + # push + fetch over sc+http://, a signed ~1 MiB diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2dad7e62..b437cef3 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -35,7 +35,7 @@ composability held across every phase since. **All three long-term-thesis pillar now ship** — in-RAM worktrees (P1), committed secrets (P2), and per-file permissions (P7, encrypted paths; per-branch in P34) — atop a full collaborative VCS: three-way merge and history editing, packfiles + GC, ssh:// / sc+http(s):// / -bucket (sc+wal://, sc+s3://) / Git network transports, sparse and partial clone +bucket (sc+wal://, sc+s3://, sc+git::) / Git network transports, sparse and partial clone (with lazy fetch, merge and agent sessions on a partial clone), signed commits, intent-deep provenance via sealed agent-session transcripts, and a read-only native desktop browser. The phase-by-phase build log is in @@ -82,8 +82,9 @@ src-control/ │ │ v2 dual-key identities (scl-crypto; the only crate linking RustCrypto) │ ├── tlsio/ rustls/rcgen wrappers for sc+https:// (P32); a leaf, the only crate │ │ linking rustls -│ ├── objio/ `Bucket` trait + directory and S3 backends for sc+wal:// / sc+s3:// (P36); -│ │ a leaf, the only crate linking the S3 SDK +│ ├── objio/ `Bucket` trait + directory, S3 and Git backends for sc+wal:// / sc+s3:// / +│ │ sc+git:: (P36, P39); a leaf, the only crate linking the S3 SDK; its Git +│ │ backend spawns the system `git` (never gix) │ ├── repo/ durable .sc/ repo: refs/branches/working tree, three-way merge, history │ │ editing (cherry-pick/rebase/amend/undo), remotes + wire transport (ssh://, │ │ sc+http(s)://, bucket WAL), protection + secret lifecycle, GC, sparse & @@ -220,7 +221,8 @@ lazily-built bare `mirror.git` beside the marks map is the transport). All of it keeps `gix` quarantined in `gitio`. This is distinct from sc's **own** network transport: `repo` speaks a framed wire protocol over ssh:// (P12), sc+http:// (P26) and sc+https:// (P32) that mirrors its `Transport` trait, and treats a -bucket as a remote (P36) — no `gix` involved (see the appendix). +bucket as a remote (P36), which P39 can store inside a Git repository (`sc+git::`, +through the system `git` binary, not `gix`) — no `gix` involved (see the appendix). ## Cryptography and key management @@ -1344,3 +1346,39 @@ out-of-view subtrees from the base without reading them; and harvest lands through the ADR-0049 merge, with any conflict falling back to a `work-` branch as on a full clone. `sc backfill --all` remains the way to a genuine full clone. See ADR-0050. + +## Phase 39 — Git-carried bucket remotes: sc+git:: (built) + +The bucket WAL (P36) stored inside any Git repository, so a Git host can carry +private branches, protected paths and secrets that the Git bridge cannot. +`sc+git::` opens `objio::GitBucket` behind the same `Bucket` trait, so +`BucketTransport`, the log, the manifest and every format stay unchanged: no wire +change, no `PROTOCOL_VERSION` bump, no new crypto. Key `K` is the ref `refs/sc/K`; +its value is a parentless commit whose tree holds the bytes as chunk blobs +`c0000`, `c0001`, ... (at most 32 MiB each, 2 GiB per value), and the commit id is +the CAS tag. A default `git clone` or `git fetch` never takes `refs/sc/*`, and the +P18 bridge touches only `refs/heads/*`, so one repository can hold the sc store and +a plain Git mirror of the public branches. + +`put_new` and `put_if_tag` push with `--force-with-lease` (an empty lease for +create-only), never a `+` refspec, so `receive-pack` checks the sent old value under +its ref lock and exactly one racer wins. A rejected push is classified by re-reading +the exact ref, so a hook refusal is an error and never a lost race. Reads keep a 2 s +freshness window for a tag the instance itself saw or wrote, serve create-only values +from the local cache, and fetch the `log/`, `checkpoints/` and `packs/*.idx` families +in one shallow `git fetch`, which bounds a cold clone's round trips independent of log +length. Git is the system binary (`SC_GIT` overrides) with a scrubbed environment, a +transport allowlist (`https`, `http`, `ssh`, `file`), a URL allowlist and no password +in the URL; a private bare repo in `sc-git-bucket--` caches fetched +objects and is removed on `Drop`. + +A private branch goes to an `sc+git::` remote only under the per-remote opt-in +`--allow-private-branches`, since its name, recipient ids and sizes become readable by +every reader of the repository and the ciphertext cannot be taken back. `sc remote +compact` (no trustworthy age for a stored pack) and `sc serve --store` (a daemon cannot +answer prompts, and it would bypass the opt-in) refuse an `sc+git::` store in v1. +sc-forge accepts exactly the `refs/sc/` prefix (ADR-0058 addendum). The dependency rule +is amended: `objio` may spawn the system `git`, and only `gitio::bridge` and objio's +git backend spawn it; `gix` stays in `gitio`. Proven by `crates/cli/tests/git_bucket.rs`, +the forge end-to-end tests and `demo/run_git_carrier_demo.sh`. Not tested against real +GitHub. See ADR-0060. diff --git a/CLAUDE.md b/CLAUDE.md index 94b89a0c..476ad1e3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,7 +47,11 @@ yourself reaching for it elsewhere, add a function to `crypto` instead. **rustls/rcgen must stay quarantined in `tlsio`** — if you find yourself reaching for TLS elsewhere, add a function to `tlsio` instead. **Object-store SDKs must stay quarantined in `objio`** — if you find yourself reaching for -S3 elsewhere, add a function to `objio` instead. **`repo` must not depend on +S3 elsewhere, add a function to `objio` instead. **`objio` also holds the +`sc+git::` backend (P39), which may spawn the system `git` binary (never `gix`); +the system `git` binary is spawned only by `gitio::bridge` and objio's git +backend** (`repo` must not depend on `gitio`, and `repo` picks the bucket +backend). **`repo` must not depend on `gitio`** — `cli` links both and passes imported snapshots down; `repo` stays Git-agnostic. **`forge` (the `sc-forge` Git forge, P38, unix-only) is a third top-level adapter and may link only `{crypto, core}`** — never `repo`, `vfs` or `gitio` (hosted @@ -110,7 +114,7 @@ the project tree if desired. ## Capability map (what's built, by phase) -All 36 phases, P37a, P37b-i, P37b-ii, P37c, P37d and P38 are built and tested. One line of current fact per phase; the +All 36 phases, P37a, P37b-i, P37b-ii, P37c, P37d, P38 and P39 are built and tested. One line of current fact per phase; the authoritative rationale and full semantics live in the linked ADR, the design in `ARCHITECTURE.md`. The old per-phase narrative log this table replaced is archived verbatim at `docs/archive/claude-md-phase-log-2026-07.md` — do not @@ -160,6 +164,7 @@ the code, those win. | P37c | Merge, cherry-pick and rebase on a partial clone: a tree-recursive engine (`merge_trees`) carries a subtree by id when the sides agree and no protection rule applies, reads only where they differ or a rule intersects, and on a lazy, online clone fills what it asks for by id outside the store lock (a non-lazy clone gets the gap error with `sc backfill `); a carried protected subtree keeps the complete input wrap union; a conflict inside the view is marked and resolved like on a full clone (`sc conflicts`/`sc resolve`/`merge --abort`/`sc commit`; the out-of-view part is grafted from the decided root), while a conflict outside the view, or a result that must be sealed outside it, is refused before any ref, merge-state or working-tree change (fetched objects stay). A subtree identical on all three sides is carried even under a protection rule. Cherry-pick and rebase use the same engine (`replay_commit`): the fold's accumulator may hold absent subtrees, the view is filled before the final write, and a stopped pick or rebase keeps its carried subtrees in the decided root. Mode-only edits now merge and a file/directory collision fails before any write, on full clones too. | [0049](docs/adr/0049-merge-on-partial-clones.md) | | P37d | Agent sessions on a partial clone: `sc ws fork`/`run`/`harvest`/`abandon` and `sc work` work; a workspace or agent sees the host's sparse view only (`sc work` no longer gets a complete tree there), the view closure is filled (lazy, online) or refused with the gap error before anything is written, a workspace commit grafts the out-of-view subtrees from the base without reading them, and harvest lands through the ADR-0049 merge (any conflict falls back to a `work-` branch as on a full clone) | [0050](docs/adr/0050-sessions-on-partial-clones.md) | | P38 | Gitea-compatible forge over bare Git and smart HTTP: recorded API subset, scoped auth, protected merges, maintenance and container operations; 24 recorded scenarios and six first-consumer acceptance scenarios pass. Hosted repos have none of the `.sc/` features; SSH and Actions remain outside v1. | [0058](docs/adr/0058-sc-forge-gitea-compatible-git-forge.md) | +| P39 | `sc+git::` remotes: the bucket WAL stored as one parentless commit per key at `refs/sc/` in any Git repository (system `git` via `objio::GitBucket`, lease CAS, no `+` refspec), so private branches, protected paths and secrets travel through a Git host (local bare repo, sc-forge; GitHub untested); private-branch push needs per-remote `--allow-private-branches`; `remote compact` and `serve --store` refuse it in v1; sc-forge accepts `refs/sc/` | [0060](docs/adr/0060-git-carried-bucket-remotes.md) | ## Standing boundaries & gotchas @@ -216,6 +221,24 @@ transport-adjacent. The rest, imperatively: - **Bucket remotes hold public content plaintext at rest** — bucket ACL is the perimeter (sealed content stays ciphertext, unchanged); partial-clone `filter` against bucket remotes is refused. +- **`sc+git::` (P39, ADR-0060) stores the bucket WAL as `refs/sc/*` in a + Git repository.** The prefix `sc+git:` is reserved: only `sc+git::` + parses (inner: `https://`, `http://`, `ssh://`, `file:///abs`, scp-form or an + absolute path; no password in the URL, no `git://`, no `::`), `--git` is an + error on it and it never reaches the P18 bridge. Pushing a private branch needs + the per-remote opt-in `--allow-private-branches` (`remote add`/`clone`; + enforced in `repo`, refuses before any upload); protected paths and secrets + need none. `sc remote compact` and `sc serve --store` refuse it (deferred: no + trustworthy pack age, no prompts in a daemon). Every git call goes through + `objio::GitBucket` with a scrubbed environment; CAS is `--force-with-lease`, + never a `+` refspec, and a rejected push is classified by re-reading the ref. + **Everyone with push to the Git repository holds bucket put and delete** (no + versioning; wider than S3), branch protection does not cover `refs/sc/*`, and + rewind stays the tracked follow-on. **Deleting a ref is not erasure** (hosts + and forks may keep objects by id). Public content is plaintext, and private + branch names, recipient ids and sizes are readable by every reader of the + repository. A killed process can leave `sc-git-bucket--` in the + temp dir. Not tested against real GitHub. - **`DirBucket` (`sc+wal://`) is a single-host, single-owner backend (#264).** Its CAS takes an OS advisory file lock (`File::lock`: `flock` on unix, `LockFileEx` on Windows) on `/.cas-lock`, a file that is never removed and diff --git a/README.md b/README.md index 868c9c7a..e09218ce 100644 --- a/README.md +++ b/README.md @@ -64,8 +64,9 @@ crates/ Ed25519 signing (the ONLY crate that links RustCrypto) tlsio/ TLS for the sc+https:// transport (the ONLY crate that links rustls/rcgen; a leaf — no workspace deps) - objio/ object-store buckets for sc+wal:// and sc+s3:// remotes (the ONLY crate - that links the S3 SDK; a leaf — no workspace deps) + objio/ object-store buckets for sc+wal://, sc+s3:// and sc+git:: remotes (the + ONLY crate that links the S3 SDK; its Git backend spawns the system + `git`; a leaf — no workspace deps) repo/ durable .sc repo — refs, branches, working tree, merge, history editing, remotes, protection, GC, sparse/partial clone, transcripts, transport (ssh, sc+http(s), bucket WAL) @@ -136,6 +137,14 @@ Bucket remotes take `sc remote add sc+wal://` or (ADR-0057), and `sc serve --http 127.0.0.1:8730 --store ` serves a bucket from a disposable instance (`--stdio` also works; `sc serve --help`). +`sc remote add sc+git::` (or `sc clone sc+git:: `) +stores the same bucket inside any Git repository as `refs/sc/*`, so private +branches, protected paths and secrets travel through a Git host that a plain +`git clone` never sees into (ADR-0060). Pushing a private branch there needs +`--allow-private-branches` (its name and recipient ids become readable by +everyone who can read the repository); compaction and `serve --store` are not +supported on it yet. Proven by `demo/run_git_carrier_demo.sh`; not tested +against GitHub itself. ## How Phase 1 works @@ -168,11 +177,12 @@ provenance (droppable through signed tombstones), sparse and partial checkouts ( cherry-pick, rebase and agent sessions on a partial clone), and local, ssh-native, sc-native HTTP/TLS (bearer tokens with expiry and rotation, listener resource limits, TOFU pinning managed by `sc tls`, or CA validation -via `SC_HTTPS_CA_FILE`/`SC_HTTPS_CA=system`), bucket (`sc+wal://`, `sc+s3://`), -and Git (including hosted GitHub) network transports, plus private branches +via `SC_HTTPS_CA_FILE`/`SC_HTTPS_CA=system`), bucket (`sc+wal://`, `sc+s3://`, +`sc+git::` inside a Git repository), and Git (including hosted GitHub) network +transports, plus private branches sealed to a recipient set until publish and a read-only Tauri desktop -browser. All 36 phases and P37a-37d are implemented and tested; `demo/` holds -28 independent end-to-end proofs covering most major capabilities. Remaining +browser. All 36 phases, P37a-37d and P39 are implemented and tested; `demo/` holds +29 independent end-to-end proofs covering most major capabilities. Remaining follow-ons (richer signer trust models, per-path token ACLs, transcript rewrap/grant/revoke and similar items) are tracked in [ROADMAP.md](ROADMAP.md)'s Deferred section and as open GitHub issues rather than being open MVP scope. diff --git a/ROADMAP.md b/ROADMAP.md index e0b01e19..78ab3109 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -694,6 +694,19 @@ across every phase. file and the auto-minted TLS serve key get the same owner-only DACL through one shared primitive, `scl_core::owner_only` (#281). +- **Phase 39 — Git-carried bucket remotes (`sc+git::`).** The bucket WAL + stored as one parentless commit per key at `refs/sc/` in any Git + repository, through the system `git` binary (`objio::GitBucket`, lease CAS with + no `+` refspec, rejections classified by re-reading the ref, 2 s freshness + window, create-only caching and family-batched fetches). Private branches, + protected paths and secrets travel through a Git host (a local bare repo and + sc-forge verified; real GitHub untested); a plain `git clone` never fetches + `refs/sc/*`, and the same repository can also mirror public branches through a + `--git` remote. A private-branch push needs the per-remote opt-in + `--allow-private-branches`; `sc remote compact` and `sc serve --store` refuse + `sc+git::` in v1; sc-forge accepts the `refs/sc/` prefix. No format, wire or + crypto change. Proven by `demo/run_git_carrier_demo.sh`. (ADR-0060.) + ## Active No phase is currently active. Follow-up work is tracked under Deferred below. @@ -702,6 +715,7 @@ No phase is currently active. Follow-up work is tracked under Deferred below. | Phase | Goal | Demoable outcome | ADR | |-------|------|------------------|-----| +| **P39 — Git-carried bucket remotes** | Carry private branches, protected paths and secrets through a Git host | `sc remote add origin sc+git::` (or `sc clone sc+git:: `): the bucket WAL as `refs/sc/*` in any Git repository; a private-branch push needs `--allow-private-branches`; a plain `git clone` of the same repo sees none of it; sc-forge accepts `refs/sc/`; proven by `demo/run_git_carrier_demo.sh` | [0060](docs/adr/0060-git-carried-bucket-remotes.md) | | **P38 — sc-forge** | Configurable project Git host | Bare Git over smart HTTP, recorded Gitea API subset, protected merges and operations tooling; all 24 recorded scenarios and all six first-consumer acceptance scenarios pass. Local acceptance only; production migration is separate. | [0058](docs/adr/0058-sc-forge-gitea-compatible-git-forge.md) | | **P4 — Merge & conflict resolution** | Combine work from two branches | `sc merge ` creates a merge snapshot; clean merges auto-resolve, conflicts are detected and reported | [0012](docs/adr/0012-three-way-merge.md) | | **P5 — Secret scanner (accidental-plaintext guard)** | Stop plaintext secrets being committed | pattern + entropy scan hard-rejects plaintext secrets, with `sc scan` and a hash-scoped allowlist | [0017](docs/adr/0017-secret-scanner.md) | @@ -1002,6 +1016,19 @@ P28 review follow-ons, the port de-flake, the clippy allow-list) are recorded in issues a `has_object` round trip per object over S3; batching those probes into one `refresh()` plus local index lookups (instead of one S3 round trip per object) is deferred. +- **`sc+git::` follow-ons (P39, ADR-0060).** Deferred rather than built: (1) + `sc remote compact` on an `sc+git::` remote, which needs an age source the + compactor controls (a Git repository gives only writer-controlled committer + time, so `list_meta` reports observation time and v1 refuses the command); + (2) `sc serve --store sc+git::...` (a daemon cannot answer ssh or credential + prompts, and it would sit between clients and the private-branch opt-in); (3) + rollback and rewind of `refs/sc/manifest` by anyone with push access, tracked + under P34 follow-on (6) and ADR-0059 follow-on (2), now reachable by more + people; (4) validation against real GitHub (per-ref and per-push limits, + rulesets, hosts that refuse unknown ref namespaces), untested so far; (5) + batched deletes (one `git push` for many `:refs/sc/K`) and a batched + `has_objects`, since a Git host costs a round trip per probe or delete; (6) + `list_meta` downloads every listed value to read its size. - **Serve-side persistent pack cache (P36c follow-on).** A bucket-backed serve instance re-downloads packs per connection; a content-addressed on-disk cache in the serve home would make warm instances cheap without diff --git a/docs/THREAT-MODEL.md b/docs/THREAT-MODEL.md index a2bd22cf..f4a5e285 100644 --- a/docs/THREAT-MODEL.md +++ b/docs/THREAT-MODEL.md @@ -718,6 +718,58 @@ too, not just the src-control-side metadata. the `MAX_OBJECT_SIZE` guards, so this costs bandwidth and CPU, not memory beyond one pack body. +## Git-carried bucket remotes (`sc+git::`) — ADR-0060 + +The bucket model above applies unchanged, with the Git repository as the bucket and +`refs/sc/*` as its keys. What differs: + +- **The writer set is wider than S3.** Everyone with push access to the Git repository + (collaborators, deploy keys, CI tokens) holds bucket put **and** delete, with no object + versioning to recover from. A writer can rewind `refs/sc/manifest` to a pre-revoke + manifest, delete keys or add packs. Branch protection and rulesets do not cover + `refs/sc/*` (sc-forge applies none to it either). Rewind is the same tracked follow-on as + for any bucket (a fetch-time freshness attestation, ROADMAP P34 follow-on 6, and + ADR-0059 follow-on 2); `sc+git::` makes it reachable by more people. +- **Metadata exposure.** Anyone who can read the repository can list `refs/sc/*` (`git + ls-remote`) and fetch it by exact refspec; a default `git clone`/`git fetch` does not. They + see public content as plaintext in the packs, and for private branches the branch name, + recipient ids, sealed-object counts and sizes and the public fork point (ADR-0044's accepted + leak, now to every reader of a Git host). Protected-path and secret ciphertext stays + ciphertext. Because that exposure is wider, pushing a private branch needs the per-remote + opt-in `--allow-private-branches`, enforced in `repo` before any upload; protected paths and + secrets need none. +- **Deleting a ref is not erasure.** A host can keep the objects fetchable by id (sc-forge does, + until its gc prune stage), and forks, mirrors and clones keep their own copies. A deleted + private-branch ciphertext or a pre-rewrap value should be treated as published; the cutover + is rotation (`sc rewrap --rotate`), as everywhere. +- **Transport allowlist and process hygiene.** The inner URL is `https://`, `http://`, `ssh://`, + `file:///abs`, scp-form or an absolute path. `git://`, relative paths, `::` transport helpers, + a leading `-`, control characters and a password in the URL are refused, and every git call + carries `-c protocol.allow=never` plus `always` for `file`, `ssh`, `http` and `https`, so a + hostile URL or redirect cannot reach `ext::` or `fd::`. The environment is scrubbed of + `GIT_DIR`-family and config-injection variables, hooks and signing are off, stdin is closed and + prompts are disabled. `http` stays allowed because plaintext is the user's choice, as in git. + Errors carry git's captured stderr with URL userinfo redacted and control bytes escaped, so a + hostile hook message cannot drive the terminal. +- **A hostile host is bounded.** A value is capped at 2 GiB and a chunk at 32 MiB, a tree at 10,000 + chunks, captured `ls-remote` output at 64 MiB; fetches are shallow so a hostile commit's parents + are never downloaded; a fetched value must be a parentless commit over contiguous regular chunk + blobs or it is refused as corrupt. The WAL's own 256 MiB metadata cap and pack bounds still + apply. A host can still serve large values (disk in the temp cache, RAM in `get`), the same + class as S3. +- **Rejections are classified by re-reading the ref**, not by the host's wording, so a hook or + permission refusal is never reported as a lost race. The atomic CAS relies on the host checking + the sent old value under its ref lock: stock `git receive-pack` and sc-forge do (verified); + another host that does not cannot carry the store safely. +- **Not built in v1.** `sc remote compact` is refused (committer time is writer-controlled, so it + cannot be an age source) and so is `sc serve --store sc+git::` (a daemon cannot answer prompts + and would bypass the client-side private-branch opt-in). +- **Local cache.** A temporary bare repository `sc-git-bucket--` (mode 0700) holds + fetched store objects and is removed on exit; a killed process (SIGKILL, or SIGINT) can leave it + behind, and there is no reaper. +- Not tested against real GitHub. The "not independently audited, do not commit production + secrets" warning at the top of this document applies unchanged. + ## By-id object reads (`GetObjects`, `sc cat`) — ADR-0047 - **A hash is not a capability.** Anyone who can reach a served repo and diff --git a/docs/adr/0060-git-carried-bucket-remotes.md b/docs/adr/0060-git-carried-bucket-remotes.md new file mode 100644 index 00000000..5dd85aa7 --- /dev/null +++ b/docs/adr/0060-git-carried-bucket-remotes.md @@ -0,0 +1,254 @@ +# ADR-0060: Git-carried bucket remotes (`sc+git::`) + +- **Status:** Accepted +- **Date:** 2026-11-21 +- **Phase:** 39 +- **Builds on:** ADR-0046 (bucket WAL remotes), ADR-0044 (private branches), + ADR-0058 (sc-forge, addendum for `refs/sc/*`) +- **Amends:** ADR-0020 (dependency rule: `objio` may spawn the system `git` + binary), ADR-0044 (reason 4 for rejecting opaque sealed refs through the Git + bridge, reversed only under a per-remote opt-in) + +## Context + +Theo asked for three things in the place teams already host code, a Git host: +files only some teammates can read, branches only some teammates can see, and +secrets committed with the code. src-control has all three (P7 protected paths, +P34 private branches, P2 secrets), and none survives a Git host today. The Git +bridge (ADR-0016/0018/0028) refuses private branches, drops secrets, and +exports a protected path as bare ciphertext that loses its rules on re-import. +sc-forge (P38) is plain Git and adds nothing here. + +An experiment showed the bucket WAL already carries all three. A repository +with a protected `config/`, a secret `API_TOKEN` and a private branch `hotfix` +was pushed to `sc+wal://`. A keyless clone listed `hotfix (private, no +access)`, had no `config/prod.env`, and saw the secret name without its value. +A keyed clone decrypted the file, injected the secret with `sc run`, and read +the embargoed branch. The WAL moves opaque values, so sealed content passes +through untouched. What is missing is a `Bucket` backend that stores those +values in a Git repository, which any Git host can hold. + +## Decision + +Add `sc+git::`: the existing bucket WAL, with `objio::GitBucket` +storing the bucket inside the Git repository at ``. No format change, +no wire change, no `PROTOCOL_VERSION` bump, no new crypto. + +### Mapping + +Bucket key `K` lives at the ref `refs/sc/K` (`refs/sc/manifest`, +`refs/sc/log/`, `refs/sc/packs/.pack`). Keys get a strict check on +top of `validate_key`, on every method including reads: `[A-Za-z0-9._/-]`, at +most 1024 bytes, no `..`, no component that starts with `.` or ends with `.` or +`.lock`. A default `git clone` or `git fetch` takes only heads and tags, and +the P18 bridge touches only `refs/heads/*`, so one Git repository can hold an +sc store and a plain Git mirror of the public branches side by side. + +### Value encoding + +A value is a parentless commit whose tree holds the bytes as chunk blobs `c0000`, +`c0001`, ... of at most 32 MiB each (GitHub's per-blob limit is 100 MB). The +commit message is `sc-bucket new ` for a create-only write and +`sc-bucket cas ` for a compare-and-swap write. The nonce makes every +write a new commit, so a fresh write can never be "up to date". Author and +committer are fixed (`sc `), so a clean account with no +`user.name` can write. The tag is the commit id. There is no parent link, so +fetching the manifest never pulls an old manifest. A value is at most 2 GiB, +for writes and reads (GitHub's per-push cap, and sc-forge's +`receive.maxInputSize` default); a larger value fails with an error naming the +cap. + +### Atomicity: leases, never `+` + +`put_new` pushes with `--force-with-lease=refs/sc/K:` (an empty lease: the ref +must not exist). `put_if_tag(Some(t))` pushes with `--force-with-lease=refs/sc/K:`. +The receiving `receive-pack` checks the sent old value under its ref lock, so +two racing creates yield one `[new reference]` and one `[remote rejected] +(reference already exists)`, and two racing swaps on the same expected value +yield one winner. A `+` refspec is never used: with `+`, a wrong lease still +force-updated in the experiment (git 2.54). + +A rejected push is classified by re-reading the exact ref with `ls-remote`, +never by the host's wording: + +- `put_new`: the ref exists means someone else created it (`false`); absent + means a genuine refusal (error). +- `put_if_tag`: the ref no longer equals the expected tag (changed or deleted) + means someone else won (`None`); it still equals the expected tag means a + genuine refusal such as `pre-receive hook declined` (error). + +So a hook or permission refusal never looks like a lost race, which +`BucketTransport` would report as a non-fast-forward and the user would retry +forever. A tag that is not a full nonzero hex object id returns `None`. Delete +pushes `:refs/sc/K` and treats an already absent ref as success. + +### Reads: freshness, create-only caching, family batching + +Each `GitBucket` has a private bare repository in `sc-git-bucket--` +under the temp directory (mode 0700), removed on `Drop`. + +- A `get` whose cached tag equals the tag this instance observed or wrote for + that key less than 2 s ago returns `Unchanged` with no network call. + `BucketTransport::has_object` refreshes per object, which would otherwise cost + one `ls-remote` each. Staleness is bounded and only causes a failed server-side + CAS or a redundant upload. +- A key whose cached commit says `new` is create-only. This instance serves it + from the local cache with no network call at all; its bytes stay valid even + if compaction later deletes the key. +- The first cache miss under `log/`, `checkpoints/` or a `packs/*.idx` key + fetches that whole family in one `git fetch` (a glob refspec). This is + GitBucket's knowledge of the WAL key layout. `.pack` values are fetched one + by one. All fetches are `--depth=1`, so a hostile commit's parents are never + downloaded. A counting `SC_GIT` shim test bounds the git network calls of a + cold clone and an incremental push independent of the log length. +- A fetched value is validated before assembly: a commit with no parent, a tree + of regular blobs named `c0000` onward without gaps, nothing else, each chunk + at most 32 MiB, total at most 2 GiB. Anything else is a corrupt store and an + error. + +### URL allowlist and process hygiene + +The inner URL must be `https://`, `http://`, `ssh://`, `file:///abs`, scp-form +`user@host:path` or an absolute local path. Refused: `git://`, relative paths, +a leading `-`, control characters, `::` (git transport helpers), and a +password in the URL (use git's credential helper; sc never stores one). `sc+git:` +with a single colon, or any near miss, is a reserved prefix and is refused +unless it is exactly `sc+git::`, so it never falls through to a local path +or to the mirror bridge. `--git` is an error on an `sc+git::` URL. + +Every git call runs with `-c protocol.allow=never` plus `always` for `file`, +`ssh`, `http` and `https`; `--end-of-options` before the URL; stdin closed; +`GIT_TERMINAL_PROMPT=0` and `LC_ALL=C`; hooks off (`core.hooksPath=/dev/null`), +`gc.auto=0`, `maintenance.auto=false`, signing off, `core.useReplaceRefs=false`, +and `http.lowSpeedLimit`/`lowSpeedTime` so a stalled host cannot hang a push. +`GIT_DIR` and every other `GIT_*` variable is removed except the user's +transport and credential setup (`GIT_SSH*`, `GIT_ASKPASS`, `GIT_CONFIG_GLOBAL`, +`GIT_SSL_*` and a few more), then `GIT_DIR` is set to the cache. The user's +credential helper and ssh agent keep working. `SC_GIT` overrides the binary, as +for the P18 bridge. Captured stderr is scrubbed (URL userinfo redacted, control +bytes dropped, length bounded), and the `repo` error type escapes control bytes +from any bucket error and exposes no raw `source`, so a hostile hook message +cannot drive the terminal. `ls-remote` patterns tail-match, so every result is +compared by exact full ref name. + +### Caps against a hostile host + +2 GiB per value, 32 MiB per chunk, 10,000 chunks per value, 64 MiB of captured +`ls-remote` output, 1 MiB for other captured output. The WAL's own caps on +manifest, log, checkpoint and idx reads (256 MiB) and the pack record bounds +apply on top. + +### Private-branch opt-in + +Pushing a private branch to an `sc+git::` remote needs a per-remote setting, +`allow_private_branches`, set by `sc remote add sc+git::... --allow-private-branches` +or `sc clone sc+git::... --allow-private-branches` (which sets it on the +new `origin`). Without it, `sc push` refuses before any upload and names the +flag. The gate lives in `repo` (the push path), not only the CLI. Protected paths +and secrets need no extra opt-in: choosing the scheme is the opt-in. `remote +add` and `clone` print one notice that public content and sealed metadata are +readable by everyone who can read the Git repository. + +This reverses ADR-0044's reason (4) for rejecting "opaque sealed refs through +the git bridge", and only under that opt-in. Reasons (1) to (3) do not apply: +there is no Git rendering of sc history (the store holds opaque values, so +there is no incoherent Git structure to write), no marks map is involved (so it +gains no second meaning), and publish is an ordinary push to the same store (so +it cannot go stale against an exported artifact). Reason (4), a one-command +path to durably publishing embargo ciphertext on a public host, does apply, and +is why it is gated and why the notice and the refusal message say what stays +behind. + +### Deferred in v1 + +- `sc remote compact` on an `sc+git::` remote is refused. A Git repository gives + the compactor no trustworthy age for a stored pack: committer time is + writer-controlled and predates publication, so it cannot drive pruning. + `list_meta` exists for the trait contract and reports the observation time as + `modified`. +- `sc serve --store sc+git::...` is refused. A daemon cannot answer ssh or + credential prompts, and it would sit between clients and the client-side + private-branch gate. + +### sc-forge + +The forge accepts exactly the `refs/sc/` prefix for create, update (including +non-fast-forward) and delete, for every repository, with no per-repository +switch. The details and tests are in the ADR-0058 addendum and +`docs/FORGE-OPERATIONS.md`. Stock `git receive-pack` and sc-forge are verified +to check the sent old value under the ref lock; a host that does not has no +atomic CAS and cannot carry this store safely. + +### Dependency rule + +`objio` stays a leaf, but it may spawn the system `git` binary for `sc+git::`. +`gix` stays quarantined in `gitio`. The system `git` binary is spawned only by +`gitio::bridge` and by objio's git backend. `GitBucket` does not live in `gitio` +because `repo` must not depend on `gitio`, and `repo` chooses the bucket backend +(clone, fetch, push); injecting it from `cli` would thread a factory through +every transport path. + +## Consequences + +- A user can run `sc remote add origin sc+git::git@github.com:org/repo.git` and + push, fetch and clone private branches, protected paths and secrets through a + Git repository. A keyless clone sees what a keyless clone of an `sc+wal://` + remote sees. Proven by `crates/cli/tests/git_bucket.rs`, the forge end-to-end + tests, and `demo/run_git_carrier_demo.sh`. Not tested against real GitHub + (outside this work's grant); expect per-ref and per-push limits and any host + that refuses unknown ref namespaces to be the failure modes. +- **The writer set is wider than S3.** Everyone with push access to the Git + repository (collaborators, deploy keys, CI tokens) holds bucket put and + delete, and there is no object versioning to recover. They can rewind + `refs/sc/manifest` to a pre-revoke manifest, delete keys, or add packs. Branch + protection and rulesets do not cover `refs/sc/*`. This is the same class as S3 + write access, with more holders. Rollback and rewind stay the tracked + follow-ons (ROADMAP: P34 follow-on 6, a fetch-time freshness attestation; + ADR-0059 follow-on 2). +- **Deleting a ref is not erasure.** A host may keep the objects fetchable by + id, forks and mirrors keep their own copies, and compaction (when it exists + for Git) only unreferences values. Sealed content stays ciphertext, so + rotation is the cutover (`sc rewrap --rotate`), as for any transport. +- **Public content is plaintext in packs** on the host, as on any bucket remote. +- **Sealed metadata is readable by every reader of the repository:** private + branch names, recipient ids, sealed-object counts and sizes, and the public + fork point. Any one of them can fetch `refs/sc/*` by exact refspec, and + `git ls-remote` lists the refs to everyone. +- A killed process (SIGKILL, or SIGINT, which skips `Drop`) can leave + `sc-git-bucket--` behind in the temp directory. It holds fetched + store objects. There is no reaper. A normal exit and an error returned to `main` remove it. +- Each cold read or push costs git round trips. The freshness window and family + batching bound them, but a Git host is slower than S3. If push over `sc+git::` + stays slow on large histories, the next step is a batched `has_objects`. +- sc-forge repositories keep a deleted or replaced value fetchable by id until + the gc prune stage runs. +- The "not independently audited, do not commit production secrets" warning is + unchanged. + +## Alternatives considered + +- **The Git bridge.** Lossy: private branches are refused, secrets dropped, + protected paths lose their rules. Rejected; it stays the way to publish public + history as a plain Git branch, and the same repository can host both. +- **One carrier ref holding the whole bucket tree** (Astra's alternative). One + commit per bucket state gives coherent snapshots, but it serializes every pack + upload behind one ref compare-and-swap and moves the transaction above the + `Bucket` trait, so `BucketTransport` would need a second write path. Rejected. +- **A forge-native sc protocol** (sc-forge speaking `sc+http`). Works only where + we run the server, not on GitHub or any other Git host. Rejected for v1. +- **`GitBucket` in `gitio`.** `repo` must not depend on `gitio`, and the backend + is chosen inside `repo`. Rejected; see the dependency rule above. +- **`gix` instead of the system binary.** `gix` can fetch but not push (ADR-0028), + and the user's credential helper and ssh setup only work through `git`. + Rejected. + +## As built + +`crates/objio/src/git.rs` (`GitBucket`, the contract suite and race tests); +`crates/repo/src/bucket_transport.rs` (`BucketScheme::Git`, `is_bucket_url`, +`check_private_push_allowed`, `refuse_git_store`, `refuse_git_compact`); +`crates/repo/src/remote.rs` (`allow_private_branches`); `crates/cli` (flags, +notice, URL routing); `crates/forge/src/policy.rs` (`refs/sc/`). Tests: +`crates/objio/src/git.rs`, `crates/repo/src/bucket_transport/git_remote_tests.rs`, +`crates/cli/tests/git_bucket.rs`, `crates/forge/tests/e2e_git.rs`; +proof: `demo/run_git_carrier_demo.sh`. diff --git a/docs/adr/README.md b/docs/adr/README.md index 1845de69..48299b80 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -30,7 +30,7 @@ supersedes the old one and update the **Status** line of the superseded record. | [0016](0016-git-export.md) | Git export for round-trip interop | Accepted | 9 | | [0018](0018-git-as-a-remote.md) | Git as a remote (bidirectional sync) | Accepted | 10 | | [0019](0019-secret-lifecycle.md) | Secret/permission lifecycle (rotation + escrow) | Accepted; per-path rotation added by 0051 | 11 | -| [0020](0020-six-crate-workspace.md) | Six-crate workspace with a strict dependency direction | Accepted; crate set extended by 0042, 0045, 0046 | foundation | +| [0020](0020-six-crate-workspace.md) | Six-crate workspace with a strict dependency direction | Accepted; crate set extended by 0042, 0045, 0046; `objio` may spawn the system `git` (0060) | foundation | | [0021](0021-durability-and-concurrency-hardening.md) | Durability and concurrency hardening for `.sc/` | Accepted | hardening | | [0022](0022-ssh-native-transport.md) | sc-native network transport over SSH (trait-mirror wire protocol) | Accepted | 12 | | [0023](0023-agent-workspaces.md) | Agent workspaces — vfs-backed sessions over the persistent store | Accepted | 13 | @@ -54,9 +54,9 @@ supersedes the old one and update the **Status** line of the superseded record. | [0041](0041-listener-resource-limits.md) | Listener resource limits (connection cap, session timeout, spool cap, accept backoff) | Accepted | 31 | | [0042](0042-in-binary-tls-sc-https.md) | In-binary TLS — `sc+https://` via rustls | Accepted; CA validation 0054, pin management 0053 | 32 | | [0043](0043-randomized-protected-encryption.md) | Randomized protected-path encryption (dual-read, randomized-write) | Accepted; rotate-for-paths built as 0051 | 33 | -| [0044](0044-per-branch-access-control.md) | Per-branch access control with opaque private branches | Accepted; sparse merge-in section (#240) | 34 | +| [0044](0044-per-branch-access-control.md) | Per-branch access control with opaque private branches | Accepted; sparse merge-in section (#240); reason 4 against opaque refs through Git reversed under an opt-in by 0060 | 34 | | [0045](0045-native-desktop-read-model.md) | Native desktop adapter and keyless read model | Accepted | 35 | -| [0046](0046-wal-bucket-remotes.md) | Bucket remotes via an immutable-pack WAL + one CAS'd manifest | Accepted; compaction built as 0057 | 36 | +| [0046](0046-wal-bucket-remotes.md) | Bucket remotes via an immutable-pack WAL + one CAS'd manifest | Accepted; compaction built as 0057; Git-carried store as 0060 | 36 | | [0047](0047-want-by-id-fetch-and-gap-fill.md) | Want-by-id fetch and explicit gap fill for partial clones | Accepted (built) | 37 | | [0048](0048-presence-based-partial-clones.md) | Presence-based walks for partial clones | Accepted (built); 0055 adds a second kind of gap | 37b | | [0049](0049-merge-on-partial-clones.md) | Merge on partial clones | Accepted (built) | 37c | @@ -70,6 +70,7 @@ supersedes the old one and update the **Status** line of the superseded record. | [0057](0057-bucket-compaction.md) | Bucket remote compaction and gc (`sc remote compact`) | Accepted | 36 | | [0058](0058-sc-forge-gitea-compatible-git-forge.md) | sc-forge: a Gitea-compatible Git forge (bare Git, recorded API subset) | Proposed | 38 | | [0059](0059-locally-accepted-recipients.md) | Locally accepted recipients: a seal wraps only to keys this clone accepted (`sc policy`) | Accepted | hardening | +| [0060](0060-git-carried-bucket-remotes.md) | Git-carried bucket remotes (`sc+git::`): the bucket WAL stored as `refs/sc/*` in any Git repository | Accepted | 39 | See [`ROADMAP.md`](../../ROADMAP.md) for the phase sequence. ADR numbers are assigned in creation order, so 0017 — the secret scanner — sequences as phase P5, From 10461d954bad5e26554d76d732d91f0fca7011d3 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Sat, 21 Nov 2026 11:42:09 +0200 Subject: [PATCH 07/15] demo: sc+git:: carrier demo Self-checking proof for P39: a bare Git repo and a real sc-forge carry a protected path, a secret and a private branch as refs/sc/*. Covers the private-branch opt-in gate, a plain git clone that sees no sc store, keyless and keyed sc clones, the Git bridge mirroring public main in the same repo, and zero sc-git-bucket-* residue across two runs. --- demo/run_git_carrier_demo.sh | 268 +++++++++++++++++++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100755 demo/run_git_carrier_demo.sh diff --git a/demo/run_git_carrier_demo.sh b/demo/run_git_carrier_demo.sh new file mode 100755 index 00000000..642c8955 --- /dev/null +++ b/demo/run_git_carrier_demo.sh @@ -0,0 +1,268 @@ +#!/usr/bin/env bash +# P39 demo: `sc+git::` remotes (ADR-0060). A Git repository carries a +# native sc store as `refs/sc/*`, so private branches, protected paths and +# secrets travel through a Git host, which the Git bridge cannot do (it refuses +# sealed content). Run first against a bare Git repo on disk (file://), then +# against a real sc-forge on loopback HTTP. Proves end-to-end: +# +# 1. BRIDGE LIMIT: public `main` mirrors into the repository as an ordinary +# Git branch through a `--git` remote, and the same bridge refuses the +# branch once it holds a protected path and a secret. +# 2. GATE: pushing a private branch to an `sc+git::` remote is refused before +# any upload (nothing changes under refs/sc/*) until the remote is added +# with --allow-private-branches; protected paths and secrets need no opt-in. +# 3. CARRIER: `git ls-remote` lists refs/sc/*; a plain `git clone` of the same +# repository fetches none of them and holds no sc objects. +# 4. KEYLESS: an sc clone sees `hotfix (private, no access)`, no protected +# plaintext, no embargoed file, and cannot read the secret value. +# 5. KEYED: an sc clone with alice's identity reads the protected file, the +# secret and the embargoed branch. +# 6. Zero residue: no leftover `sc-git-bucket-*` temp dirs (TMPDIR is private +# to each run, so any leftover is ours) and no .sc/tmp residue. +# +# Self-checking: every claim is an assertion; any failure exits non-zero before +# the RESULT line. Run twice to prove repeatability. The forge leg needs +# `sc-forge` (unix only), curl and python3 (for a free port); SKIP_FORGE=1 +# skips it. Auth to the forge goes through a temporary GIT_CONFIG_GLOBAL with +# `http.extraHeader` (a password in the URL is refused by sc+git::). +set -euo pipefail + +cd "$(dirname "$0")/.." +cargo build --quiet --bin sc +TARGET="${CARGO_TARGET_DIR:-$PWD/target}" +SC="$TARGET/debug/sc" +FORGE="" +if [ "${SKIP_FORGE:-0}" != "1" ]; then + if cargo build --quiet --bin sc-forge 2>/dev/null; then + FORGE="$TARGET/debug/sc-forge" + else + echo "note: sc-forge did not build (unix only); skipping the forge leg" + fi +fi + +tmp_base="${TMPDIR:-/tmp}" +tmp_base="${tmp_base%/}" + +fail() { echo "FAIL: $1"; exit 1; } + +WORK="" +FORGE_PID="" +cleanup() { + if [ -n "$FORGE_PID" ]; then + kill "$FORGE_PID" 2>/dev/null || true + wait "$FORGE_PID" 2>/dev/null || true + FORGE_PID="" + fi + if [ -n "$WORK" ]; then rm -rf "$WORK"; WORK=""; fi +} +trap cleanup EXIT + +SENT_FILE="PROD-ENV-SENTINEL-4c1e" +SENT_EMBARGO="EMBARGOED-FIX-SENTINEL-4c1e" +SENT_MSG="EMBARGOED-MESSAGE-SENTINEL-4c1e" +SENT_SECRET="SECRET-VALUE-SENTINEL-4c1e" + +# No sc-git-bucket-* directory may outlive the sc process that made it. +no_cache_residue() { + if ls "$WORK/tmp" 2>/dev/null | grep -q '^sc-git-bucket-'; then + fail "leftover sc-git-bucket-* temp dir in $WORK/tmp" + fi +} + +# scenario