diff --git a/README.md b/README.md index d1649ae5..9f3c6f6d 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ git.agentic/ │ │ ├── roadmap.md 12-week build narrative; repo went public 2026-05-22 │ │ └── demo-scenario.md the canonical demo │ ├── adr/ -│ │ └── 0001 … 0017 — accepted architectural decision records +│ │ └── 0001 … 0018 — accepted architectural decision records │ └── architecture/ │ ├── overview.md system diagram and component boundaries │ └── snapshot-model.md the technical heart diff --git a/crates/agentic-core/src/gcs_store.rs b/crates/agentic-core/src/gcs_store.rs index 4496651b..b264513b 100644 --- a/crates/agentic-core/src/gcs_store.rs +++ b/crates/agentic-core/src/gcs_store.rs @@ -44,7 +44,10 @@ use std::time::Duration; use crate::hash::Hash; use crate::object::{Object, ObjectKind}; use crate::scanner::{Allowlist, ScanPolicy, Scanner}; -use crate::store::{check_integrity, ObjectStore}; +use crate::store::{ + check_blob_unambiguous, check_integrity, classify_raw, classify_typed, decode_legacy_raw, + decode_typed, ObjectStore, Occupant, +}; use crate::{Error, Result}; /// Default upstream — GCS's public JSON API host. @@ -60,6 +63,39 @@ const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); /// tier. const STORAGE_CLASS: &str = "STANDARD"; +/// Which address namespace an object lives in. Typed objects are JSON +/// envelopes; raw objects are bare bytes. They must never share a name. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Ns { + Typed, + Raw, +} + +impl Ns { + /// (directory prefix, file extension) of this namespace's objects. + fn layout(self) -> (&'static str, &'static str) { + match self { + Ns::Typed => ("", "zst"), + Ns::Raw => ("raw/", "raw.zst"), + } + } +} + +/// What a conditional upload did. +#[derive(Debug, PartialEq, Eq)] +enum Upload { + Stored, + /// HTTP 412: the object exists (or, for a generation match, changed). + PreconditionFailed, +} + +/// A downloaded object: the stored (compressed) bytes and the GCS +/// generation they were read at, when the server reported one. +struct Downloaded { + compressed: Vec, + generation: Option, +} + #[derive(Debug, Clone)] pub struct GcsObjectStore { bucket: String, @@ -111,24 +147,30 @@ impl GcsObjectStore { /// GCS object name for a given hash. Sharded by the first two hex /// chars (matches the on-disk layout) so a ``gsutil ls gs://b/

`` - /// looks familiar. - fn object_name(&self, hash: &Hash) -> String { + /// looks familiar. Raw objects (`put_raw`) live under `raw/` with a + /// distinct `.raw.zst` suffix, so they can never share a name with a + /// typed object, even for a store whose prefix is `

/raw`. + fn object_name(&self, hash: &Hash, ns: Ns) -> String { let (a, b) = hash.shard(); let p = self.prefix.trim_matches('/'); + let (dir, ext) = ns.layout(); if p.is_empty() { - format!("{a}/{b}.zst") + format!("{dir}{a}/{b}.{ext}") } else { - format!("{p}/{a}/{b}.zst") + format!("{p}/{dir}{a}/{b}.{ext}") } } - fn cache_path(&self, hash: &Hash) -> PathBuf { + fn cache_path(&self, hash: &Hash, ns: Ns) -> PathBuf { let (a, b) = hash.shard(); - self.cache_dir.join(a).join(format!("{b}.zst")) + let (dir, ext) = ns.layout(); + self.cache_dir + .join(format!("{dir}{a}")) + .join(format!("{b}.{ext}")) } - fn cache_read(&self, hash: &Hash) -> Option> { - let path = self.cache_path(hash); + fn cache_read(&self, hash: &Hash, ns: Ns) -> Option> { + let path = self.cache_path(hash, ns); let compressed = match fs::read(&path) { Ok(bytes) => bytes, Err(e) if e.kind() == std::io::ErrorKind::NotFound => return None, @@ -145,8 +187,8 @@ impl GcsObjectStore { zstd::stream::decode_all(&compressed[..]).ok() } - fn cache_write_compressed(&self, hash: &Hash, compressed: &[u8]) -> Result<()> { - let path = self.cache_path(hash); + fn cache_write_compressed(&self, hash: &Hash, ns: Ns, compressed: &[u8]) -> Result<()> { + let path = self.cache_path(hash, ns); if let Some(parent) = path.parent() { fs::create_dir_all(parent)?; } @@ -160,12 +202,15 @@ impl GcsObjectStore { // ---------- HTTP plumbing ---------------------------------------------- - fn upload_url(&self, name: &str) -> String { + /// `if_generation` is the `ifGenerationMatch` precondition: `0` means + /// "only if the object does not exist yet". + fn upload_url(&self, name: &str, if_generation: &str) -> String { let encoded = urlencode(name); format!( - "{}/upload/storage/v1/b/{}/o?uploadType=media&name={}", + "{}/upload/storage/v1/b/{}/o?uploadType=media&ifGenerationMatch={}&name={}", self.endpoint.trim_end_matches('/'), self.bucket, + if_generation, encoded, ) } @@ -205,9 +250,18 @@ impl GcsObjectStore { Error::Other(anyhow::anyhow!("{context}: HTTP {status}: {body}")) } - fn upload_compressed(&self, hash: &Hash, compressed: &[u8]) -> Result<()> { - let name = self.object_name(hash); - let url = self.upload_url(&name); + /// Conditional upload (`ifGenerationMatch=`): an existing + /// object is never overwritten blindly. HTTP 412 is reported to the + /// caller, which must decide what the existing object means. + fn upload_compressed( + &self, + hash: &Hash, + ns: Ns, + compressed: &[u8], + if_generation: &str, + ) -> Result { + let name = self.object_name(hash, ns); + let url = self.upload_url(&name, if_generation); let resp = self .with_auth( self.client @@ -218,19 +272,22 @@ impl GcsObjectStore { .body(compressed.to_vec()) .send() .map_err(|e| Error::Other(anyhow::anyhow!("upload {url}: {e}")))?; + if resp.status() == reqwest::StatusCode::PRECONDITION_FAILED { + return Ok(Upload::PreconditionFailed); + } if !resp.status().is_success() { let status = resp.status(); let body = resp.text().unwrap_or_default(); return Err(Self::http_err("uploading object", status, &body)); } - Ok(()) + Ok(Upload::Stored) } - /// Returns ``Some((decompressed_bytes, raw_compressed_bytes))`` if - /// the object exists in GCS. The raw bytes are returned so the - /// caller can populate the local cache without re-compressing. - fn download_compressed(&self, hash: &Hash) -> Result, Vec)>> { - let name = self.object_name(hash); + /// Returns the stored object if it exists in GCS. The compressed bytes + /// are returned as stored so the caller can populate the local cache + /// without re-compressing; decoding is left to the caller. + fn download_compressed(&self, hash: &Hash, ns: Ns) -> Result> { + let name = self.object_name(hash, ns); let url = self.download_url(&name); let resp = self .with_auth(self.client.get(&url)) @@ -244,12 +301,19 @@ impl GcsObjectStore { let body = resp.text().unwrap_or_default(); return Err(Self::http_err("downloading object", status, &body)); } + let generation = resp + .headers() + .get("x-goog-generation") + .and_then(|v| v.to_str().ok()) + .map(str::to_owned); let compressed = resp .bytes() .map_err(|e| Error::Other(anyhow::anyhow!("reading body: {e}")))? .to_vec(); - let bytes = zstd::stream::decode_all(&compressed[..])?; - Ok(Some((bytes, compressed))) + Ok(Some(Downloaded { + compressed, + generation, + })) } /// Fetch the bytes for `hash` and map them to `T`, where `map` also @@ -271,39 +335,60 @@ impl GcsObjectStore { /// * A **corrupt download** is rejected *before* `cache_write`, so a /// bad object is never written to the local cache ("never cache a /// failed verification"). - fn fetch_map(&self, hash: &Hash, map: impl Fn(Vec) -> Result) -> Result { - let cache_path = self.cache_path(hash); - if cache_path.exists() { - match self.cache_read(hash) { - Some(cached) => match map(cached) { - Ok(value) => return Ok(value), - Err(err) => { - tracing::warn!( - target: "agentic-core::gcs_store", - hash = %hash.to_hex(), - error = %err, - "cached object failed integrity check; evicting and refetching from GCS", - ); - // Evict the poisoned entry so `has`/subsequent reads - // don't keep trusting it. - self.evict_cache_entry(hash, &cache_path); - } - }, - None => { - // Cache file exists but is unreadable/corrupt (e.g. torn - // write or zstd decode failure). Evict so `has` stays - // consistent and future reads can heal from GCS. + fn fetch_map(&self, hash: &Hash, ns: Ns, map: impl Fn(Vec) -> Result) -> Result { + if let Some(value) = self.try_cache(hash, ns, &map) { + return Ok(value); + } + self.fetch_remote(hash, ns, &map) + } + + /// The cache half of [`Self::fetch_map`]: a hit that passes `map`, or + /// `None` (evicting a poisoned or unreadable entry on the way). + fn try_cache(&self, hash: &Hash, ns: Ns, map: &impl Fn(Vec) -> Result) -> Option { + let cache_path = self.cache_path(hash, ns); + if !cache_path.exists() { + return None; + } + match self.cache_read(hash, ns) { + Some(cached) => match map(cached) { + Ok(value) => return Some(value), + Err(err) => { + tracing::warn!( + target: "agentic-core::gcs_store", + hash = %hash.to_hex(), + error = %err, + "cached object failed integrity check; evicting and refetching from GCS", + ); + // Evict the poisoned entry so `has`/subsequent reads + // don't keep trusting it. self.evict_cache_entry(hash, &cache_path); } + }, + None => { + // Cache file exists but is unreadable/corrupt (e.g. torn + // write or zstd decode failure). Evict so `has` stays + // consistent and future reads can heal from GCS. + self.evict_cache_entry(hash, &cache_path); } } - match self.download_compressed(hash)? { + None + } + + /// The network half of [`Self::fetch_map`]. + fn fetch_remote( + &self, + hash: &Hash, + ns: Ns, + map: &impl Fn(Vec) -> Result, + ) -> Result { + match self.download_compressed(hash, ns)? { None => Err(Error::NotFound(*hash)), - Some((bytes, compressed)) => { + Some(Downloaded { compressed, .. }) => { + let bytes = zstd::stream::decode_all(&compressed[..])?; // Map (which verifies) BEFORE caching: a corrupt GCS object // must not be persisted to the local cache. let value = map(bytes)?; - let _ = self.cache_write_compressed(hash, &compressed); + let _ = self.cache_write_compressed(hash, ns, &compressed); Ok(value) } } @@ -336,8 +421,8 @@ impl GcsObjectStore { } } - fn remote_exists(&self, hash: &Hash) -> bool { - let name = self.object_name(hash); + fn remote_exists(&self, hash: &Hash, ns: Ns) -> bool { + let name = self.object_name(hash, ns); let url = self.metadata_url(&name); // HEAD against the metadata endpoint is the cheapest existence // check the JSON API offers. @@ -346,6 +431,75 @@ impl GcsObjectStore { } } +impl GcsObjectStore { + /// Put `compressed` at `hash`, never replacing a *valid* object. + /// + /// The upload is conditional on the name being free. On HTTP 412 the + /// occupant is downloaded and judged by `classify` (exactly as the + /// matching read would judge it). The same content is a dedup hit; a + /// valid object of another kind is refused with + /// [`Error::AddressOccupied`], writing nothing. Only a corrupt occupant + /// (a pre-fix squatter or damage) is replaced, conditional on the + /// generation the download reported; if it reported none, or the + /// object changed meanwhile, the put fails rather than succeeding. The + /// local cache is written only once the bucket is known to hold the + /// verified object, so a failed or refused put never leaves a cache + /// entry. + /// + /// Network cost: 1 request for a new object; 2 (upload, download) for a + /// dedup hit or a refusal; 3 when a corrupt occupant is replaced. + fn store_compressed( + &self, + hash: &Hash, + ns: Ns, + compressed: &[u8], + classify: impl FnOnce(Vec) -> Occupant, + ) -> Result<()> { + if self.upload_compressed(hash, ns, compressed, "0")? == Upload::PreconditionFailed { + let existing = self.download_compressed(hash, ns)?.ok_or_else(|| { + Error::Other(anyhow::anyhow!("object {hash} vanished during put")) + })?; + let occupant = match zstd::stream::decode_all(&existing.compressed[..]) { + Ok(stored) => classify(stored), + Err(_) => Occupant::Corrupt, + }; + match occupant { + Occupant::Same => {} + Occupant::Different => return Err(Error::AddressOccupied { hash: *hash }), + Occupant::Corrupt => { + let generation = existing.generation.ok_or_else(|| { + Error::Other(anyhow::anyhow!( + "cannot replace corrupt object {hash}: download carried no \ + x-goog-generation" + )) + })?; + tracing::warn!( + target: "agentic-core::gcs_store", + hash = %hash, + "replacing corrupt object at content address", + ); + if self.upload_compressed(hash, ns, compressed, &generation)? + == Upload::PreconditionFailed + { + return Err(Error::Other(anyhow::anyhow!( + "object {hash} changed while replacing a corrupt occupant" + ))); + } + } + } + } + if let Err(e) = self.cache_write_compressed(hash, ns, compressed) { + tracing::warn!( + target: "agentic-core::gcs_store", + hash = %hash, + error = %e, + "failed to write local cache entry after GCS upload", + ); + } + Ok(()) + } +} + impl ObjectStore for GcsObjectStore { fn put_with_policy(&self, object: &Object, policy: ScanPolicy) -> Result { // Scanner pre-hook (ADR-0013). Reject blobs containing secrets @@ -363,31 +517,16 @@ impl ObjectStore for GcsObjectStore { } } + if let Object::Blob(blob) = object { + check_blob_unambiguous(&blob.bytes)?; + } + let bytes = serde_json::to_vec(object)?; let hash = object.hash(); let compressed = zstd::stream::encode_all(&bytes[..], 3)?; - // Cache locally first so a concurrent reader that lands during - // the upload doesn't miss; on upload failure the cache entry - // gets blown away. (The cache is a hint, not a source of - // truth — see post-MVP `agentic gc` for sweeping.) - self.cache_write_compressed(&hash, &compressed)?; - if let Err(e) = self.upload_compressed(&hash, &compressed) { - // Best-effort cache rollback so a failed upload can't masquerade as - // a successful put on the next get/has. If the unlink itself fails - // (permissions, transient FS error) we surface it via tracing so an - // operator can investigate — but still return the upload error, - // which is the user-visible cause. - if let Err(unlink_err) = fs::remove_file(self.cache_path(&hash)) { - if unlink_err.kind() != std::io::ErrorKind::NotFound { - tracing::warn!( - hash = %hash, - error = %unlink_err, - "failed to roll back cache entry after GCS upload error", - ); - } - } - return Err(e); - } + self.store_compressed(&hash, Ns::Typed, &compressed, |stored| { + classify_typed(&hash, stored, object) + })?; Ok(hash) } @@ -405,52 +544,51 @@ impl ObjectStore for GcsObjectStore { let hash = Hash::of(bytes); let compressed = zstd::stream::encode_all(bytes, 3)?; - self.cache_write_compressed(&hash, &compressed)?; - if let Err(e) = self.upload_compressed(&hash, &compressed) { - // Best-effort cache rollback so a failed upload can't masquerade as - // a successful put on the next get/has. If the unlink itself fails - // (permissions, transient FS error) we surface it via tracing so an - // operator can investigate — but still return the upload error, - // which is the user-visible cause. - if let Err(unlink_err) = fs::remove_file(self.cache_path(&hash)) { - if unlink_err.kind() != std::io::ErrorKind::NotFound { - tracing::warn!( - hash = %hash, - error = %unlink_err, - "failed to roll back cache entry after GCS upload error", - ); - } - } - return Err(e); - } + self.store_compressed(&hash, Ns::Raw, &compressed, |stored| { + classify_raw(&hash, &stored) + })?; Ok(hash) } fn get(&self, hash: &Hash) -> Result { // Typed objects are addressed by `object.hash()`. Parse once inside // the map closure and verify that hash — no second deserialize. - self.fetch_map(hash, |bytes| { - let object: Object = serde_json::from_slice(&bytes)?; - check_integrity(hash, object.hash())?; - Ok(object) - }) + self.fetch_map(hash, Ns::Typed, |bytes| decode_typed(hash, bytes)) } fn get_raw(&self, hash: &Hash) -> Result> { // Raw objects are addressed by `Hash::of(bytes)` (the `put_raw` // contract) — verify against that, then return the bytes // unchanged (zero-copy: `bytes` moves out). Audit finding #3. - self.fetch_map(hash, |bytes| { + let verify_raw = |bytes: Vec| { check_integrity(hash, Hash::of(&bytes))?; Ok(bytes) - }) + }; + // Legacy layout: raw objects used to share the typed name (and may + // hold a blob envelope of the same bytes). + let verify_legacy = |bytes: Vec| decode_legacy_raw(hash, bytes); + // Hash-verified cache entries first, under both names, so a cached + // snapshot restores without touching the network. + if let Some(bytes) = self.try_cache(hash, Ns::Raw, &verify_raw) { + return Ok(bytes); + } + if let Some(bytes) = self.try_cache(hash, Ns::Typed, &verify_legacy) { + return Ok(bytes); + } + match self.fetch_remote(hash, Ns::Raw, &verify_raw) { + Err(Error::NotFound(_)) => self.fetch_remote(hash, Ns::Typed, &verify_legacy), + other => other, + } } fn has(&self, hash: &Hash) -> bool { - if self.cache_path(hash).exists() { - return true; - } - self.remote_exists(hash) + // Both cache namespaces first, so a cached object answers without + // any network request; only then ask the bucket. + let namespaces = [Ns::Raw, Ns::Typed]; + namespaces + .iter() + .any(|&ns| self.cache_path(hash, ns).exists()) + || namespaces.iter().any(|&ns| self.remote_exists(hash, ns)) } } @@ -494,7 +632,7 @@ mod tests { ) .unwrap(); let h = Hash::of(b"hello"); - let name = store.object_name(&h); + let name = store.object_name(&h, Ns::Typed); let hex = h.to_hex(); assert!(name.starts_with("repo-A/")); assert!(name.contains(&format!("{}/", &hex[..2]))); @@ -514,7 +652,7 @@ mod tests { ) .unwrap(); let h = Hash::of(b"hello"); - let name = store.object_name(&h); + let name = store.object_name(&h, Ns::Typed); assert!(!name.starts_with('/')); let hex = h.to_hex(); assert!(name.starts_with(&format!("{}/", &hex[..2]))); @@ -540,7 +678,7 @@ mod tests { ) .unwrap(); let h = Hash::of(b"nope"); - assert!(store.cache_read(&h).is_none()); + assert!(store.cache_read(&h, Ns::Typed).is_none()); } #[test] @@ -557,8 +695,10 @@ mod tests { let h = Hash::of(b"hello"); let payload = b"hello-world"; let compressed = zstd::stream::encode_all(&payload[..], 3).unwrap(); - store.cache_write_compressed(&h, &compressed).unwrap(); - let got = store.cache_read(&h).unwrap(); + store + .cache_write_compressed(&h, Ns::Typed, &compressed) + .unwrap(); + let got = store.cache_read(&h, Ns::Typed).unwrap(); assert_eq!(got, payload); } @@ -591,8 +731,10 @@ mod tests { let good = b"the canonical bytes"; let hash = Hash::of(good); let poison = zstd::stream::encode_all(&b"tampered!!"[..], 3).unwrap(); - store.cache_write_compressed(&hash, &poison).unwrap(); - assert!(store.cache_path(&hash).exists()); + store + .cache_write_compressed(&hash, Ns::Raw, &poison) + .unwrap(); + assert!(store.cache_path(&hash, Ns::Raw).exists()); let result = store.get_raw(&hash); // Never returns the poison. (The fall-through GCS fetch errors @@ -607,7 +749,7 @@ mod tests { } assert!(result.is_err(), "no server to heal from, so read errors"); assert!( - !store.cache_path(&hash).exists(), + !store.cache_path(&hash, Ns::Raw).exists(), "poisoned cache entry must be evicted" ); } @@ -633,7 +775,7 @@ mod tests { // Corrupt the cache slot into a directory (unreadable as a file, so // cache_read returns None → the eviction path runs). let hash = Hash::of(b"anything"); - let cache_path = store.cache_path(&hash); + let cache_path = store.cache_path(&hash, Ns::Raw); std::fs::create_dir_all(&cache_path).unwrap(); assert!(cache_path.is_dir()); @@ -643,4 +785,326 @@ mod tests { "a cache entry corrupted into a directory must still be evicted" ); } + + // ---------- mock GCS server ------------------------------------------- + + #[derive(Clone, Debug)] + struct Req { + method: String, + target: String, + } + + struct Resp { + status: u16, + headers: Vec<(&'static str, String)>, + body: Vec, + } + + impl Resp { + fn status(status: u16) -> Self { + Resp { + status, + headers: Vec::new(), + body: Vec::new(), + } + } + } + + /// Minimal HTTP/1.1 server (one request per connection) driven by a + /// handler closure; records every request it sees. + struct Mock { + addr: std::net::SocketAddr, + log: Arc>>, + stop: Arc, + } + + impl Mock { + fn start(handler: impl Fn(&Req) -> Resp + Send + 'static) -> Mock { + use std::io::{BufRead, BufReader, Read, Write}; + use std::sync::atomic::Ordering; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let log: Arc>> = Arc::default(); + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let (log2, stop2) = (log.clone(), stop.clone()); + std::thread::spawn(move || { + for conn in listener.incoming() { + if stop2.load(Ordering::SeqCst) { + return; + } + let Ok(mut sock) = conn else { continue }; + let mut reader = BufReader::new(sock.try_clone().unwrap()); + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 { + continue; + } + let mut parts = line.split_whitespace(); + let req = Req { + method: parts.next().unwrap_or_default().to_string(), + target: parts.next().unwrap_or_default().to_string(), + }; + let mut content_length = 0usize; + loop { + let mut h = String::new(); + reader.read_line(&mut h).unwrap(); + if h == "\r\n" { + break; + } + if let Some(v) = h.to_ascii_lowercase().strip_prefix("content-length:") { + content_length = v.trim().parse().unwrap(); + } + } + let mut body = vec![0u8; content_length]; + reader.read_exact(&mut body).unwrap(); + log2.lock().unwrap().push(req.clone()); + let resp = handler(&req); + let mut out = format!( + "HTTP/1.1 {} X\r\nContent-Length: {}\r\nConnection: close\r\n", + resp.status, + resp.body.len() + ); + for (k, v) in &resp.headers { + out.push_str(&format!("{k}: {v}\r\n")); + } + out.push_str("\r\n"); + let _ = sock.write_all(out.as_bytes()); + let _ = sock.write_all(&resp.body); + } + }); + Mock { addr, log, stop } + } + + fn store(&self, cache: &Path) -> GcsObjectStore { + GcsObjectStore::new("b", "p", cache, Some(format!("http://{}", self.addr)), None) + .unwrap() + } + + fn requests(&self) -> Vec { + self.log.lock().unwrap().clone() + } + } + + impl Drop for Mock { + fn drop(&mut self) { + self.stop.store(true, std::sync::atomic::Ordering::SeqCst); + let _ = std::net::TcpStream::connect(self.addr); + } + } + + fn zst(bytes: &[u8]) -> Vec { + zstd::stream::encode_all(bytes, 3).unwrap() + } + + /// A bucket whose first upload hits an occupied name (412), serving + /// `occupant` (generation 77). `replace_status` answers the follow-up + /// upload that carries `ifGenerationMatch=77`. + fn occupied_bucket(occupant: Vec, replace_status: u16) -> Mock { + Mock::start(move |req| { + if req.method == "POST" && req.target.contains("ifGenerationMatch=0") { + Resp::status(412) + } else if req.method == "POST" && req.target.contains("ifGenerationMatch=77") { + Resp::status(replace_status) + } else if req.method == "GET" && req.target.contains("alt=media") { + Resp { + status: 200, + headers: vec![("x-goog-generation", "77".into())], + body: occupant.clone(), + } + } else { + Resp::status(500) + } + }) + } + + fn is_post(r: &Req, needle: &str) -> bool { + r.method == "POST" && r.target.contains(needle) + } + + // Uploads are conditional on the name being free (fake-gcs-server + // ignores the precondition, so it is checked here), and a 412 over the + // *same* object is a dedup hit: verified by download, nothing replaced. + #[test] + fn upload_is_conditional_and_412_over_same_object_is_dedup() { + let dir = tempdir().unwrap(); + let payload = b"already there"; + let mock = occupied_bucket(zst(payload), 500); + let store = mock.store(dir.path()); + let h = store.put_raw(ObjectKind::Segment, payload).unwrap(); + + let reqs = mock.requests(); + assert_eq!(reqs.len(), 2, "dedup hit costs upload + download: {reqs:?}"); + assert!(is_post(&reqs[0], "ifGenerationMatch=0"), "{reqs:?}"); + assert!(reqs[0].target.contains("raw%2F"), "{reqs:?}"); + assert!(reqs[0].target.contains(".raw.zst"), "{reqs:?}"); + assert!(store.cache_path(&h, Ns::Raw).exists()); + } + + // A 412 over a *different* occupant (pre-fix squatter / corruption) is + // replaced, conditional on the generation observed, then cached. + #[test] + fn put_over_different_raw_occupant_replaces_it() { + let dir = tempdir().unwrap(); + let mock = occupied_bucket(zst(b"squatter"), 200); + let store = mock.store(dir.path()); + let h = store.put_raw(ObjectKind::Segment, b"honest").unwrap(); + + let reqs = mock.requests(); + assert_eq!(reqs.len(), 3, "{reqs:?}"); + assert!(is_post(&reqs[2], "ifGenerationMatch=77"), "{reqs:?}"); + assert!(store.cache_path(&h, Ns::Raw).exists()); + } + + #[test] + fn put_over_different_typed_occupant_replaces_it() { + let dir = tempdir().unwrap(); + let mock = occupied_bucket(zst(b"squatter"), 200); + let store = mock.store(dir.path()); + let obj = Object::Blob(crate::object::Blob::new(b"prompt".to_vec())); + let h = store.put(&obj).unwrap(); + + let reqs = mock.requests(); + assert!(is_post(&reqs[2], "ifGenerationMatch=77"), "{reqs:?}"); + assert!(store.cache_path(&h, Ns::Typed).exists()); + } + + // Losing the generation-conditional replace is an error, never success, + // and leaves no cache entry. + #[test] + fn second_412_while_replacing_is_an_error() { + let dir = tempdir().unwrap(); + let mock = occupied_bucket(zst(b"squatter"), 412); + let store = mock.store(dir.path()); + let h = Hash::of(b"honest"); + assert!(store.put_raw(ObjectKind::Segment, b"honest").is_err()); + assert!(!store.cache_path(&h, Ns::Raw).exists()); + } + + // A failed upload must not leave a cache entry behind. + #[test] + fn failed_upload_leaves_no_cache_entry() { + let dir = tempdir().unwrap(); + let mock = Mock::start(|_| Resp::status(500)); + let store = mock.store(dir.path()); + let h = Hash::of(b"will not upload"); + assert!(store + .put_raw(ObjectKind::Segment, b"will not upload") + .is_err()); + assert!(!store.cache_path(&h, Ns::Raw).exists()); + } + + // A hash-verified cached legacy object restores with no network at all. + #[test] + fn get_raw_of_cached_legacy_object_makes_no_request() { + let dir = tempdir().unwrap(); + let mock = Mock::start(|_| Resp::status(500)); + let store = mock.store(dir.path()); + let bytes = b"legacy segment".to_vec(); + let h = Hash::of(&bytes); + store + .cache_write_compressed(&h, Ns::Typed, &zst(&bytes)) + .unwrap(); + + assert_eq!(store.get_raw(&h).unwrap(), bytes); + assert!(mock.requests().is_empty(), "{:?}", mock.requests()); + } + + // A legacy name holding a blob envelope of the same bytes is a valid raw + // object, and its (valid typed) cache entry must not be evicted. + #[test] + fn get_raw_tolerates_cached_blob_envelope_without_evicting() { + let dir = tempdir().unwrap(); + let mock = Mock::start(|_| Resp::status(500)); + let store = mock.store(dir.path()); + let bytes = b"legacy segment".to_vec(); + let h = Hash::of(&bytes); + let envelope = + serde_json::to_vec(&Object::Blob(crate::object::Blob::new(bytes.clone()))).unwrap(); + store + .cache_write_compressed(&h, Ns::Typed, &zst(&envelope)) + .unwrap(); + + assert_eq!(store.get_raw(&h).unwrap(), bytes); + assert!(store.cache_path(&h, Ns::Typed).exists()); + assert!(mock.requests().is_empty()); + } + + // `

/raw` as a prefix must not collide with raw objects of prefix `

`. + #[test] + fn raw_names_cannot_collide_with_typed_names_of_a_raw_prefix() { + let dir = tempdir().unwrap(); + let h = Hash::of(b"x"); + let outer = GcsObjectStore::new("b", "p", dir.path(), None, None).unwrap(); + let inner = GcsObjectStore::new("b", "p/raw", dir.path(), None, None).unwrap(); + assert_ne!( + outer.object_name(&h, Ns::Raw), + inner.object_name(&h, Ns::Typed) + ); + } + + // A valid object of another kind (here a legacy Blob whose bytes are the + // canonical JSON of a Tree) is never replaced: the put is refused, and + // nothing is written or cached. + #[test] + fn put_over_valid_other_kind_occupant_is_refused() { + let dir = tempdir().unwrap(); + let tree = crate::object::Tree::new(); + let tree_json = serde_json::to_vec(&tree).unwrap(); + let envelope = + serde_json::to_vec(&Object::Blob(crate::object::Blob::new(tree_json))).unwrap(); + let mock = occupied_bucket(zst(&envelope), 200); + let store = mock.store(dir.path()); + + match store.put(&Object::Tree(tree.clone())) { + Err(Error::AddressOccupied { hash }) => assert_eq!(hash, tree.hash()), + other => panic!("expected AddressOccupied, got {other:?}"), + } + let reqs = mock.requests(); + assert_eq!(reqs.len(), 2, "upload + download only: {reqs:?}"); + assert!(!store.cache_path(&tree.hash(), Ns::Typed).exists()); + } + + // Replacing a corrupt occupant needs the generation from the download; + // without one the put fails, with no separate metadata lookup. + #[test] + fn replacing_without_generation_header_fails() { + let dir = tempdir().unwrap(); + let mock = Mock::start(|req| { + if req.method == "POST" && req.target.contains("ifGenerationMatch=0") { + Resp::status(412) + } else if req.method == "GET" && req.target.contains("alt=media") { + Resp { + status: 200, + headers: Vec::new(), + body: zst(b"squatter"), + } + } else { + Resp::status(500) + } + }); + let store = mock.store(dir.path()); + let h = Hash::of(b"honest"); + assert!(store.put_raw(ObjectKind::Segment, b"honest").is_err()); + assert_eq!(mock.requests().len(), 2, "{:?}", mock.requests()); + assert!(!store.cache_path(&h, Ns::Raw).exists()); + } + + // A cached object (either namespace) answers `has` with no request. + #[test] + fn has_checks_both_caches_before_the_network() { + let dir = tempdir().unwrap(); + let mock = Mock::start(|_| Resp::status(500)); + let store = mock.store(dir.path()); + let typed = Hash::of(b"typed"); + let raw = Hash::of(b"raw"); + store + .cache_write_compressed(&typed, Ns::Typed, &zst(b"t")) + .unwrap(); + store + .cache_write_compressed(&raw, Ns::Raw, &zst(b"r")) + .unwrap(); + + assert!(store.has(&typed)); + assert!(store.has(&raw)); + assert!(mock.requests().is_empty(), "{:?}", mock.requests()); + } } diff --git a/crates/agentic-core/src/lib.rs b/crates/agentic-core/src/lib.rs index 4e1d812a..a16f5f3c 100644 --- a/crates/agentic-core/src/lib.rs +++ b/crates/agentic-core/src/lib.rs @@ -26,7 +26,7 @@ pub use gcs_store::{GcsObjectStore, DEFAULT_GCS_ENDPOINT}; pub use hash::Hash; pub use object::{Attestation, Blob, Commit, Object, ObjectKind, Tree, TypedRef}; pub use refs::{HeadRef, Refs, RefsSnapshot}; -pub use store::{FsObjectStore, ObjectStore}; +pub use store::{check_blob_unambiguous, FsObjectStore, ObjectStore}; /// Crate-level error type. Individual modules contribute their own variants. #[derive(Debug, thiserror::Error)] @@ -52,6 +52,18 @@ pub enum Error { #[error("blob rejected by secret scanner: {hits:?}")] SecretDetected { hits: Vec }, + #[error( + "blob rejected: its bytes are the canonical encoding of a tree or commit \ + and would share address {hash} with that object" + )] + AmbiguousBlob { hash: Hash }, + + #[error( + "address {hash} is already occupied by a different valid object; \ + refusing to replace it" + )] + AddressOccupied { hash: Hash }, + #[error("invalid branch name {name:?}: {reason}")] InvalidBranchName { name: String, reason: String }, diff --git a/crates/agentic-core/src/store.rs b/crates/agentic-core/src/store.rs index fa3eb080..e3589ad4 100644 --- a/crates/agentic-core/src/store.rs +++ b/crates/agentic-core/src/store.rs @@ -4,6 +4,8 @@ //! - Objects are zstd-compressed on disk. //! - First two hex chars of the BLAKE3 hash form the shard directory. //! - Filename is the remaining 62 hex chars plus `.zst`. +//! - Raw objects (`put_raw`) live apart, at `raw//.raw`; see +//! `docs/adr/0018-separate-raw-object-namespace.md`. //! //! `FsObjectStore` is the local-filesystem implementation; `GcsObjectStore` //! (see `gcs.rs`) is the remote backend introduced for the sidecar @@ -11,13 +13,18 @@ //! where additional backends (e.g. S3, network-replicated) will plug in. use crate::hash::Hash; -use crate::object::{Object, ObjectKind}; +use crate::object::{Blob, Commit, Object, ObjectKind, Tree}; use crate::scanner::{Allowlist, ScanPolicy, Scanner}; use crate::{Error, Result}; +use std::io::Write; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; +/// Counter for [`FsObjectStore::write_at`] temp file names. +static NEXT_TMP: AtomicU64 = AtomicU64::new(0); + /// Fail if `computed` doesn't match the content address `declared`. /// /// The single chokepoint for the content-addressed integrity guarantee. @@ -39,6 +46,97 @@ pub(crate) fn check_integrity(declared: &Hash, computed: Hash) -> Result<()> { Ok(()) } +/// Reject a blob whose bytes are exactly the canonical JSON of a `Tree` or +/// `Commit`. Those objects hash `json(inner)`, so such a blob would share a +/// content address with the typed object while being stored under a +/// different envelope; whichever write came first would win the address. +pub fn check_blob_unambiguous(bytes: &[u8]) -> Result<()> { + let canonical_tree = serde_json::from_slice::(bytes) + .ok() + .is_some_and(|t| serde_json::to_vec(&t).is_ok_and(|c| c == bytes)); + let canonical_commit = serde_json::from_slice::(bytes) + .ok() + .is_some_and(|c| serde_json::to_vec(&c).is_ok_and(|c| c == bytes)); + if canonical_tree || canonical_commit { + return Err(Error::AmbiguousBlob { + hash: Hash::of(bytes), + }); + } + Ok(()) +} + +/// Decode the bytes stored at a typed address and verify them against it. +/// +/// Stores written before raw objects got their own namespace may hold bare +/// `put_raw` bytes at a typed address. If the stored bytes do not decode to +/// an object with the requested hash but hash to it directly, they are a +/// legacy raw object, and a blob with the same content is the right answer. +pub(crate) fn decode_typed(hash: &Hash, stored: Vec) -> Result { + let parsed = serde_json::from_slice::(&stored); + if let Ok(object) = &parsed { + if object.hash() == *hash { + return parsed.map_err(Error::from); + } + } + if Hash::of(&stored) == *hash { + return Ok(Object::Blob(Blob::new(stored))); + } + let object = parsed?; + check_integrity(hash, object.hash())?; + Ok(object) +} + +/// What an occupied address holds, relative to the object being written. +pub(crate) enum Occupant { + /// The same content: a dedup hit. + Same, + /// A valid object that verifies at this address but is not the one + /// being written. Never replaced. + Different, + /// Undecodable or failing its integrity check: corruption or a + /// pre-fix squatter. Safe to replace. + Corrupt, +} + +/// Judge the bytes stored at typed address `hash` against `object`, exactly +/// as `get` would read them. +pub(crate) fn classify_typed(hash: &Hash, stored: Vec, object: &Object) -> Occupant { + match decode_typed(hash, stored) { + Ok(existing) if existing == *object => Occupant::Same, + Ok(_) => Occupant::Different, + Err(_) => Occupant::Corrupt, + } +} + +/// Judge the bytes stored at raw address `hash`. The address determines +/// the content, so a verified occupant is always the same object. +pub(crate) fn classify_raw(hash: &Hash, stored: &[u8]) -> Occupant { + if Hash::of(stored) == *hash { + Occupant::Same + } else { + Occupant::Corrupt + } +} + +/// Read a raw object found at a legacy (typed) address. Besides bare +/// `put_raw` bytes, that address may hold a blob envelope written by a +/// `put` of the same bytes; both carry the same content, so both are valid. +pub(crate) fn decode_legacy_raw(hash: &Hash, stored: Vec) -> Result> { + let computed = Hash::of(&stored); + if computed == *hash { + return Ok(stored); + } + if let Ok(Object::Blob(blob)) = serde_json::from_slice::(&stored) { + if blob.hash() == *hash { + return Ok(blob.bytes); + } + } + Err(Error::IntegrityError { + declared: *hash, + computed, + }) +} + /// The store contract. MVP implementer is `FsObjectStore`; future /// implementers include S3, GCS, and a network-replicated store. pub trait ObjectStore: Send + Sync { @@ -84,21 +182,105 @@ impl FsObjectStore { self } + /// Typed objects (`put_with_policy`): `//.zst`. fn path_for(&self, hash: &Hash) -> PathBuf { let (prefix, rest) = hash.shard(); self.root.join(prefix).join(format!("{rest}.zst")) } + /// Raw objects (`put_raw`): `/raw//.raw`. A separate + /// namespace, with its own suffix, so bare bytes and typed envelopes + /// can never claim the same file, even for a store rooted at `

/raw`. + fn raw_path_for(&self, hash: &Hash) -> PathBuf { + let (prefix, rest) = hash.shard(); + self.root + .join("raw") + .join(prefix) + .join(format!("{rest}.raw")) + } + + /// Write `bytes` at `path` unless it is already occupied. `classify` + /// judges the occupant exactly as the matching read would: the same + /// content is a dedup hit, a valid object of another kind is refused + /// (nothing is written), and a missing or corrupt occupant is replaced + /// atomically. "Corrupt" means the bytes were read but do not decode or + /// verify; a read that fails for any other reason (for example EACCES) + /// is an error, not licence to replace. + fn write_unless_occupied( + &self, + hash: &Hash, + path: &Path, + bytes: &[u8], + classify: impl FnOnce(Vec) -> Occupant, + ) -> Result<()> { + match std::fs::read(path) { + Ok(compressed) => { + let occupant = match zstd::stream::decode_all(&compressed[..]) { + Ok(stored) => classify(stored), + Err(_) => Occupant::Corrupt, + }; + match occupant { + Occupant::Same => return Ok(()), + Occupant::Different => return Err(Error::AddressOccupied { hash: *hash }), + Occupant::Corrupt => {} + } + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e.into()), + } + self.write_at(path, bytes) + } + fn write_at(&self, path: &Path, bytes: &[u8]) -> Result<()> { + Self::write_at_with(path, bytes, || NEXT_TMP.fetch_add(1, Ordering::Relaxed)) + } + + /// [`Self::write_at`] with an injectable temp-name counter, so tests can + /// force a collision deterministically. + fn write_at_with(path: &Path, bytes: &[u8], mut next: impl FnMut() -> u64) -> Result<()> { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } - // Write to a temp file in the same directory and atomically rename, - // so a crash mid-write never leaves a torn object in the store. - let tmp = path.with_extension("tmp"); + // Write to a uniquely named temp file in the same directory and + // atomically rename, so a crash mid-write never leaves a torn object + // in the store and no other writer (including a store whose root + // nests this one) can share the temp file. + let name = path.file_name().unwrap_or_default().to_string_lossy(); let compressed = zstd::stream::encode_all(bytes, 3)?; - std::fs::write(&tmp, &compressed)?; - std::fs::rename(&tmp, path)?; + // Retry on a taken name (another process with the same pid in a + // different namespace); only a file this call created is ever removed. + let mut opened = None; + for _ in 0..16 { + let tmp = path.with_file_name(format!("{name}.{}.{}.tmp", std::process::id(), next())); + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp) + { + Ok(file) => { + opened = Some((tmp, file)); + break; + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(e.into()), + } + } + let Some((tmp, mut file)) = opened else { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!("no free temp file name next to {}", path.display()), + ) + .into()); + }; + let written = (|| { + file.write_all(&compressed)?; + drop(file); + std::fs::rename(&tmp, path) + })(); + if let Err(e) = written { + let _ = std::fs::remove_file(&tmp); + return Err(e.into()); + } Ok(()) } @@ -127,12 +309,16 @@ impl ObjectStore for FsObjectStore { } } + if let Object::Blob(blob) = object { + check_blob_unambiguous(&blob.bytes)?; + } + let bytes = serde_json::to_vec(object)?; let hash = object.hash(); let path = self.path_for(&hash); - if !path.exists() { - self.write_at(&path, &bytes)?; - } + self.write_unless_occupied(&hash, &path, &bytes, |stored| { + classify_typed(&hash, stored, object) + })?; Ok(hash) } @@ -150,10 +336,8 @@ impl ObjectStore for FsObjectStore { } let hash = Hash::of(bytes); - let path = self.path_for(&hash); - if !path.exists() { - self.write_at(&path, bytes)?; - } + let path = self.raw_path_for(&hash); + self.write_unless_occupied(&hash, &path, bytes, |stored| classify_raw(&hash, &stored))?; Ok(hash) } @@ -163,19 +347,21 @@ impl ObjectStore for FsObjectStore { return Err(Error::NotFound(*hash)); } let bytes = self.read_at(&path)?; - let object: Object = serde_json::from_slice(&bytes)?; - // Integrity check: did we get back what the address says we should? - check_integrity(hash, object.hash())?; - Ok(object) + decode_typed(hash, bytes) } fn get_raw(&self, hash: &Hash) -> Result> { - let path = self.path_for(hash); - if !path.exists() { - return Err(Error::NotFound(*hash)); + let raw_path = self.raw_path_for(hash); + if !raw_path.exists() { + // Legacy layout: raw objects used to share the typed path. + let legacy = self.path_for(hash); + if !legacy.exists() { + return Err(Error::NotFound(*hash)); + } + return decode_legacy_raw(hash, self.read_at(&legacy)?); } - let bytes = self.read_at(&path)?; + let bytes = self.read_at(&raw_path)?; // Raw objects are addressed by `Hash::of(bytes)` (the `put_raw` // contract). Verify before returning so a tampered segment or // manifest on disk is rejected, not parsed. Audit finding #3. @@ -184,7 +370,7 @@ impl ObjectStore for FsObjectStore { } fn has(&self, hash: &Hash) -> bool { - self.path_for(hash).exists() + self.path_for(hash).exists() || self.raw_path_for(hash).exists() } } @@ -335,7 +521,7 @@ mod tests { // Overwrite the stored object with different bytes at the same // path (same content address) — the exact shape of a corrupted // disk or a store writer that altered an object in place. - let path = store.path_for(&hash); + let path = store.raw_path_for(&hash); store.write_at(&path, b"tampered bytes").unwrap(); match store.get_raw(&hash) { @@ -398,4 +584,255 @@ mod tests { other => panic!("expected SecretDetected, got {other:?}"), } } + + fn sample_commit() -> Commit { + Commit { + parent: None, + author: "a".into(), + timestamp: chrono::DateTime::from_timestamp(0, 0).unwrap(), + message: "m".into(), + code_sha: None, + prompts: None, + tools: None, + model: None, + memory_snapshot: None, + schema_version: None, + intent: None, + plan: None, + transcript: None, + evals: None, + cost_cents: 0, + signatures: Vec::new(), + peer_uid: None, + } + } + + #[test] + fn blob_put_does_not_shadow_later_put_raw() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let b = b"segment-like bytes".to_vec(); + store.put(&Object::Blob(Blob::new(b.clone()))).unwrap(); + let h = store.put_raw(ObjectKind::Segment, &b).unwrap(); + assert_eq!(store.get_raw(&h).unwrap(), b); + } + + #[test] + fn put_raw_does_not_shadow_later_blob_put() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let b = b"segment-like bytes".to_vec(); + let h = store.put_raw(ObjectKind::Segment, &b).unwrap(); + store.put(&Object::Blob(Blob::new(b.clone()))).unwrap(); + match store.get(&h).unwrap() { + Object::Blob(got) => assert_eq!(got.bytes, b), + other => panic!("expected Blob, got {other:?}"), + } + assert_eq!(store.get_raw(&h).unwrap(), b); + } + + #[test] + fn blob_equal_to_canonical_tree_or_commit_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + + let tree_json = serde_json::to_vec(&Tree::new()).unwrap(); + let commit_json = serde_json::to_vec(&sample_commit()).unwrap(); + for bytes in [tree_json, commit_json] { + match store.put(&Object::Blob(Blob::new(bytes.clone()))) { + Err(Error::AmbiguousBlob { hash }) => assert_eq!(hash, Hash::of(&bytes)), + other => panic!("expected AmbiguousBlob, got {other:?}"), + } + } + + // Ordinary JSON, including a non-canonical spelling of a tree, is fine. + store + .put(&Object::Blob(Blob::new(br#"{"prompt":"hi"}"#.to_vec()))) + .unwrap(); + store + .put(&Object::Blob(Blob::new(br#"{ "entries": {} }"#.to_vec()))) + .unwrap(); + } + + #[test] + fn legacy_raw_layout_is_still_readable() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let bytes = b"written by an older version".to_vec(); + let hash = Hash::of(&bytes); + store.write_at(&store.path_for(&hash), &bytes).unwrap(); + + assert!(store.has(&hash)); + assert_eq!(store.get_raw(&hash).unwrap(), bytes); + match store.get(&hash).unwrap() { + Object::Blob(got) => assert_eq!(got.bytes, bytes), + other => panic!("expected Blob, got {other:?}"), + } + } + + #[test] + fn typed_put_heals_a_wrong_occupant() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let b = b"some prompt".to_vec(); + let hash = Hash::of(&b); + store.write_at(&store.path_for(&hash), b"squatter").unwrap(); + + store.put(&Object::Blob(Blob::new(b.clone()))).unwrap(); + match store.get(&hash).unwrap() { + Object::Blob(got) => assert_eq!(got.bytes, b), + other => panic!("expected Blob, got {other:?}"), + } + } + + #[test] + fn raw_put_heals_a_wrong_occupant() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let b = b"segment bytes".to_vec(); + let hash = Hash::of(&b); + store + .write_at(&store.raw_path_for(&hash), b"corrupted") + .unwrap(); + + store.put_raw(ObjectKind::Segment, &b).unwrap(); + assert_eq!(store.get_raw(&hash).unwrap(), b); + } + + #[test] + fn blob_put_does_not_destroy_a_legacy_raw_object() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let b = b"legacy segment bytes".to_vec(); + let hash = Hash::of(&b); + store.write_at(&store.path_for(&hash), &b).unwrap(); + + store.put(&Object::Blob(Blob::new(b.clone()))).unwrap(); + assert_eq!(store.get_raw(&hash).unwrap(), b); + } + + #[test] + fn legacy_raw_read_tolerates_a_blob_envelope() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let b = b"legacy segment bytes".to_vec(); + let hash = Hash::of(&b); + let envelope = serde_json::to_vec(&Object::Blob(Blob::new(b.clone()))).unwrap(); + store.write_at(&store.path_for(&hash), &envelope).unwrap(); + + assert_eq!(store.get_raw(&hash).unwrap(), b); + } + + // A valid object of another kind owns the address (a legacy Blob whose + // bytes are canonical Tree JSON): the Tree put is refused and the Blob + // stays readable. + #[test] + fn typed_put_refuses_a_valid_other_kind_occupant() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let tree = Tree::new(); + let tree_json = serde_json::to_vec(&tree).unwrap(); + let hash = tree.hash(); + let envelope = serde_json::to_vec(&Object::Blob(Blob::new(tree_json.clone()))).unwrap(); + store.write_at(&store.path_for(&hash), &envelope).unwrap(); + + match store.put(&Object::Tree(tree)) { + Err(Error::AddressOccupied { hash: h }) => assert_eq!(h, hash), + other => panic!("expected AddressOccupied, got {other:?}"), + } + match store.get(&hash).unwrap() { + Object::Blob(got) => assert_eq!(got.bytes, tree_json), + other => panic!("expected the Blob to survive, got {other:?}"), + } + } + + // Stores rooted at `

` and `

/raw` must not share a file. + #[test] + fn raw_paths_cannot_collide_with_typed_paths_of_a_raw_root() { + let dir = tempfile::tempdir().unwrap(); + let outer = FsObjectStore::open(dir.path()).unwrap(); + let inner = FsObjectStore::open(dir.path().join("raw")).unwrap(); + let h = Hash::of(b"x"); + assert_ne!(outer.raw_path_for(&h), inner.path_for(&h)); + } + + // An occupant that cannot be read (here: no permission) is neither a + // dedup hit nor corruption; the put fails and the file is untouched. + #[cfg(unix)] + #[test] + fn unreadable_occupant_fails_the_put_and_is_left_alone() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let b = b"some prompt".to_vec(); + let hash = Hash::of(&b); + let path = store.path_for(&hash); + store.write_at(&path, b"occupant").unwrap(); + let before = std::fs::read(&path).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o000)).unwrap(); + if std::fs::read(&path).is_ok() { + eprintln!("running as root: permissions not enforced, skipping"); + return; + } + + let result = store.put(&Object::Blob(Blob::new(b))); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(matches!(result, Err(Error::Io(_))), "{result:?}"); + assert_eq!(std::fs::read(&path).unwrap(), before); + } + + // Writers never share a temp file, so a foreign file at the old shared + // temp name survives, and a failed write leaves no temp file behind. + #[test] + fn write_at_uses_private_temp_files() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let hash = Hash::of(b"x"); + let path = store.path_for(&hash); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let foreign = path.with_extension("tmp"); + std::fs::write(&foreign, b"someone else's temp").unwrap(); + + store.write_at(&path, b"payload").unwrap(); + assert_eq!(std::fs::read(&foreign).unwrap(), b"someone else's temp"); + assert_eq!(store.read_at(&path).unwrap(), b"payload"); + + // Rename onto a non-empty directory fails; the temp file must go. + let blocked = store.path_for(&Hash::of(b"y")); + std::fs::create_dir_all(blocked.join("child")).unwrap(); + assert!(store.write_at(&blocked, b"payload").is_err()); + let leftovers: Vec<_> = std::fs::read_dir(blocked.parent().unwrap()) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect(); + assert_eq!(leftovers.len(), 1, "temp file left behind: {leftovers:?}"); + } + + #[test] + fn write_at_skips_and_preserves_an_occupied_temp_name() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let path = store.path_for(&Hash::of(b"z")); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let name = path.file_name().unwrap().to_string_lossy().into_owned(); + let pid = std::process::id(); + // Occupy the first four names the (fixed, deterministic) counter yields. + let planted: Vec<_> = (0..4) + .map(|n| path.with_file_name(format!("{name}.{pid}.{n}.tmp"))) + .collect(); + for p in &planted { + std::fs::write(p, b"other writer").unwrap(); + } + let mut counter = 0u64; + FsObjectStore::write_at_with(&path, b"payload", || { + counter += 1; + counter - 1 + }) + .unwrap(); + for p in &planted { + assert_eq!(std::fs::read(p).unwrap(), b"other writer"); + } + assert_eq!(store.read_at(&path).unwrap(), b"payload"); + assert_eq!(counter, 5, "names 0..4 were taken, 4 was the first free"); + } } diff --git a/crates/agentic-core/tests/gcs_integration.rs b/crates/agentic-core/tests/gcs_integration.rs index 8a2e6e03..247a0681 100644 --- a/crates/agentic-core/tests/gcs_integration.rs +++ b/crates/agentic-core/tests/gcs_integration.rs @@ -39,7 +39,11 @@ fn fresh_prefix() -> String { .duration_since(UNIX_EPOCH) .unwrap() .as_nanos(); - format!("test-{nanos}") + // Clock ticks can coincide across parallel tests; the counter keeps + // prefixes (and so objects) unique per call. + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + format!("test-{nanos}-{n}") } fn make_store(prefix: &str) -> Option<(GcsObjectStore, tempfile::TempDir)> { @@ -164,9 +168,10 @@ fn corrupt_download_is_rejected_and_not_cached() { // Overwrite the bucket object at its exact key with zstd of DIFFERENT // bytes — the shape of a bucket writer tampering in place. The object - // name mirrors GcsObjectStore::object_name: `//.zst`. + // name mirrors GcsObjectStore::object_name for raw objects: + // `/raw//.raw.zst`. let hex = hash.to_hex(); - let object_name = format!("{prefix}/{}/{}.zst", &hex[..2], &hex[2..]); + let object_name = format!("{prefix}/raw/{}/{}.raw.zst", &hex[..2], &hex[2..]); let corrupt = zstd::stream::encode_all(&b"CORRUPTED"[..], 3).unwrap(); let upload_url = format!( "{}/upload/storage/v1/b/{}/o?uploadType=media&name={}", @@ -209,11 +214,12 @@ fn corrupt_download_is_rejected_and_not_cached() { } // Never cache a failed verification: the corrupt bytes must not have // been persisted to the cold cache. Cache layout mirrors - // GcsObjectStore::cache_path: `//.zst`. + // GcsObjectStore::cache_path for raw objects: `/raw//.raw.zst`. let cache_file = cache2 .path() + .join("raw") .join(&hex[..2]) - .join(format!("{}.zst", &hex[2..])); + .join(format!("{}.raw.zst", &hex[2..])); assert!( !cache_file.exists(), "corrupt download must not be written to the cache at {cache_file:?}" @@ -235,3 +241,43 @@ fn missing_object_returns_not_found() { other => panic!("expected NotFound, got {other:?}"), } } + +// An existing object is never overwritten: a typed put at an address that +// already holds a raw object must leave the raw object intact (they live in +// different namespaces), and repeat puts of the same object are dedup hits. +#[test] +#[ignore] +fn second_upload_never_overwrites_and_dedup_succeeds() { + let prefix = fresh_prefix(); + let Some((store, _cache)) = make_store(&prefix) else { + eprintln!("GCS_BUCKET not set — skipping"); + return; + }; + + let payload = b"honest segment bytes".to_vec(); + let h = store.put_raw(ObjectKind::Segment, &payload).unwrap(); + // Same-address typed put, then dedup re-puts of both kinds. + store + .put(&Object::Blob(Blob::new(payload.clone()))) + .unwrap(); + assert_eq!(store.put_raw(ObjectKind::Segment, &payload).unwrap(), h); + store + .put(&Object::Blob(Blob::new(payload.clone()))) + .unwrap(); + + // Cold cache so reads go to the bucket. + let cache2 = tempfile::tempdir().unwrap(); + let cold = GcsObjectStore::new( + bucket().unwrap(), + prefix.clone(), + cache2.path(), + endpoint(), + bearer(), + ) + .unwrap(); + assert_eq!(cold.get_raw(&h).unwrap(), payload); + match cold.get(&h).unwrap() { + Object::Blob(b) => assert_eq!(b.bytes, payload), + other => panic!("expected Blob, got {other:?}"), + } +} diff --git a/crates/agenticd/src/rollback/mod.rs b/crates/agenticd/src/rollback/mod.rs index 0075972e..10cf2b83 100644 --- a/crates/agenticd/src/rollback/mod.rs +++ b/crates/agenticd/src/rollback/mod.rs @@ -40,7 +40,7 @@ use std::path::PathBuf; use agentic_core::approval::{verify_token, ApprovalRejection}; use agentic_core::commit::{stage_and_commit, CommitInputs}; use agentic_core::refs::HeadRef; -use agentic_core::{Commit, Hash}; +use agentic_core::{check_blob_unambiguous, Commit, Hash}; use agentic_memory::adapter::SnapshotHandle; use agentic_memory::MemoryAdapter; use anyhow::{anyhow, Context}; @@ -263,6 +263,21 @@ pub async fn execute( preflight_prompts(&state, &args.repo, &prompts_hash)?; } + // Load everything the forward-record step will re-stage, and refuse + // before any side effect (migration, memory restore, prompt write; dry + // runs included) if a blob would be rejected as ambiguous. A pre-upgrade + // store can hold a prompt or tool blob whose bytes are canonical Tree or + // Commit JSON; staging it now fails with `AmbiguousBlob`, which must not + // happen after live state has already been rolled back. + let prompts_payload = read_text_blobs(&state, target.prompts)?; + let tools_payload = read_text_blobs(&state, target.tools)?; + let model_text = read_model_text(&state, &target)?; + ensure_unambiguous("prompts", &prompts_payload)?; + ensure_unambiguous("tools", &tools_payload)?; + if let Some(model) = &model_text { + check_blob_unambiguous(model.as_bytes()).context("target model blob")?; + } + let mut plan: Vec = Vec::new(); let branch = match state.refs.read_head()? { @@ -427,10 +442,6 @@ pub async fn execute( // Build a fresh Commit whose dimensions match the target but whose // parent is the current branch tip. History shows C → C' even though // C' contains A's state. - let prompts_payload = read_text_blobs(&state, target.prompts)?; - let tools_payload = read_text_blobs(&state, target.tools)?; - let model_text = read_model_text(&state, &target)?; - let inputs = CommitInputs { author: "agentic-rollback".to_string(), message: format!("Rollback to {}", target_hash.short()), @@ -464,6 +475,18 @@ pub async fn execute( }) } +/// Refuse a tree of blobs that forward-record would re-stage as ambiguous. +fn ensure_unambiguous( + what: &str, + blobs: &std::collections::BTreeMap>, +) -> anyhow::Result<()> { + for (name, bytes) in blobs { + check_blob_unambiguous(bytes) + .with_context(|| format!("target {what} entry {name:?} cannot be re-staged"))?; + } + Ok(()) +} + /// Reject target Commits whose `(memory_snapshot, schema_version)` pair is /// in a shape the daemon should not have produced. /// diff --git a/crates/agenticd/src/wire_error.rs b/crates/agenticd/src/wire_error.rs index eec7c166..9a3a2f78 100644 --- a/crates/agenticd/src/wire_error.rs +++ b/crates/agenticd/src/wire_error.rs @@ -98,6 +98,12 @@ fn classify_core_error(err: &CoreError) -> (ErrorClass, &'static str, bool) { (ErrorClass::Storage, "object_integrity_failure", false) } CoreError::SecretDetected { .. } => (ErrorClass::Validation, "secret_detected", false), + // Structural rejection of untrusted blob bytes; the caller must + // change the content, so retrying can never succeed. + CoreError::AmbiguousBlob { .. } => (ErrorClass::Validation, "ambiguous_blob", false), + // Same structural rejection: a valid object of another kind + // already owns this address, so the write can never succeed. + CoreError::AddressOccupied { .. } => (ErrorClass::Validation, "address_occupied", false), // Structural rejection of an untrusted branch name (path // traversal defence). The caller must change the name; retrying // the identical request can never succeed. @@ -412,6 +418,46 @@ mod tests { } } + #[test] + fn core_ambiguous_blob_classifies_as_validation() { + let core_err: anyhow::Error = anyhow::Error::new(agentic_core::Error::AmbiguousBlob { + hash: agentic_core::Hash::of(b"x"), + }); + match map_anyhow_to_response_error(core_err) { + Response::Error { + class, + code, + retryable, + .. + } => { + assert_eq!(class, ErrorClass::Validation); + assert_eq!(code, "ambiguous_blob"); + assert!(!retryable); + } + _ => panic!("expected Response::Error"), + } + } + + #[test] + fn core_address_occupied_classifies_as_validation() { + let core_err: anyhow::Error = anyhow::Error::new(agentic_core::Error::AddressOccupied { + hash: agentic_core::Hash::of(b"x"), + }); + match map_anyhow_to_response_error(core_err) { + Response::Error { + class, + code, + retryable, + .. + } => { + assert_eq!(class, ErrorClass::Validation); + assert_eq!(code, "address_occupied"); + assert!(!retryable); + } + _ => panic!("expected Response::Error"), + } + } + #[test] fn core_secret_detected_classifies_as_validation() { // ADR-0013: SecretDetected is a structural rejection — the diff --git a/crates/agenticd/tests/rollback_in_memory.rs b/crates/agenticd/tests/rollback_in_memory.rs index d06bc60a..47de6443 100644 --- a/crates/agenticd/tests/rollback_in_memory.rs +++ b/crates/agenticd/tests/rollback_in_memory.rs @@ -10,7 +10,7 @@ use std::sync::Arc; use agentic_core::refs::Refs; -use agentic_core::{FsObjectStore, ObjectStore}; +use agentic_core::{FsObjectStore, Hash, Object, ObjectStore, Tree, TypedRef}; use agentic_memory::in_memory::InMemoryAdapter; use agentic_memory::MemoryAdapter; use agentic_proto::CommitInput; @@ -774,3 +774,105 @@ async fn rollback_refuses_invalid_prompt_name_before_side_effects() { ) .await; } + +// --------------------------------------------------------------------------- +// Pre-upgrade stores: a prompt blob whose bytes are canonical Tree JSON. +// --------------------------------------------------------------------------- + +/// Store `bytes` the way a pre-upgrade store did: bare bytes at the typed +/// path (`//.zst`), which the current code reads as a +/// blob. `put_raw` already produces that exact compressed file, under +/// `raw/`, so copy it into the legacy place. +fn plant_legacy_blob(objects: &std::path::Path, store: &dyn ObjectStore, bytes: &[u8]) -> Hash { + let hash = store + .put_raw(agentic_core::ObjectKind::Blob, bytes) + .unwrap(); + let (a, b) = hash.shard(); + let from = objects.join("raw").join(&a).join(format!("{b}.raw")); + let to = objects.join(&a).join(format!("{b}.zst")); + std::fs::create_dir_all(to.parent().unwrap()).unwrap(); + std::fs::copy(from, to).unwrap(); + hash +} + +#[tokio::test] +async fn rollback_refuses_legacy_ambiguous_blob_before_any_side_effect() { + let (state, adapter, baseline, _dir) = setup_gate_state(false).await; + let agentic_dir = state.repo_root.join(".agentic"); + + // A target commit whose prompts tree holds a legacy blob that is + // exactly the canonical JSON of a Tree. + let tree_json = serde_json::to_vec(&Tree::new()).unwrap(); + let blob_hash = plant_legacy_blob( + &agentic_dir.join("objects"), + state.store.as_ref(), + &tree_json, + ); + let mut prompts = Tree::new(); + prompts.insert( + "system.md", + TypedRef { + kind: agentic_core::ObjectKind::Blob, + hash: blob_hash, + }, + ); + let prompts_hash = state.store.put(&Object::Tree(prompts)).unwrap(); + let baseline_hash = state.refs.resolve(&baseline).unwrap().unwrap(); + let Object::Commit(mut target) = state.store.get(&baseline_hash).unwrap() else { + panic!("baseline is a commit"); + }; + target.prompts = Some(prompts_hash); + target.message = "legacy ambiguous prompt".into(); + let target_hash = state.store.put(&Object::Commit(target)).unwrap(); + + // Contaminate live state so a rollback that proceeded would be visible. + adapter.apply_migration("002_bump").await; + adapter + .insert_rows( + "messages", + vec![serde_json::json!({"id": 99, "body": "bad"})], + ) + .await; + let schema_dir = agentic_dir.join("schema"); + std::fs::create_dir_all(&schema_dir).unwrap(); + std::fs::write( + schema_dir.join("002_bump.down.sql"), + "-- no-op for fixture\n", + ) + .unwrap(); + let tip = state.refs.read_branch("main").unwrap(); + + for dry_run in [true, false] { + let err = rollback::execute( + Arc::clone(&state), + RollbackArgs { + target: target_hash.to_hex(), + dry_run, + accept_data_loss: false, + approval_token: None, + repo: state.repo_root.clone(), + }, + None, + ) + .await + .expect_err("a legacy ambiguous blob must be refused"); + assert!( + err.chain().any(|e| matches!( + e.downcast_ref::(), + Some(agentic_core::Error::AmbiguousBlob { .. }) + )), + "expected AmbiguousBlob (dry_run={dry_run}), got: {err:#}" + ); + + assert_eq!( + adapter.current_schema_version().await.unwrap(), + "002_bump", + "schema must not be reversed (dry_run={dry_run})" + ); + let rows = adapter.rows_of("messages").await; + assert_eq!(rows.len(), 1, "memory must not be restored"); + assert_eq!(rows[0]["body"], "bad"); + assert_eq!(state.refs.read_branch("main").unwrap(), tip); + assert!(!state.repo_root.join("prompts").exists()); + } +} diff --git a/docs/adr/0018-separate-raw-object-namespace.md b/docs/adr/0018-separate-raw-object-namespace.md new file mode 100644 index 00000000..8b1b6292 --- /dev/null +++ b/docs/adr/0018-separate-raw-object-namespace.md @@ -0,0 +1,77 @@ +# ADR-0018: Separate Namespace for Raw Objects + +Status: Accepted +Owner: Toni Bergholm +Date: 2026-11-20 +Amends: ADR-0002 (object store layout) + +## Context + +The object store keeps two kinds of content under one address space. +Typed objects (`put`, `get`) are stored as a JSON envelope tagged with their kind. +Raw objects (`put_raw`, `get_raw`), used for memory segments and segment manifests, are stored as the bare bytes. +Blobs hash their bytes directly, so a Blob and a raw object with the same bytes have the same address. +Trees and Commits hash `json(inner)` but are stored in the tagged envelope, so a Blob whose bytes equal that JSON shares an address with the Tree or Commit. + +Whoever wrote first owned the address. +The filesystem store skipped a write when the path existed, and the GCS store uploaded unconditionally and could overwrite. +A client that controls prompt blob bytes can predict segment bytes, because they derive from rows it wrote. +It could therefore occupy a raw object's address with a blob envelope. +A later honest `put_raw` then did nothing, and `get_raw` failed its integrity check. +The commit landed but could never be restored. + +## Decision 1: raw objects live in their own namespace + +Raw objects are stored apart from typed objects, with a distinct suffix, so the two can never share a path or object name, even for stores whose roots or prefixes nest. + +- Filesystem: `/raw//.raw`. Typed objects stay at `//.zst`. +- GCS: object name `/raw//.raw.zst`, with the local cache at `/raw//.raw.zst`. Typed objects keep their current names. + +`get_raw` falls back to the legacy shared path when the raw path is missing, so existing stores keep reading. +A legacy address may hold bare raw bytes or a blob envelope of the same bytes, and both are accepted. +`get` accepts bare raw bytes at a typed address as a Blob with the same content. +Every read is still verified against its address. +`has` is true if either namespace holds the object. + +## Decision 2: reject ambiguous blobs + +A Blob whose bytes are exactly the canonical JSON of a Tree or a Commit is rejected with `Error::AmbiguousBlob`. +That is the only way a Blob can share an address with a typed object. +The daemon maps it to a validation error (`ambiguous_blob`, not retryable), the same class as `SecretDetected`. + +## Decision 3: verify an occupied address, never trust it + +A put that finds its address occupied reads the occupant exactly as the matching read would, and classifies it. + +- Same content: a dedup hit, nothing is written. +- A valid object of another kind: the put fails with `Error::AddressOccupied` (a validation error on the wire, `address_occupied`, not retryable). Nothing is written or cached. +- Corrupt, meaning the bytes were read but do not decode or verify: the occupant is replaced. + +On the filesystem the replacement is atomic (unique temp file, then rename). A read failure other than "not found" is an error, not corruption. +On GCS every upload sends `ifGenerationMatch=0`, so an existing object is never overwritten blindly. HTTP 412 means the name is taken, and the occupant is downloaded and classified. +A corrupt occupant is replaced with `ifGenerationMatch=`. If the download reports no generation, or the object changed meanwhile, the put fails. + +## Decision 4: the cache follows the bucket + +The GCS local cache is written only once the bucket is known to hold the verified object: after a successful upload, or after a 412 whose occupant classified as the same content. +A failed or refused put leaves no cache entry. +`get_raw` and `has` consult the hash-verified cache under both names before any network request. + +## Decision 5: rollback refuses legacy ambiguous blobs up front + +A pre-upgrade store can already hold a prompt, tool or model blob whose bytes are canonical Tree or Commit JSON. +Rollback forward-records by re-staging those blobs, which would now fail with `AmbiguousBlob` after migrations, memory restore and prompt writes had already changed live state. +Rollback therefore loads everything it will re-stage right after validating the target, and refuses with `AmbiguousBlob` before any side effect, dry runs included. + +## Consequences + +- A dedup hit on GCS costs two requests (the conditional upload and a download) where it used to cost one. A new object still costs one. +- Existing stores keep working without migration. New raw objects land in the new namespace, and old ones are read through the legacy fallback. +- Legacy ambiguous blobs stay in the store. They are readable, but a rollback to a commit that references one is refused with a clear error before anything changes. +- A legacy occupant that is a valid object of another kind blocks a later put at its address with a non-retryable `address_occupied`. The realistic case is a pre-upgrade prompt or tool blob whose bytes are the canonical JSON of a predictable Tree: every later commit that produces that Tree fails until the occupant is removed. A bare raw segment or manifest cannot equal Tree or Commit JSON, so it does not cause this. +- Repair: the error names the address. An operator who confirms the occupant is not referenced by a commit they need deletes that one object file (or GCS object) and retries the commit. + +## Alternatives considered + +Kind-domain-separated addressing, where the kind is part of the hash input, would remove the whole class of collision. +It changes every address and the Commit wire contract, so it needs its own ADR and a store migration, and it is out of scope here.