From 7a3019398cd85409bf6504158ccab553df0133b9 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Fri, 20 Nov 2026 08:58:26 +0200 Subject: [PATCH 1/8] Keep raw and typed objects at separate addresses in the object store Typed objects are stored as JSON envelopes and raw objects (memory segments and manifests) as bare bytes, but both were addressed in one shared namespace. A blob whose content equals a raw object's bytes could therefore occupy that address first, after which the honest raw write was skipped and the later read failed its integrity check, leaving a commit that could not be restored. On GCS a later upload could also overwrite an existing object. Raw objects now live under a raw/ sub-namespace in both stores, with a fallback read of the legacy shared path. Blobs that are the canonical encoding of a tree or commit are rejected with a typed AmbiguousBlob error, mapped to a validation error on the wire. GCS uploads are conditional on the object not existing (HTTP 412 counts as already stored) and the local cache is written only after the upload succeeds. --- crates/agentic-core/src/gcs_store.rs | 245 ++++++++++++------- crates/agentic-core/src/lib.rs | 6 + crates/agentic-core/src/store.rs | 160 +++++++++++- crates/agentic-core/tests/gcs_integration.rs | 48 +++- crates/agenticd/src/wire_error.rs | 23 ++ 5 files changed, 387 insertions(+), 95 deletions(-) diff --git a/crates/agentic-core/src/gcs_store.rs b/crates/agentic-core/src/gcs_store.rs index 4496651b..c629acd6 100644 --- a/crates/agentic-core/src/gcs_store.rs +++ b/crates/agentic-core/src/gcs_store.rs @@ -44,7 +44,7 @@ use std::time::Duration; use crate::hash::Hash; use crate::object::{Object, ObjectKind}; use crate::scanner::{Allowlist, ScanPolicy, Scanner}; -use crate::store::{check_integrity, ObjectStore}; +use crate::store::{check_blob_unambiguous, check_integrity, decode_typed, ObjectStore}; use crate::{Error, Result}; /// Default upstream — GCS's public JSON API host. @@ -60,6 +60,14 @@ const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); /// tier. const STORAGE_CLASS: &str = "STANDARD"; +/// Which address namespace an object lives in. Typed objects are JSON +/// envelopes; raw objects are bare bytes. They must never share a name. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Ns { + Typed, + Raw, +} + #[derive(Debug, Clone)] pub struct GcsObjectStore { bucket: String, @@ -111,24 +119,31 @@ impl GcsObjectStore { /// GCS object name for a given hash. Sharded by the first two hex /// chars (matches the on-disk layout) so a ``gsutil ls gs://b/

`` - /// looks familiar. - fn object_name(&self, hash: &Hash) -> String { + /// looks familiar. Raw objects (`put_raw`) live under a `raw/` + /// sub-namespace so they can never share a name with a typed object. + fn object_name(&self, hash: &Hash, ns: Ns) -> String { let (a, b) = hash.shard(); let p = self.prefix.trim_matches('/'); + let raw = if ns == Ns::Raw { "raw/" } else { "" }; if p.is_empty() { - format!("{a}/{b}.zst") + format!("{raw}{a}/{b}.zst") } else { - format!("{p}/{a}/{b}.zst") + format!("{p}/{raw}{a}/{b}.zst") } } - fn cache_path(&self, hash: &Hash) -> PathBuf { + fn cache_path(&self, hash: &Hash, ns: Ns) -> PathBuf { let (a, b) = hash.shard(); - self.cache_dir.join(a).join(format!("{b}.zst")) + let base = if ns == Ns::Raw { + self.cache_dir.join("raw") + } else { + self.cache_dir.clone() + }; + base.join(a).join(format!("{b}.zst")) } - fn cache_read(&self, hash: &Hash) -> Option> { - let path = self.cache_path(hash); + fn cache_read(&self, hash: &Hash, ns: Ns) -> Option> { + let path = self.cache_path(hash, ns); let compressed = match fs::read(&path) { Ok(bytes) => bytes, Err(e) if e.kind() == std::io::ErrorKind::NotFound => return None, @@ -145,8 +160,8 @@ impl GcsObjectStore { zstd::stream::decode_all(&compressed[..]).ok() } - fn cache_write_compressed(&self, hash: &Hash, compressed: &[u8]) -> Result<()> { - let path = self.cache_path(hash); + fn cache_write_compressed(&self, hash: &Hash, ns: Ns, compressed: &[u8]) -> Result<()> { + let path = self.cache_path(hash, ns); if let Some(parent) = path.parent() { fs::create_dir_all(parent)?; } @@ -163,7 +178,7 @@ impl GcsObjectStore { fn upload_url(&self, name: &str) -> String { let encoded = urlencode(name); format!( - "{}/upload/storage/v1/b/{}/o?uploadType=media&name={}", + "{}/upload/storage/v1/b/{}/o?uploadType=media&ifGenerationMatch=0&name={}", self.endpoint.trim_end_matches('/'), self.bucket, encoded, @@ -205,8 +220,10 @@ impl GcsObjectStore { Error::Other(anyhow::anyhow!("{context}: HTTP {status}: {body}")) } - fn upload_compressed(&self, hash: &Hash, compressed: &[u8]) -> Result<()> { - let name = self.object_name(hash); + /// Upload with `ifGenerationMatch=0`: an existing object is never + /// overwritten, and HTTP 412 (already exists) counts as a dedup hit. + fn upload_compressed(&self, hash: &Hash, ns: Ns, compressed: &[u8]) -> Result<()> { + let name = self.object_name(hash, ns); let url = self.upload_url(&name); let resp = self .with_auth( @@ -218,6 +235,9 @@ impl GcsObjectStore { .body(compressed.to_vec()) .send() .map_err(|e| Error::Other(anyhow::anyhow!("upload {url}: {e}")))?; + if resp.status() == reqwest::StatusCode::PRECONDITION_FAILED { + return Ok(()); + } if !resp.status().is_success() { let status = resp.status(); let body = resp.text().unwrap_or_default(); @@ -229,8 +249,8 @@ impl GcsObjectStore { /// Returns ``Some((decompressed_bytes, raw_compressed_bytes))`` if /// the object exists in GCS. The raw bytes are returned so the /// caller can populate the local cache without re-compressing. - fn download_compressed(&self, hash: &Hash) -> Result, Vec)>> { - let name = self.object_name(hash); + fn download_compressed(&self, hash: &Hash, ns: Ns) -> Result, Vec)>> { + let name = self.object_name(hash, ns); let url = self.download_url(&name); let resp = self .with_auth(self.client.get(&url)) @@ -271,10 +291,10 @@ impl GcsObjectStore { /// * A **corrupt download** is rejected *before* `cache_write`, so a /// bad object is never written to the local cache ("never cache a /// failed verification"). - fn fetch_map(&self, hash: &Hash, map: impl Fn(Vec) -> Result) -> Result { - let cache_path = self.cache_path(hash); + fn fetch_map(&self, hash: &Hash, ns: Ns, map: impl Fn(Vec) -> Result) -> Result { + let cache_path = self.cache_path(hash, ns); if cache_path.exists() { - match self.cache_read(hash) { + match self.cache_read(hash, ns) { Some(cached) => match map(cached) { Ok(value) => return Ok(value), Err(err) => { @@ -297,13 +317,13 @@ impl GcsObjectStore { } } } - match self.download_compressed(hash)? { + match self.download_compressed(hash, ns)? { None => Err(Error::NotFound(*hash)), Some((bytes, compressed)) => { // Map (which verifies) BEFORE caching: a corrupt GCS object // must not be persisted to the local cache. let value = map(bytes)?; - let _ = self.cache_write_compressed(hash, &compressed); + let _ = self.cache_write_compressed(hash, ns, &compressed); Ok(value) } } @@ -336,8 +356,8 @@ impl GcsObjectStore { } } - fn remote_exists(&self, hash: &Hash) -> bool { - let name = self.object_name(hash); + fn remote_exists(&self, hash: &Hash, ns: Ns) -> bool { + let name = self.object_name(hash, ns); let url = self.metadata_url(&name); // HEAD against the metadata endpoint is the cheapest existence // check the JSON API offers. @@ -346,6 +366,24 @@ impl GcsObjectStore { } } +impl GcsObjectStore { + /// Upload first, cache second: the cache only ever holds objects that + /// are (or already were) in the bucket, so a failed upload leaves + /// nothing to roll back. Cache write is a best-effort hint. + fn store_compressed(&self, hash: &Hash, ns: Ns, compressed: &[u8]) -> Result<()> { + self.upload_compressed(hash, ns, compressed)?; + if let Err(e) = self.cache_write_compressed(hash, ns, compressed) { + tracing::warn!( + target: "agentic-core::gcs_store", + hash = %hash, + error = %e, + "failed to write local cache entry after GCS upload", + ); + } + Ok(()) + } +} + impl ObjectStore for GcsObjectStore { fn put_with_policy(&self, object: &Object, policy: ScanPolicy) -> Result { // Scanner pre-hook (ADR-0013). Reject blobs containing secrets @@ -363,31 +401,14 @@ impl ObjectStore for GcsObjectStore { } } + if let Object::Blob(blob) = object { + check_blob_unambiguous(&blob.bytes)?; + } + let bytes = serde_json::to_vec(object)?; let hash = object.hash(); let compressed = zstd::stream::encode_all(&bytes[..], 3)?; - // Cache locally first so a concurrent reader that lands during - // the upload doesn't miss; on upload failure the cache entry - // gets blown away. (The cache is a hint, not a source of - // truth — see post-MVP `agentic gc` for sweeping.) - self.cache_write_compressed(&hash, &compressed)?; - if let Err(e) = self.upload_compressed(&hash, &compressed) { - // Best-effort cache rollback so a failed upload can't masquerade as - // a successful put on the next get/has. If the unlink itself fails - // (permissions, transient FS error) we surface it via tracing so an - // operator can investigate — but still return the upload error, - // which is the user-visible cause. - if let Err(unlink_err) = fs::remove_file(self.cache_path(&hash)) { - if unlink_err.kind() != std::io::ErrorKind::NotFound { - tracing::warn!( - hash = %hash, - error = %unlink_err, - "failed to roll back cache entry after GCS upload error", - ); - } - } - return Err(e); - } + self.store_compressed(&hash, Ns::Typed, &compressed)?; Ok(hash) } @@ -405,52 +426,35 @@ impl ObjectStore for GcsObjectStore { let hash = Hash::of(bytes); let compressed = zstd::stream::encode_all(bytes, 3)?; - self.cache_write_compressed(&hash, &compressed)?; - if let Err(e) = self.upload_compressed(&hash, &compressed) { - // Best-effort cache rollback so a failed upload can't masquerade as - // a successful put on the next get/has. If the unlink itself fails - // (permissions, transient FS error) we surface it via tracing so an - // operator can investigate — but still return the upload error, - // which is the user-visible cause. - if let Err(unlink_err) = fs::remove_file(self.cache_path(&hash)) { - if unlink_err.kind() != std::io::ErrorKind::NotFound { - tracing::warn!( - hash = %hash, - error = %unlink_err, - "failed to roll back cache entry after GCS upload error", - ); - } - } - return Err(e); - } + self.store_compressed(&hash, Ns::Raw, &compressed)?; Ok(hash) } fn get(&self, hash: &Hash) -> Result { // Typed objects are addressed by `object.hash()`. Parse once inside // the map closure and verify that hash — no second deserialize. - self.fetch_map(hash, |bytes| { - let object: Object = serde_json::from_slice(&bytes)?; - check_integrity(hash, object.hash())?; - Ok(object) - }) + self.fetch_map(hash, Ns::Typed, |bytes| decode_typed(hash, bytes)) } fn get_raw(&self, hash: &Hash) -> Result> { // Raw objects are addressed by `Hash::of(bytes)` (the `put_raw` // contract) — verify against that, then return the bytes // unchanged (zero-copy: `bytes` moves out). Audit finding #3. - self.fetch_map(hash, |bytes| { + let verify = |bytes: Vec| { check_integrity(hash, Hash::of(&bytes))?; Ok(bytes) - }) + }; + match self.fetch_map(hash, Ns::Raw, verify) { + // Legacy layout: raw objects used to share the typed name. + Err(Error::NotFound(_)) => self.fetch_map(hash, Ns::Typed, verify), + other => other, + } } fn has(&self, hash: &Hash) -> bool { - if self.cache_path(hash).exists() { - return true; - } - self.remote_exists(hash) + [Ns::Raw, Ns::Typed] + .into_iter() + .any(|ns| self.cache_path(hash, ns).exists() || self.remote_exists(hash, ns)) } } @@ -494,7 +498,7 @@ mod tests { ) .unwrap(); let h = Hash::of(b"hello"); - let name = store.object_name(&h); + let name = store.object_name(&h, Ns::Typed); let hex = h.to_hex(); assert!(name.starts_with("repo-A/")); assert!(name.contains(&format!("{}/", &hex[..2]))); @@ -514,7 +518,7 @@ mod tests { ) .unwrap(); let h = Hash::of(b"hello"); - let name = store.object_name(&h); + let name = store.object_name(&h, Ns::Typed); assert!(!name.starts_with('/')); let hex = h.to_hex(); assert!(name.starts_with(&format!("{}/", &hex[..2]))); @@ -540,7 +544,7 @@ mod tests { ) .unwrap(); let h = Hash::of(b"nope"); - assert!(store.cache_read(&h).is_none()); + assert!(store.cache_read(&h, Ns::Typed).is_none()); } #[test] @@ -557,8 +561,10 @@ mod tests { let h = Hash::of(b"hello"); let payload = b"hello-world"; let compressed = zstd::stream::encode_all(&payload[..], 3).unwrap(); - store.cache_write_compressed(&h, &compressed).unwrap(); - let got = store.cache_read(&h).unwrap(); + store + .cache_write_compressed(&h, Ns::Typed, &compressed) + .unwrap(); + let got = store.cache_read(&h, Ns::Typed).unwrap(); assert_eq!(got, payload); } @@ -591,8 +597,10 @@ mod tests { let good = b"the canonical bytes"; let hash = Hash::of(good); let poison = zstd::stream::encode_all(&b"tampered!!"[..], 3).unwrap(); - store.cache_write_compressed(&hash, &poison).unwrap(); - assert!(store.cache_path(&hash).exists()); + store + .cache_write_compressed(&hash, Ns::Raw, &poison) + .unwrap(); + assert!(store.cache_path(&hash, Ns::Raw).exists()); let result = store.get_raw(&hash); // Never returns the poison. (The fall-through GCS fetch errors @@ -607,7 +615,7 @@ mod tests { } assert!(result.is_err(), "no server to heal from, so read errors"); assert!( - !store.cache_path(&hash).exists(), + !store.cache_path(&hash, Ns::Raw).exists(), "poisoned cache entry must be evicted" ); } @@ -633,7 +641,7 @@ mod tests { // Corrupt the cache slot into a directory (unreadable as a file, so // cache_read returns None → the eviction path runs). let hash = Hash::of(b"anything"); - let cache_path = store.cache_path(&hash); + let cache_path = store.cache_path(&hash, Ns::Raw); std::fs::create_dir_all(&cache_path).unwrap(); assert!(cache_path.is_dir()); @@ -643,4 +651,75 @@ mod tests { "a cache entry corrupted into a directory must still be evicted" ); } + + /// One-shot HTTP server answering a single request with `status`; + /// returns its address and a handle yielding the request line. + fn one_shot_server(status: u16) -> (std::net::SocketAddr, std::thread::JoinHandle) { + use std::io::{BufRead, BufReader, Read, Write}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let handle = std::thread::spawn(move || { + let (mut sock, _) = listener.accept().unwrap(); + let mut reader = BufReader::new(sock.try_clone().unwrap()); + let mut request_line = String::new(); + reader.read_line(&mut request_line).unwrap(); + let mut content_length = 0usize; + loop { + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + if line == "\r\n" { + break; + } + if let Some(v) = line.to_ascii_lowercase().strip_prefix("content-length:") { + content_length = v.trim().parse().unwrap(); + } + } + let mut body = vec![0u8; content_length]; + reader.read_exact(&mut body).unwrap(); + write!( + sock, + "HTTP/1.1 {status} X\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ) + .unwrap(); + request_line + }); + (addr, handle) + } + + // Uploads must carry `ifGenerationMatch=0` so an existing object is never + // overwritten, and 412 means "already exists": a dedup hit, not an error. + // (fake-gcs-server ignores this precondition, so it is checked here.) + #[test] + fn upload_sends_precondition_and_treats_412_as_dedup() { + let dir = tempdir().unwrap(); + let (addr, server) = one_shot_server(412); + let store = GcsObjectStore::new("b", "p", dir.path(), Some(format!("http://{addr}")), None) + .unwrap(); + let h = store + .put_raw(ObjectKind::Segment, b"already there") + .unwrap(); + let request_line = server.join().unwrap(); + assert!( + request_line.contains("ifGenerationMatch=0"), + "upload must be conditional: {request_line}" + ); + assert!(request_line.contains("raw%2F"), "{request_line}"); + assert!(store.cache_path(&h, Ns::Raw).exists()); + } + + // A failed upload must not leave a cache entry behind (the cache is only + // written after upload success or 412). + #[test] + fn failed_upload_leaves_no_cache_entry() { + let dir = tempdir().unwrap(); + let (addr, server) = one_shot_server(500); + let store = GcsObjectStore::new("b", "p", dir.path(), Some(format!("http://{addr}")), None) + .unwrap(); + let h = Hash::of(b"will not upload"); + assert!(store + .put_raw(ObjectKind::Segment, b"will not upload") + .is_err()); + server.join().unwrap(); + assert!(!store.cache_path(&h, Ns::Raw).exists()); + } } diff --git a/crates/agentic-core/src/lib.rs b/crates/agentic-core/src/lib.rs index 4e1d812a..0eac809c 100644 --- a/crates/agentic-core/src/lib.rs +++ b/crates/agentic-core/src/lib.rs @@ -52,6 +52,12 @@ pub enum Error { #[error("blob rejected by secret scanner: {hits:?}")] SecretDetected { hits: Vec }, + #[error( + "blob rejected: its bytes are the canonical encoding of a tree or commit \ + and would share address {hash} with that object" + )] + AmbiguousBlob { hash: Hash }, + #[error("invalid branch name {name:?}: {reason}")] InvalidBranchName { name: String, reason: String }, diff --git a/crates/agentic-core/src/store.rs b/crates/agentic-core/src/store.rs index fa3eb080..5eb08fb1 100644 --- a/crates/agentic-core/src/store.rs +++ b/crates/agentic-core/src/store.rs @@ -11,7 +11,7 @@ //! where additional backends (e.g. S3, network-replicated) will plug in. use crate::hash::Hash; -use crate::object::{Object, ObjectKind}; +use crate::object::{Blob, Commit, Object, ObjectKind, Tree}; use crate::scanner::{Allowlist, ScanPolicy, Scanner}; use crate::{Error, Result}; @@ -39,6 +39,46 @@ pub(crate) fn check_integrity(declared: &Hash, computed: Hash) -> Result<()> { Ok(()) } +/// Reject a blob whose bytes are exactly the canonical JSON of a `Tree` or +/// `Commit`. Those objects hash `json(inner)`, so such a blob would share a +/// content address with the typed object while being stored under a +/// different envelope; whichever write came first would win the address. +pub(crate) fn check_blob_unambiguous(bytes: &[u8]) -> Result<()> { + let canonical_tree = serde_json::from_slice::(bytes) + .ok() + .is_some_and(|t| serde_json::to_vec(&t).is_ok_and(|c| c == bytes)); + let canonical_commit = serde_json::from_slice::(bytes) + .ok() + .is_some_and(|c| serde_json::to_vec(&c).is_ok_and(|c| c == bytes)); + if canonical_tree || canonical_commit { + return Err(Error::AmbiguousBlob { + hash: Hash::of(bytes), + }); + } + Ok(()) +} + +/// Decode the bytes stored at a typed address and verify them against it. +/// +/// Stores written before raw objects got their own namespace may hold bare +/// `put_raw` bytes at a typed address. If the stored bytes do not decode to +/// an object with the requested hash but hash to it directly, they are a +/// legacy raw object, and a blob with the same content is the right answer. +pub(crate) fn decode_typed(hash: &Hash, stored: Vec) -> Result { + let parsed = serde_json::from_slice::(&stored); + if let Ok(object) = &parsed { + if object.hash() == *hash { + return parsed.map_err(Error::from); + } + } + if Hash::of(&stored) == *hash { + return Ok(Object::Blob(Blob::new(stored))); + } + let object = parsed?; + check_integrity(hash, object.hash())?; + Ok(object) +} + /// The store contract. MVP implementer is `FsObjectStore`; future /// implementers include S3, GCS, and a network-replicated store. pub trait ObjectStore: Send + Sync { @@ -84,11 +124,23 @@ impl FsObjectStore { self } + /// Typed objects (`put_with_policy`): `//.zst`. fn path_for(&self, hash: &Hash) -> PathBuf { let (prefix, rest) = hash.shard(); self.root.join(prefix).join(format!("{rest}.zst")) } + /// Raw objects (`put_raw`): `/raw//.zst`. A separate + /// namespace so bare bytes and typed envelopes can never claim the + /// same file. + fn raw_path_for(&self, hash: &Hash) -> PathBuf { + let (prefix, rest) = hash.shard(); + self.root + .join("raw") + .join(prefix) + .join(format!("{rest}.zst")) + } + fn write_at(&self, path: &Path, bytes: &[u8]) -> Result<()> { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; @@ -127,6 +179,10 @@ impl ObjectStore for FsObjectStore { } } + if let Object::Blob(blob) = object { + check_blob_unambiguous(&blob.bytes)?; + } + let bytes = serde_json::to_vec(object)?; let hash = object.hash(); let path = self.path_for(&hash); @@ -150,7 +206,7 @@ impl ObjectStore for FsObjectStore { } let hash = Hash::of(bytes); - let path = self.path_for(&hash); + let path = self.raw_path_for(&hash); if !path.exists() { self.write_at(&path, bytes)?; } @@ -163,15 +219,16 @@ impl ObjectStore for FsObjectStore { return Err(Error::NotFound(*hash)); } let bytes = self.read_at(&path)?; - let object: Object = serde_json::from_slice(&bytes)?; - // Integrity check: did we get back what the address says we should? - check_integrity(hash, object.hash())?; - Ok(object) + decode_typed(hash, bytes) } fn get_raw(&self, hash: &Hash) -> Result> { - let path = self.path_for(hash); + let mut path = self.raw_path_for(hash); + if !path.exists() { + // Legacy layout: raw objects used to share the typed path. + path = self.path_for(hash); + } if !path.exists() { return Err(Error::NotFound(*hash)); } @@ -184,7 +241,7 @@ impl ObjectStore for FsObjectStore { } fn has(&self, hash: &Hash) -> bool { - self.path_for(hash).exists() + self.path_for(hash).exists() || self.raw_path_for(hash).exists() } } @@ -335,7 +392,7 @@ mod tests { // Overwrite the stored object with different bytes at the same // path (same content address) — the exact shape of a corrupted // disk or a store writer that altered an object in place. - let path = store.path_for(&hash); + let path = store.raw_path_for(&hash); store.write_at(&path, b"tampered bytes").unwrap(); match store.get_raw(&hash) { @@ -398,4 +455,89 @@ mod tests { other => panic!("expected SecretDetected, got {other:?}"), } } + + fn sample_commit() -> Commit { + Commit { + parent: None, + author: "a".into(), + timestamp: chrono::DateTime::from_timestamp(0, 0).unwrap(), + message: "m".into(), + code_sha: None, + prompts: None, + tools: None, + model: None, + memory_snapshot: None, + schema_version: None, + intent: None, + plan: None, + transcript: None, + evals: None, + cost_cents: 0, + signatures: Vec::new(), + peer_uid: None, + } + } + + #[test] + fn blob_put_does_not_shadow_later_put_raw() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let b = b"segment-like bytes".to_vec(); + store.put(&Object::Blob(Blob::new(b.clone()))).unwrap(); + let h = store.put_raw(ObjectKind::Segment, &b).unwrap(); + assert_eq!(store.get_raw(&h).unwrap(), b); + } + + #[test] + fn put_raw_does_not_shadow_later_blob_put() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let b = b"segment-like bytes".to_vec(); + let h = store.put_raw(ObjectKind::Segment, &b).unwrap(); + store.put(&Object::Blob(Blob::new(b.clone()))).unwrap(); + match store.get(&h).unwrap() { + Object::Blob(got) => assert_eq!(got.bytes, b), + other => panic!("expected Blob, got {other:?}"), + } + assert_eq!(store.get_raw(&h).unwrap(), b); + } + + #[test] + fn blob_equal_to_canonical_tree_or_commit_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + + let tree_json = serde_json::to_vec(&Tree::new()).unwrap(); + let commit_json = serde_json::to_vec(&sample_commit()).unwrap(); + for bytes in [tree_json, commit_json] { + match store.put(&Object::Blob(Blob::new(bytes.clone()))) { + Err(Error::AmbiguousBlob { hash }) => assert_eq!(hash, Hash::of(&bytes)), + other => panic!("expected AmbiguousBlob, got {other:?}"), + } + } + + // Ordinary JSON, including a non-canonical spelling of a tree, is fine. + store + .put(&Object::Blob(Blob::new(br#"{"prompt":"hi"}"#.to_vec()))) + .unwrap(); + store + .put(&Object::Blob(Blob::new(br#"{ "entries": {} }"#.to_vec()))) + .unwrap(); + } + + #[test] + fn legacy_raw_layout_is_still_readable() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let bytes = b"written by an older version".to_vec(); + let hash = Hash::of(&bytes); + store.write_at(&store.path_for(&hash), &bytes).unwrap(); + + assert!(store.has(&hash)); + assert_eq!(store.get_raw(&hash).unwrap(), bytes); + match store.get(&hash).unwrap() { + Object::Blob(got) => assert_eq!(got.bytes, bytes), + other => panic!("expected Blob, got {other:?}"), + } + } } diff --git a/crates/agentic-core/tests/gcs_integration.rs b/crates/agentic-core/tests/gcs_integration.rs index 8a2e6e03..735d2154 100644 --- a/crates/agentic-core/tests/gcs_integration.rs +++ b/crates/agentic-core/tests/gcs_integration.rs @@ -164,9 +164,10 @@ fn corrupt_download_is_rejected_and_not_cached() { // Overwrite the bucket object at its exact key with zstd of DIFFERENT // bytes — the shape of a bucket writer tampering in place. The object - // name mirrors GcsObjectStore::object_name: `//.zst`. + // name mirrors GcsObjectStore::object_name for raw objects: + // `/raw//.zst`. let hex = hash.to_hex(); - let object_name = format!("{prefix}/{}/{}.zst", &hex[..2], &hex[2..]); + let object_name = format!("{prefix}/raw/{}/{}.zst", &hex[..2], &hex[2..]); let corrupt = zstd::stream::encode_all(&b"CORRUPTED"[..], 3).unwrap(); let upload_url = format!( "{}/upload/storage/v1/b/{}/o?uploadType=media&name={}", @@ -209,9 +210,10 @@ fn corrupt_download_is_rejected_and_not_cached() { } // Never cache a failed verification: the corrupt bytes must not have // been persisted to the cold cache. Cache layout mirrors - // GcsObjectStore::cache_path: `//.zst`. + // GcsObjectStore::cache_path for raw objects: `/raw//.zst`. let cache_file = cache2 .path() + .join("raw") .join(&hex[..2]) .join(format!("{}.zst", &hex[2..])); assert!( @@ -235,3 +237,43 @@ fn missing_object_returns_not_found() { other => panic!("expected NotFound, got {other:?}"), } } + +// An existing object is never overwritten: a typed put at an address that +// already holds a raw object must leave the raw object intact (they live in +// different namespaces), and repeat puts of the same object are dedup hits. +#[test] +#[ignore] +fn second_upload_never_overwrites_and_dedup_succeeds() { + let prefix = fresh_prefix(); + let Some((store, _cache)) = make_store(&prefix) else { + eprintln!("GCS_BUCKET not set — skipping"); + return; + }; + + let payload = b"honest segment bytes".to_vec(); + let h = store.put_raw(ObjectKind::Segment, &payload).unwrap(); + // Same-address typed put, then dedup re-puts of both kinds. + store + .put(&Object::Blob(Blob::new(payload.clone()))) + .unwrap(); + assert_eq!(store.put_raw(ObjectKind::Segment, &payload).unwrap(), h); + store + .put(&Object::Blob(Blob::new(payload.clone()))) + .unwrap(); + + // Cold cache so reads go to the bucket. + let cache2 = tempfile::tempdir().unwrap(); + let cold = GcsObjectStore::new( + bucket().unwrap(), + prefix.clone(), + cache2.path(), + endpoint(), + bearer(), + ) + .unwrap(); + assert_eq!(cold.get_raw(&h).unwrap(), payload); + match cold.get(&h).unwrap() { + Object::Blob(b) => assert_eq!(b.bytes, payload), + other => panic!("expected Blob, got {other:?}"), + } +} diff --git a/crates/agenticd/src/wire_error.rs b/crates/agenticd/src/wire_error.rs index eec7c166..127ca6a5 100644 --- a/crates/agenticd/src/wire_error.rs +++ b/crates/agenticd/src/wire_error.rs @@ -98,6 +98,9 @@ fn classify_core_error(err: &CoreError) -> (ErrorClass, &'static str, bool) { (ErrorClass::Storage, "object_integrity_failure", false) } CoreError::SecretDetected { .. } => (ErrorClass::Validation, "secret_detected", false), + // Structural rejection of untrusted blob bytes; the caller must + // change the content, so retrying can never succeed. + CoreError::AmbiguousBlob { .. } => (ErrorClass::Validation, "ambiguous_blob", false), // Structural rejection of an untrusted branch name (path // traversal defence). The caller must change the name; retrying // the identical request can never succeed. @@ -412,6 +415,26 @@ mod tests { } } + #[test] + fn core_ambiguous_blob_classifies_as_validation() { + let core_err: anyhow::Error = anyhow::Error::new(agentic_core::Error::AmbiguousBlob { + hash: agentic_core::Hash::of(b"x"), + }); + match map_anyhow_to_response_error(core_err) { + Response::Error { + class, + code, + retryable, + .. + } => { + assert_eq!(class, ErrorClass::Validation); + assert_eq!(code, "ambiguous_blob"); + assert!(!retryable); + } + _ => panic!("expected Response::Error"), + } + } + #[test] fn core_secret_detected_classifies_as_validation() { // ADR-0013: SecretDetected is a structural rejection — the From 5f1e2befc358f45839916860c6d1e738a05f9e21 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Fri, 20 Nov 2026 09:22:40 +0200 Subject: [PATCH 2/8] Verify and heal occupied object addresses instead of trusting them The first fix stopped new collisions but still accepted an occupied address without looking at it, so an object squatted before the fix could win forever and the local cache would hide it. A put now reads what is already stored and compares it with what is being written. The same content is a dedup hit, anything else is replaced: atomically on disk, and on GCS with a precondition on the generation that was just observed, where losing that race is an error. The GCS cache is written only once the bucket is known to hold the verified object. Legacy raw reads now accept a blob envelope of the same bytes, so replacing a squatter cannot destroy a legacy segment. GCS raw objects get a distinct .raw.zst suffix so a store with prefix p and one with prefix p/raw cannot share a name, and get_raw consults the verified local cache under both names before any network request. --- crates/agentic-core/src/gcs_store.rs | 547 +++++++++++++++---- crates/agentic-core/src/store.rs | 115 +++- crates/agentic-core/tests/gcs_integration.rs | 8 +- 3 files changed, 544 insertions(+), 126 deletions(-) diff --git a/crates/agentic-core/src/gcs_store.rs b/crates/agentic-core/src/gcs_store.rs index c629acd6..9b469877 100644 --- a/crates/agentic-core/src/gcs_store.rs +++ b/crates/agentic-core/src/gcs_store.rs @@ -44,7 +44,9 @@ use std::time::Duration; use crate::hash::Hash; use crate::object::{Object, ObjectKind}; use crate::scanner::{Allowlist, ScanPolicy, Scanner}; -use crate::store::{check_blob_unambiguous, check_integrity, decode_typed, ObjectStore}; +use crate::store::{ + check_blob_unambiguous, check_integrity, decode_legacy_raw, decode_typed, ObjectStore, +}; use crate::{Error, Result}; /// Default upstream — GCS's public JSON API host. @@ -68,6 +70,31 @@ enum Ns { Raw, } +impl Ns { + /// (directory prefix, file extension) of this namespace's objects. + fn layout(self) -> (&'static str, &'static str) { + match self { + Ns::Typed => ("", "zst"), + Ns::Raw => ("raw/", "raw.zst"), + } + } +} + +/// What a conditional upload did. +#[derive(Debug, PartialEq, Eq)] +enum Upload { + Stored, + /// HTTP 412: the object exists (or, for a generation match, changed). + PreconditionFailed, +} + +/// A downloaded object: the stored (compressed) bytes and the GCS +/// generation they were read at, when the server reported one. +struct Downloaded { + compressed: Vec, + generation: Option, +} + #[derive(Debug, Clone)] pub struct GcsObjectStore { bucket: String, @@ -119,27 +146,26 @@ impl GcsObjectStore { /// GCS object name for a given hash. Sharded by the first two hex /// chars (matches the on-disk layout) so a ``gsutil ls gs://b/

`` - /// looks familiar. Raw objects (`put_raw`) live under a `raw/` - /// sub-namespace so they can never share a name with a typed object. + /// looks familiar. Raw objects (`put_raw`) live under `raw/` with a + /// distinct `.raw.zst` suffix, so they can never share a name with a + /// typed object, even for a store whose prefix is `

/raw`. fn object_name(&self, hash: &Hash, ns: Ns) -> String { let (a, b) = hash.shard(); let p = self.prefix.trim_matches('/'); - let raw = if ns == Ns::Raw { "raw/" } else { "" }; + let (dir, ext) = ns.layout(); if p.is_empty() { - format!("{raw}{a}/{b}.zst") + format!("{dir}{a}/{b}.{ext}") } else { - format!("{p}/{raw}{a}/{b}.zst") + format!("{p}/{dir}{a}/{b}.{ext}") } } fn cache_path(&self, hash: &Hash, ns: Ns) -> PathBuf { let (a, b) = hash.shard(); - let base = if ns == Ns::Raw { - self.cache_dir.join("raw") - } else { - self.cache_dir.clone() - }; - base.join(a).join(format!("{b}.zst")) + let (dir, ext) = ns.layout(); + self.cache_dir + .join(format!("{dir}{a}")) + .join(format!("{b}.{ext}")) } fn cache_read(&self, hash: &Hash, ns: Ns) -> Option> { @@ -175,12 +201,15 @@ impl GcsObjectStore { // ---------- HTTP plumbing ---------------------------------------------- - fn upload_url(&self, name: &str) -> String { + /// `if_generation` is the `ifGenerationMatch` precondition: `0` means + /// "only if the object does not exist yet". + fn upload_url(&self, name: &str, if_generation: &str) -> String { let encoded = urlencode(name); format!( - "{}/upload/storage/v1/b/{}/o?uploadType=media&ifGenerationMatch=0&name={}", + "{}/upload/storage/v1/b/{}/o?uploadType=media&ifGenerationMatch={}&name={}", self.endpoint.trim_end_matches('/'), self.bucket, + if_generation, encoded, ) } @@ -220,11 +249,18 @@ impl GcsObjectStore { Error::Other(anyhow::anyhow!("{context}: HTTP {status}: {body}")) } - /// Upload with `ifGenerationMatch=0`: an existing object is never - /// overwritten, and HTTP 412 (already exists) counts as a dedup hit. - fn upload_compressed(&self, hash: &Hash, ns: Ns, compressed: &[u8]) -> Result<()> { + /// Conditional upload (`ifGenerationMatch=`): an existing + /// object is never overwritten blindly. HTTP 412 is reported to the + /// caller, which must decide what the existing object means. + fn upload_compressed( + &self, + hash: &Hash, + ns: Ns, + compressed: &[u8], + if_generation: &str, + ) -> Result { let name = self.object_name(hash, ns); - let url = self.upload_url(&name); + let url = self.upload_url(&name, if_generation); let resp = self .with_auth( self.client @@ -236,20 +272,20 @@ impl GcsObjectStore { .send() .map_err(|e| Error::Other(anyhow::anyhow!("upload {url}: {e}")))?; if resp.status() == reqwest::StatusCode::PRECONDITION_FAILED { - return Ok(()); + return Ok(Upload::PreconditionFailed); } if !resp.status().is_success() { let status = resp.status(); let body = resp.text().unwrap_or_default(); return Err(Self::http_err("uploading object", status, &body)); } - Ok(()) + Ok(Upload::Stored) } - /// Returns ``Some((decompressed_bytes, raw_compressed_bytes))`` if - /// the object exists in GCS. The raw bytes are returned so the - /// caller can populate the local cache without re-compressing. - fn download_compressed(&self, hash: &Hash, ns: Ns) -> Result, Vec)>> { + /// Returns the stored object if it exists in GCS. The compressed bytes + /// are returned as stored so the caller can populate the local cache + /// without re-compressing; decoding is left to the caller. + fn download_compressed(&self, hash: &Hash, ns: Ns) -> Result> { let name = self.object_name(hash, ns); let url = self.download_url(&name); let resp = self @@ -264,12 +300,44 @@ impl GcsObjectStore { let body = resp.text().unwrap_or_default(); return Err(Self::http_err("downloading object", status, &body)); } + let generation = resp + .headers() + .get("x-goog-generation") + .and_then(|v| v.to_str().ok()) + .map(str::to_owned); let compressed = resp .bytes() .map_err(|e| Error::Other(anyhow::anyhow!("reading body: {e}")))? .to_vec(); - let bytes = zstd::stream::decode_all(&compressed[..])?; - Ok(Some((bytes, compressed))) + Ok(Some(Downloaded { + compressed, + generation, + })) + } + + /// Current generation of an object, from the metadata endpoint. Only + /// used when a download did not report one. + fn remote_generation(&self, hash: &Hash, ns: Ns) -> Result { + let url = self.metadata_url(&self.object_name(hash, ns)); + let resp = self + .with_auth(self.client.get(&url)) + .send() + .map_err(|e| Error::Other(anyhow::anyhow!("GET {url}: {e}")))?; + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().unwrap_or_default(); + return Err(Self::http_err("reading object metadata", status, &body)); + } + let meta: serde_json::Value = resp + .json() + .map_err(|e| Error::Other(anyhow::anyhow!("parsing object metadata: {e}")))?; + meta.get("generation") + .and_then(|g| { + g.as_str() + .map(str::to_owned) + .or_else(|| g.as_u64().map(|n| n.to_string())) + }) + .ok_or_else(|| Error::Other(anyhow::anyhow!("object metadata has no generation"))) } /// Fetch the bytes for `hash` and map them to `T`, where `map` also @@ -292,34 +360,55 @@ impl GcsObjectStore { /// bad object is never written to the local cache ("never cache a /// failed verification"). fn fetch_map(&self, hash: &Hash, ns: Ns, map: impl Fn(Vec) -> Result) -> Result { + if let Some(value) = self.try_cache(hash, ns, &map) { + return Ok(value); + } + self.fetch_remote(hash, ns, &map) + } + + /// The cache half of [`Self::fetch_map`]: a hit that passes `map`, or + /// `None` (evicting a poisoned or unreadable entry on the way). + fn try_cache(&self, hash: &Hash, ns: Ns, map: &impl Fn(Vec) -> Result) -> Option { let cache_path = self.cache_path(hash, ns); - if cache_path.exists() { - match self.cache_read(hash, ns) { - Some(cached) => match map(cached) { - Ok(value) => return Ok(value), - Err(err) => { - tracing::warn!( - target: "agentic-core::gcs_store", - hash = %hash.to_hex(), - error = %err, - "cached object failed integrity check; evicting and refetching from GCS", - ); - // Evict the poisoned entry so `has`/subsequent reads - // don't keep trusting it. - self.evict_cache_entry(hash, &cache_path); - } - }, - None => { - // Cache file exists but is unreadable/corrupt (e.g. torn - // write or zstd decode failure). Evict so `has` stays - // consistent and future reads can heal from GCS. + if !cache_path.exists() { + return None; + } + match self.cache_read(hash, ns) { + Some(cached) => match map(cached) { + Ok(value) => return Some(value), + Err(err) => { + tracing::warn!( + target: "agentic-core::gcs_store", + hash = %hash.to_hex(), + error = %err, + "cached object failed integrity check; evicting and refetching from GCS", + ); + // Evict the poisoned entry so `has`/subsequent reads + // don't keep trusting it. self.evict_cache_entry(hash, &cache_path); } + }, + None => { + // Cache file exists but is unreadable/corrupt (e.g. torn + // write or zstd decode failure). Evict so `has` stays + // consistent and future reads can heal from GCS. + self.evict_cache_entry(hash, &cache_path); } } + None + } + + /// The network half of [`Self::fetch_map`]. + fn fetch_remote( + &self, + hash: &Hash, + ns: Ns, + map: &impl Fn(Vec) -> Result, + ) -> Result { match self.download_compressed(hash, ns)? { None => Err(Error::NotFound(*hash)), - Some((bytes, compressed)) => { + Some(Downloaded { compressed, .. }) => { + let bytes = zstd::stream::decode_all(&compressed[..])?; // Map (which verifies) BEFORE caching: a corrupt GCS object // must not be persisted to the local cache. let value = map(bytes)?; @@ -367,11 +456,53 @@ impl GcsObjectStore { } impl GcsObjectStore { - /// Upload first, cache second: the cache only ever holds objects that - /// are (or already were) in the bucket, so a failed upload leaves - /// nothing to roll back. Cache write is a best-effort hint. - fn store_compressed(&self, hash: &Hash, ns: Ns, compressed: &[u8]) -> Result<()> { - self.upload_compressed(hash, ns, compressed)?; + /// Put `compressed` at `hash`, never replacing a *verified* object. + /// + /// The upload is conditional on the name being free. On HTTP 412 the + /// occupant is downloaded and judged by `is_same` (it must decode as + /// the content being written, exactly as the matching read would). If + /// it is the same, this is a dedup hit. If not (a pre-fix squatter or + /// corruption) it is replaced, conditional on the generation just + /// observed; losing that race is an error, not success. The local + /// cache is written only once the bucket is known to hold the verified + /// object, so a failed or refused put never leaves a cache entry. + /// + /// Network cost: 1 request for a new object; 2 (upload, download) for a + /// dedup hit; 3 when a bad occupant is replaced (4 if the download + /// carried no generation header). + fn store_compressed( + &self, + hash: &Hash, + ns: Ns, + compressed: &[u8], + is_same: impl FnOnce(Vec) -> bool, + ) -> Result<()> { + if self.upload_compressed(hash, ns, compressed, "0")? == Upload::PreconditionFailed { + let existing = self.download_compressed(hash, ns)?.ok_or_else(|| { + Error::Other(anyhow::anyhow!("object {hash} vanished during put")) + })?; + let same = zstd::stream::decode_all(&existing.compressed[..]) + .ok() + .is_some_and(is_same); + if !same { + let generation = match existing.generation { + Some(g) => g, + None => self.remote_generation(hash, ns)?, + }; + tracing::warn!( + target: "agentic-core::gcs_store", + hash = %hash, + "replacing unverifiable object at content address", + ); + if self.upload_compressed(hash, ns, compressed, &generation)? + == Upload::PreconditionFailed + { + return Err(Error::Other(anyhow::anyhow!( + "object {hash} changed while replacing an unverifiable occupant" + ))); + } + } + } if let Err(e) = self.cache_write_compressed(hash, ns, compressed) { tracing::warn!( target: "agentic-core::gcs_store", @@ -408,7 +539,9 @@ impl ObjectStore for GcsObjectStore { let bytes = serde_json::to_vec(object)?; let hash = object.hash(); let compressed = zstd::stream::encode_all(&bytes[..], 3)?; - self.store_compressed(&hash, Ns::Typed, &compressed)?; + self.store_compressed(&hash, Ns::Typed, &compressed, |stored| { + decode_typed(&hash, stored).is_ok_and(|existing| existing == *object) + })?; Ok(hash) } @@ -426,7 +559,9 @@ impl ObjectStore for GcsObjectStore { let hash = Hash::of(bytes); let compressed = zstd::stream::encode_all(bytes, 3)?; - self.store_compressed(&hash, Ns::Raw, &compressed)?; + self.store_compressed(&hash, Ns::Raw, &compressed, |stored| { + Hash::of(&stored) == hash + })?; Ok(hash) } @@ -440,13 +575,23 @@ impl ObjectStore for GcsObjectStore { // Raw objects are addressed by `Hash::of(bytes)` (the `put_raw` // contract) — verify against that, then return the bytes // unchanged (zero-copy: `bytes` moves out). Audit finding #3. - let verify = |bytes: Vec| { + let verify_raw = |bytes: Vec| { check_integrity(hash, Hash::of(&bytes))?; Ok(bytes) }; - match self.fetch_map(hash, Ns::Raw, verify) { - // Legacy layout: raw objects used to share the typed name. - Err(Error::NotFound(_)) => self.fetch_map(hash, Ns::Typed, verify), + // Legacy layout: raw objects used to share the typed name (and may + // hold a blob envelope of the same bytes). + let verify_legacy = |bytes: Vec| decode_legacy_raw(hash, bytes); + // Hash-verified cache entries first, under both names, so a cached + // snapshot restores without touching the network. + if let Some(bytes) = self.try_cache(hash, Ns::Raw, &verify_raw) { + return Ok(bytes); + } + if let Some(bytes) = self.try_cache(hash, Ns::Typed, &verify_legacy) { + return Ok(bytes); + } + match self.fetch_remote(hash, Ns::Raw, &verify_raw) { + Err(Error::NotFound(_)) => self.fetch_remote(hash, Ns::Typed, &verify_legacy), other => other, } } @@ -652,74 +797,258 @@ mod tests { ); } - /// One-shot HTTP server answering a single request with `status`; - /// returns its address and a handle yielding the request line. - fn one_shot_server(status: u16) -> (std::net::SocketAddr, std::thread::JoinHandle) { - use std::io::{BufRead, BufReader, Read, Write}; - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let addr = listener.local_addr().unwrap(); - let handle = std::thread::spawn(move || { - let (mut sock, _) = listener.accept().unwrap(); - let mut reader = BufReader::new(sock.try_clone().unwrap()); - let mut request_line = String::new(); - reader.read_line(&mut request_line).unwrap(); - let mut content_length = 0usize; - loop { - let mut line = String::new(); - reader.read_line(&mut line).unwrap(); - if line == "\r\n" { - break; + // ---------- mock GCS server ------------------------------------------- + + #[derive(Clone, Debug)] + struct Req { + method: String, + target: String, + } + + struct Resp { + status: u16, + headers: Vec<(&'static str, String)>, + body: Vec, + } + + impl Resp { + fn status(status: u16) -> Self { + Resp { + status, + headers: Vec::new(), + body: Vec::new(), + } + } + } + + /// Minimal HTTP/1.1 server (one request per connection) driven by a + /// handler closure; records every request it sees. + struct Mock { + addr: std::net::SocketAddr, + log: Arc>>, + stop: Arc, + } + + impl Mock { + fn start(handler: impl Fn(&Req) -> Resp + Send + 'static) -> Mock { + use std::io::{BufRead, BufReader, Read, Write}; + use std::sync::atomic::Ordering; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let log: Arc>> = Arc::default(); + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let (log2, stop2) = (log.clone(), stop.clone()); + std::thread::spawn(move || { + for conn in listener.incoming() { + if stop2.load(Ordering::SeqCst) { + return; + } + let Ok(mut sock) = conn else { continue }; + let mut reader = BufReader::new(sock.try_clone().unwrap()); + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 { + continue; + } + let mut parts = line.split_whitespace(); + let req = Req { + method: parts.next().unwrap_or_default().to_string(), + target: parts.next().unwrap_or_default().to_string(), + }; + let mut content_length = 0usize; + loop { + let mut h = String::new(); + reader.read_line(&mut h).unwrap(); + if h == "\r\n" { + break; + } + if let Some(v) = h.to_ascii_lowercase().strip_prefix("content-length:") { + content_length = v.trim().parse().unwrap(); + } + } + let mut body = vec![0u8; content_length]; + reader.read_exact(&mut body).unwrap(); + log2.lock().unwrap().push(req.clone()); + let resp = handler(&req); + let mut out = format!( + "HTTP/1.1 {} X\r\nContent-Length: {}\r\nConnection: close\r\n", + resp.status, + resp.body.len() + ); + for (k, v) in &resp.headers { + out.push_str(&format!("{k}: {v}\r\n")); + } + out.push_str("\r\n"); + let _ = sock.write_all(out.as_bytes()); + let _ = sock.write_all(&resp.body); } - if let Some(v) = line.to_ascii_lowercase().strip_prefix("content-length:") { - content_length = v.trim().parse().unwrap(); + }); + Mock { addr, log, stop } + } + + fn store(&self, cache: &Path) -> GcsObjectStore { + GcsObjectStore::new("b", "p", cache, Some(format!("http://{}", self.addr)), None) + .unwrap() + } + + fn requests(&self) -> Vec { + self.log.lock().unwrap().clone() + } + } + + impl Drop for Mock { + fn drop(&mut self) { + self.stop.store(true, std::sync::atomic::Ordering::SeqCst); + let _ = std::net::TcpStream::connect(self.addr); + } + } + + fn zst(bytes: &[u8]) -> Vec { + zstd::stream::encode_all(bytes, 3).unwrap() + } + + /// A bucket whose first upload hits an occupied name (412), serving + /// `occupant` (generation 77). `replace_status` answers the follow-up + /// upload that carries `ifGenerationMatch=77`. + fn occupied_bucket(occupant: Vec, replace_status: u16) -> Mock { + Mock::start(move |req| { + if req.method == "POST" && req.target.contains("ifGenerationMatch=0") { + Resp::status(412) + } else if req.method == "POST" && req.target.contains("ifGenerationMatch=77") { + Resp::status(replace_status) + } else if req.method == "GET" && req.target.contains("alt=media") { + Resp { + status: 200, + headers: vec![("x-goog-generation", "77".into())], + body: occupant.clone(), } + } else { + Resp::status(500) } - let mut body = vec![0u8; content_length]; - reader.read_exact(&mut body).unwrap(); - write!( - sock, - "HTTP/1.1 {status} X\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" - ) - .unwrap(); - request_line - }); - (addr, handle) + }) } - // Uploads must carry `ifGenerationMatch=0` so an existing object is never - // overwritten, and 412 means "already exists": a dedup hit, not an error. - // (fake-gcs-server ignores this precondition, so it is checked here.) + fn is_post(r: &Req, needle: &str) -> bool { + r.method == "POST" && r.target.contains(needle) + } + + // Uploads are conditional on the name being free (fake-gcs-server + // ignores the precondition, so it is checked here), and a 412 over the + // *same* object is a dedup hit: verified by download, nothing replaced. #[test] - fn upload_sends_precondition_and_treats_412_as_dedup() { + fn upload_is_conditional_and_412_over_same_object_is_dedup() { let dir = tempdir().unwrap(); - let (addr, server) = one_shot_server(412); - let store = GcsObjectStore::new("b", "p", dir.path(), Some(format!("http://{addr}")), None) - .unwrap(); - let h = store - .put_raw(ObjectKind::Segment, b"already there") - .unwrap(); - let request_line = server.join().unwrap(); - assert!( - request_line.contains("ifGenerationMatch=0"), - "upload must be conditional: {request_line}" - ); - assert!(request_line.contains("raw%2F"), "{request_line}"); + let payload = b"already there"; + let mock = occupied_bucket(zst(payload), 500); + let store = mock.store(dir.path()); + let h = store.put_raw(ObjectKind::Segment, payload).unwrap(); + + let reqs = mock.requests(); + assert_eq!(reqs.len(), 2, "dedup hit costs upload + download: {reqs:?}"); + assert!(is_post(&reqs[0], "ifGenerationMatch=0"), "{reqs:?}"); + assert!(reqs[0].target.contains("raw%2F"), "{reqs:?}"); + assert!(reqs[0].target.contains(".raw.zst"), "{reqs:?}"); + assert!(store.cache_path(&h, Ns::Raw).exists()); + } + + // A 412 over a *different* occupant (pre-fix squatter / corruption) is + // replaced, conditional on the generation observed, then cached. + #[test] + fn put_over_different_raw_occupant_replaces_it() { + let dir = tempdir().unwrap(); + let mock = occupied_bucket(zst(b"squatter"), 200); + let store = mock.store(dir.path()); + let h = store.put_raw(ObjectKind::Segment, b"honest").unwrap(); + + let reqs = mock.requests(); + assert_eq!(reqs.len(), 3, "{reqs:?}"); + assert!(is_post(&reqs[2], "ifGenerationMatch=77"), "{reqs:?}"); assert!(store.cache_path(&h, Ns::Raw).exists()); } - // A failed upload must not leave a cache entry behind (the cache is only - // written after upload success or 412). + #[test] + fn put_over_different_typed_occupant_replaces_it() { + let dir = tempdir().unwrap(); + let mock = occupied_bucket(zst(b"squatter"), 200); + let store = mock.store(dir.path()); + let obj = Object::Blob(crate::object::Blob::new(b"prompt".to_vec())); + let h = store.put(&obj).unwrap(); + + let reqs = mock.requests(); + assert!(is_post(&reqs[2], "ifGenerationMatch=77"), "{reqs:?}"); + assert!(store.cache_path(&h, Ns::Typed).exists()); + } + + // Losing the generation-conditional replace is an error, never success, + // and leaves no cache entry. + #[test] + fn second_412_while_replacing_is_an_error() { + let dir = tempdir().unwrap(); + let mock = occupied_bucket(zst(b"squatter"), 412); + let store = mock.store(dir.path()); + let h = Hash::of(b"honest"); + assert!(store.put_raw(ObjectKind::Segment, b"honest").is_err()); + assert!(!store.cache_path(&h, Ns::Raw).exists()); + } + + // A failed upload must not leave a cache entry behind. #[test] fn failed_upload_leaves_no_cache_entry() { let dir = tempdir().unwrap(); - let (addr, server) = one_shot_server(500); - let store = GcsObjectStore::new("b", "p", dir.path(), Some(format!("http://{addr}")), None) - .unwrap(); + let mock = Mock::start(|_| Resp::status(500)); + let store = mock.store(dir.path()); let h = Hash::of(b"will not upload"); assert!(store .put_raw(ObjectKind::Segment, b"will not upload") .is_err()); - server.join().unwrap(); assert!(!store.cache_path(&h, Ns::Raw).exists()); } + + // A hash-verified cached legacy object restores with no network at all. + #[test] + fn get_raw_of_cached_legacy_object_makes_no_request() { + let dir = tempdir().unwrap(); + let mock = Mock::start(|_| Resp::status(500)); + let store = mock.store(dir.path()); + let bytes = b"legacy segment".to_vec(); + let h = Hash::of(&bytes); + store + .cache_write_compressed(&h, Ns::Typed, &zst(&bytes)) + .unwrap(); + + assert_eq!(store.get_raw(&h).unwrap(), bytes); + assert!(mock.requests().is_empty(), "{:?}", mock.requests()); + } + + // A legacy name holding a blob envelope of the same bytes is a valid raw + // object, and its (valid typed) cache entry must not be evicted. + #[test] + fn get_raw_tolerates_cached_blob_envelope_without_evicting() { + let dir = tempdir().unwrap(); + let mock = Mock::start(|_| Resp::status(500)); + let store = mock.store(dir.path()); + let bytes = b"legacy segment".to_vec(); + let h = Hash::of(&bytes); + let envelope = + serde_json::to_vec(&Object::Blob(crate::object::Blob::new(bytes.clone()))).unwrap(); + store + .cache_write_compressed(&h, Ns::Typed, &zst(&envelope)) + .unwrap(); + + assert_eq!(store.get_raw(&h).unwrap(), bytes); + assert!(store.cache_path(&h, Ns::Typed).exists()); + assert!(mock.requests().is_empty()); + } + + // `

/raw` as a prefix must not collide with raw objects of prefix `

`. + #[test] + fn raw_names_cannot_collide_with_typed_names_of_a_raw_prefix() { + let dir = tempdir().unwrap(); + let h = Hash::of(b"x"); + let outer = GcsObjectStore::new("b", "p", dir.path(), None, None).unwrap(); + let inner = GcsObjectStore::new("b", "p/raw", dir.path(), None, None).unwrap(); + assert_ne!( + outer.object_name(&h, Ns::Raw), + inner.object_name(&h, Ns::Typed) + ); + } } diff --git a/crates/agentic-core/src/store.rs b/crates/agentic-core/src/store.rs index 5eb08fb1..4390b079 100644 --- a/crates/agentic-core/src/store.rs +++ b/crates/agentic-core/src/store.rs @@ -79,6 +79,25 @@ pub(crate) fn decode_typed(hash: &Hash, stored: Vec) -> Result { Ok(object) } +/// Read a raw object found at a legacy (typed) address. Besides bare +/// `put_raw` bytes, that address may hold a blob envelope written by a +/// `put` of the same bytes; both carry the same content, so both are valid. +pub(crate) fn decode_legacy_raw(hash: &Hash, stored: Vec) -> Result> { + let computed = Hash::of(&stored); + if computed == *hash { + return Ok(stored); + } + if let Ok(Object::Blob(blob)) = serde_json::from_slice::(&stored) { + if blob.hash() == *hash { + return Ok(blob.bytes); + } + } + Err(Error::IntegrityError { + declared: *hash, + computed, + }) +} + /// The store contract. MVP implementer is `FsObjectStore`; future /// implementers include S3, GCS, and a network-replicated store. pub trait ObjectStore: Send + Sync { @@ -141,6 +160,24 @@ impl FsObjectStore { .join(format!("{rest}.zst")) } + /// Write `bytes` at `path` unless the path already holds the same + /// content. `is_same` judges whatever is stored there exactly as the + /// matching read would; a missing, unreadable or different occupant + /// (a legacy squatter or corruption) is replaced atomically. + fn write_unless_same( + &self, + path: &Path, + bytes: &[u8], + is_same: impl FnOnce(Vec) -> bool, + ) -> Result<()> { + if let Ok(stored) = self.read_at(path) { + if is_same(stored) { + return Ok(()); + } + } + self.write_at(path, bytes) + } + fn write_at(&self, path: &Path, bytes: &[u8]) -> Result<()> { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; @@ -186,9 +223,9 @@ impl ObjectStore for FsObjectStore { let bytes = serde_json::to_vec(object)?; let hash = object.hash(); let path = self.path_for(&hash); - if !path.exists() { - self.write_at(&path, &bytes)?; - } + self.write_unless_same(&path, &bytes, |stored| { + decode_typed(&hash, stored).is_ok_and(|existing| existing == *object) + })?; Ok(hash) } @@ -207,9 +244,7 @@ impl ObjectStore for FsObjectStore { let hash = Hash::of(bytes); let path = self.raw_path_for(&hash); - if !path.exists() { - self.write_at(&path, bytes)?; - } + self.write_unless_same(&path, bytes, |stored| Hash::of(&stored) == hash)?; Ok(hash) } @@ -224,15 +259,16 @@ impl ObjectStore for FsObjectStore { } fn get_raw(&self, hash: &Hash) -> Result> { - let mut path = self.raw_path_for(hash); - if !path.exists() { + let raw_path = self.raw_path_for(hash); + if !raw_path.exists() { // Legacy layout: raw objects used to share the typed path. - path = self.path_for(hash); - } - if !path.exists() { - return Err(Error::NotFound(*hash)); + let legacy = self.path_for(hash); + if !legacy.exists() { + return Err(Error::NotFound(*hash)); + } + return decode_legacy_raw(hash, self.read_at(&legacy)?); } - let bytes = self.read_at(&path)?; + let bytes = self.read_at(&raw_path)?; // Raw objects are addressed by `Hash::of(bytes)` (the `put_raw` // contract). Verify before returning so a tampered segment or // manifest on disk is rejected, not parsed. Audit finding #3. @@ -540,4 +576,57 @@ mod tests { other => panic!("expected Blob, got {other:?}"), } } + + #[test] + fn typed_put_heals_a_wrong_occupant() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let b = b"some prompt".to_vec(); + let hash = Hash::of(&b); + store.write_at(&store.path_for(&hash), b"squatter").unwrap(); + + store.put(&Object::Blob(Blob::new(b.clone()))).unwrap(); + match store.get(&hash).unwrap() { + Object::Blob(got) => assert_eq!(got.bytes, b), + other => panic!("expected Blob, got {other:?}"), + } + } + + #[test] + fn raw_put_heals_a_wrong_occupant() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let b = b"segment bytes".to_vec(); + let hash = Hash::of(&b); + store + .write_at(&store.raw_path_for(&hash), b"corrupted") + .unwrap(); + + store.put_raw(ObjectKind::Segment, &b).unwrap(); + assert_eq!(store.get_raw(&hash).unwrap(), b); + } + + #[test] + fn blob_put_does_not_destroy_a_legacy_raw_object() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let b = b"legacy segment bytes".to_vec(); + let hash = Hash::of(&b); + store.write_at(&store.path_for(&hash), &b).unwrap(); + + store.put(&Object::Blob(Blob::new(b.clone()))).unwrap(); + assert_eq!(store.get_raw(&hash).unwrap(), b); + } + + #[test] + fn legacy_raw_read_tolerates_a_blob_envelope() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let b = b"legacy segment bytes".to_vec(); + let hash = Hash::of(&b); + let envelope = serde_json::to_vec(&Object::Blob(Blob::new(b.clone()))).unwrap(); + store.write_at(&store.path_for(&hash), &envelope).unwrap(); + + assert_eq!(store.get_raw(&hash).unwrap(), b); + } } diff --git a/crates/agentic-core/tests/gcs_integration.rs b/crates/agentic-core/tests/gcs_integration.rs index 735d2154..8471666f 100644 --- a/crates/agentic-core/tests/gcs_integration.rs +++ b/crates/agentic-core/tests/gcs_integration.rs @@ -165,9 +165,9 @@ fn corrupt_download_is_rejected_and_not_cached() { // Overwrite the bucket object at its exact key with zstd of DIFFERENT // bytes — the shape of a bucket writer tampering in place. The object // name mirrors GcsObjectStore::object_name for raw objects: - // `/raw//.zst`. + // `/raw//.raw.zst`. let hex = hash.to_hex(); - let object_name = format!("{prefix}/raw/{}/{}.zst", &hex[..2], &hex[2..]); + let object_name = format!("{prefix}/raw/{}/{}.raw.zst", &hex[..2], &hex[2..]); let corrupt = zstd::stream::encode_all(&b"CORRUPTED"[..], 3).unwrap(); let upload_url = format!( "{}/upload/storage/v1/b/{}/o?uploadType=media&name={}", @@ -210,12 +210,12 @@ fn corrupt_download_is_rejected_and_not_cached() { } // Never cache a failed verification: the corrupt bytes must not have // been persisted to the cold cache. Cache layout mirrors - // GcsObjectStore::cache_path for raw objects: `/raw//.zst`. + // GcsObjectStore::cache_path for raw objects: `/raw//.raw.zst`. let cache_file = cache2 .path() .join("raw") .join(&hex[..2]) - .join(format!("{}.zst", &hex[2..])); + .join(format!("{}.raw.zst", &hex[2..])); assert!( !cache_file.exists(), "corrupt download must not be written to the cache at {cache_file:?}" From 22cec9f39920af1ebc562bb519b59e147a0f05d7 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Fri, 20 Nov 2026 09:25:52 +0200 Subject: [PATCH 3/8] Make GCS integration test prefixes unique across parallel tests The prefix was derived from the clock alone, and parallel tests that start within the same clock tick shared one. Two tests that store the same payload then raced on one object, and one of them failed intermittently. A per-call counter keeps each test's objects separate. --- crates/agentic-core/tests/gcs_integration.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/agentic-core/tests/gcs_integration.rs b/crates/agentic-core/tests/gcs_integration.rs index 8471666f..247a0681 100644 --- a/crates/agentic-core/tests/gcs_integration.rs +++ b/crates/agentic-core/tests/gcs_integration.rs @@ -39,7 +39,11 @@ fn fresh_prefix() -> String { .duration_since(UNIX_EPOCH) .unwrap() .as_nanos(); - format!("test-{nanos}") + // Clock ticks can coincide across parallel tests; the counter keeps + // prefixes (and so objects) unique per call. + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + format!("test-{nanos}-{n}") } fn make_store(prefix: &str) -> Option<(GcsObjectStore, tempfile::TempDir)> { From 5b17bedfda71aee39bcd4966d4809dcac33ac561 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Fri, 20 Nov 2026 09:31:33 +0200 Subject: [PATCH 4/8] Refuse to replace a valid object of another kind at an occupied address Replacing every occupant that differs from the object being written could still destroy a valid legacy object, for example a blob whose bytes are the canonical encoding of a tree. Both stores now classify the occupant as the matching read would: the same content is a dedup hit, a valid object of another kind fails the put with a typed AddressOccupied error (a validation error on the wire) and nothing is written or cached, and only an undecodable or integrity-failing occupant is replaced. Filesystem raw objects now use a distinct .raw suffix so stores rooted at p and p/raw cannot share a file. The GCS metadata lookup for a missing generation is removed: replacing a corrupt object requires the generation reported by the download, and fails clearly without it. --- crates/agentic-core/src/gcs_store.rs | 148 ++++++++++++++++----------- crates/agentic-core/src/lib.rs | 6 ++ crates/agentic-core/src/store.rs | 99 +++++++++++++++--- crates/agenticd/src/wire_error.rs | 23 +++++ 4 files changed, 203 insertions(+), 73 deletions(-) diff --git a/crates/agentic-core/src/gcs_store.rs b/crates/agentic-core/src/gcs_store.rs index 9b469877..da6954f4 100644 --- a/crates/agentic-core/src/gcs_store.rs +++ b/crates/agentic-core/src/gcs_store.rs @@ -45,7 +45,8 @@ use crate::hash::Hash; use crate::object::{Object, ObjectKind}; use crate::scanner::{Allowlist, ScanPolicy, Scanner}; use crate::store::{ - check_blob_unambiguous, check_integrity, decode_legacy_raw, decode_typed, ObjectStore, + check_blob_unambiguous, check_integrity, classify_raw, classify_typed, decode_legacy_raw, + decode_typed, ObjectStore, Occupant, }; use crate::{Error, Result}; @@ -315,31 +316,6 @@ impl GcsObjectStore { })) } - /// Current generation of an object, from the metadata endpoint. Only - /// used when a download did not report one. - fn remote_generation(&self, hash: &Hash, ns: Ns) -> Result { - let url = self.metadata_url(&self.object_name(hash, ns)); - let resp = self - .with_auth(self.client.get(&url)) - .send() - .map_err(|e| Error::Other(anyhow::anyhow!("GET {url}: {e}")))?; - if !resp.status().is_success() { - let status = resp.status(); - let body = resp.text().unwrap_or_default(); - return Err(Self::http_err("reading object metadata", status, &body)); - } - let meta: serde_json::Value = resp - .json() - .map_err(|e| Error::Other(anyhow::anyhow!("parsing object metadata: {e}")))?; - meta.get("generation") - .and_then(|g| { - g.as_str() - .map(str::to_owned) - .or_else(|| g.as_u64().map(|n| n.to_string())) - }) - .ok_or_else(|| Error::Other(anyhow::anyhow!("object metadata has no generation"))) - } - /// Fetch the bytes for `hash` and map them to `T`, where `map` also /// performs the integrity check as it maps. /// @@ -456,50 +432,59 @@ impl GcsObjectStore { } impl GcsObjectStore { - /// Put `compressed` at `hash`, never replacing a *verified* object. + /// Put `compressed` at `hash`, never replacing a *valid* object. /// /// The upload is conditional on the name being free. On HTTP 412 the - /// occupant is downloaded and judged by `is_same` (it must decode as - /// the content being written, exactly as the matching read would). If - /// it is the same, this is a dedup hit. If not (a pre-fix squatter or - /// corruption) it is replaced, conditional on the generation just - /// observed; losing that race is an error, not success. The local - /// cache is written only once the bucket is known to hold the verified - /// object, so a failed or refused put never leaves a cache entry. + /// occupant is downloaded and judged by `classify` (exactly as the + /// matching read would judge it). The same content is a dedup hit; a + /// valid object of another kind is refused with + /// [`Error::AddressOccupied`], writing nothing. Only a corrupt occupant + /// (a pre-fix squatter or damage) is replaced, conditional on the + /// generation the download reported; if it reported none, or the + /// object changed meanwhile, the put fails rather than succeeding. The + /// local cache is written only once the bucket is known to hold the + /// verified object, so a failed or refused put never leaves a cache + /// entry. /// /// Network cost: 1 request for a new object; 2 (upload, download) for a - /// dedup hit; 3 when a bad occupant is replaced (4 if the download - /// carried no generation header). + /// dedup hit or a refusal; 3 when a corrupt occupant is replaced. fn store_compressed( &self, hash: &Hash, ns: Ns, compressed: &[u8], - is_same: impl FnOnce(Vec) -> bool, + classify: impl FnOnce(Vec) -> Occupant, ) -> Result<()> { if self.upload_compressed(hash, ns, compressed, "0")? == Upload::PreconditionFailed { let existing = self.download_compressed(hash, ns)?.ok_or_else(|| { Error::Other(anyhow::anyhow!("object {hash} vanished during put")) })?; - let same = zstd::stream::decode_all(&existing.compressed[..]) - .ok() - .is_some_and(is_same); - if !same { - let generation = match existing.generation { - Some(g) => g, - None => self.remote_generation(hash, ns)?, - }; - tracing::warn!( - target: "agentic-core::gcs_store", - hash = %hash, - "replacing unverifiable object at content address", - ); - if self.upload_compressed(hash, ns, compressed, &generation)? - == Upload::PreconditionFailed - { - return Err(Error::Other(anyhow::anyhow!( - "object {hash} changed while replacing an unverifiable occupant" - ))); + let occupant = match zstd::stream::decode_all(&existing.compressed[..]) { + Ok(stored) => classify(stored), + Err(_) => Occupant::Corrupt, + }; + match occupant { + Occupant::Same => {} + Occupant::Different => return Err(Error::AddressOccupied { hash: *hash }), + Occupant::Corrupt => { + let generation = existing.generation.ok_or_else(|| { + Error::Other(anyhow::anyhow!( + "cannot replace corrupt object {hash}: download carried no \ + x-goog-generation" + )) + })?; + tracing::warn!( + target: "agentic-core::gcs_store", + hash = %hash, + "replacing corrupt object at content address", + ); + if self.upload_compressed(hash, ns, compressed, &generation)? + == Upload::PreconditionFailed + { + return Err(Error::Other(anyhow::anyhow!( + "object {hash} changed while replacing a corrupt occupant" + ))); + } } } } @@ -540,7 +525,7 @@ impl ObjectStore for GcsObjectStore { let hash = object.hash(); let compressed = zstd::stream::encode_all(&bytes[..], 3)?; self.store_compressed(&hash, Ns::Typed, &compressed, |stored| { - decode_typed(&hash, stored).is_ok_and(|existing| existing == *object) + classify_typed(&hash, stored, object) })?; Ok(hash) } @@ -560,7 +545,7 @@ impl ObjectStore for GcsObjectStore { let hash = Hash::of(bytes); let compressed = zstd::stream::encode_all(bytes, 3)?; self.store_compressed(&hash, Ns::Raw, &compressed, |stored| { - Hash::of(&stored) == hash + classify_raw(&hash, &stored) })?; Ok(hash) } @@ -1051,4 +1036,51 @@ mod tests { inner.object_name(&h, Ns::Typed) ); } + + // A valid object of another kind (here a legacy Blob whose bytes are the + // canonical JSON of a Tree) is never replaced: the put is refused, and + // nothing is written or cached. + #[test] + fn put_over_valid_other_kind_occupant_is_refused() { + let dir = tempdir().unwrap(); + let tree = crate::object::Tree::new(); + let tree_json = serde_json::to_vec(&tree).unwrap(); + let envelope = + serde_json::to_vec(&Object::Blob(crate::object::Blob::new(tree_json))).unwrap(); + let mock = occupied_bucket(zst(&envelope), 200); + let store = mock.store(dir.path()); + + match store.put(&Object::Tree(tree.clone())) { + Err(Error::AddressOccupied { hash }) => assert_eq!(hash, tree.hash()), + other => panic!("expected AddressOccupied, got {other:?}"), + } + let reqs = mock.requests(); + assert_eq!(reqs.len(), 2, "upload + download only: {reqs:?}"); + assert!(!store.cache_path(&tree.hash(), Ns::Typed).exists()); + } + + // Replacing a corrupt occupant needs the generation from the download; + // without one the put fails, with no separate metadata lookup. + #[test] + fn replacing_without_generation_header_fails() { + let dir = tempdir().unwrap(); + let mock = Mock::start(|req| { + if req.method == "POST" && req.target.contains("ifGenerationMatch=0") { + Resp::status(412) + } else if req.method == "GET" && req.target.contains("alt=media") { + Resp { + status: 200, + headers: Vec::new(), + body: zst(b"squatter"), + } + } else { + Resp::status(500) + } + }); + let store = mock.store(dir.path()); + let h = Hash::of(b"honest"); + assert!(store.put_raw(ObjectKind::Segment, b"honest").is_err()); + assert_eq!(mock.requests().len(), 2, "{:?}", mock.requests()); + assert!(!store.cache_path(&h, Ns::Raw).exists()); + } } diff --git a/crates/agentic-core/src/lib.rs b/crates/agentic-core/src/lib.rs index 0eac809c..933f9c95 100644 --- a/crates/agentic-core/src/lib.rs +++ b/crates/agentic-core/src/lib.rs @@ -58,6 +58,12 @@ pub enum Error { )] AmbiguousBlob { hash: Hash }, + #[error( + "address {hash} is already occupied by a different valid object; \ + refusing to replace it" + )] + AddressOccupied { hash: Hash }, + #[error("invalid branch name {name:?}: {reason}")] InvalidBranchName { name: String, reason: String }, diff --git a/crates/agentic-core/src/store.rs b/crates/agentic-core/src/store.rs index 4390b079..61046cf9 100644 --- a/crates/agentic-core/src/store.rs +++ b/crates/agentic-core/src/store.rs @@ -79,6 +79,38 @@ pub(crate) fn decode_typed(hash: &Hash, stored: Vec) -> Result { Ok(object) } +/// What an occupied address holds, relative to the object being written. +pub(crate) enum Occupant { + /// The same content: a dedup hit. + Same, + /// A valid object that verifies at this address but is not the one + /// being written. Never replaced. + Different, + /// Undecodable or failing its integrity check: corruption or a + /// pre-fix squatter. Safe to replace. + Corrupt, +} + +/// Judge the bytes stored at typed address `hash` against `object`, exactly +/// as `get` would read them. +pub(crate) fn classify_typed(hash: &Hash, stored: Vec, object: &Object) -> Occupant { + match decode_typed(hash, stored) { + Ok(existing) if existing == *object => Occupant::Same, + Ok(_) => Occupant::Different, + Err(_) => Occupant::Corrupt, + } +} + +/// Judge the bytes stored at raw address `hash`. The address determines +/// the content, so a verified occupant is always the same object. +pub(crate) fn classify_raw(hash: &Hash, stored: &[u8]) -> Occupant { + if Hash::of(stored) == *hash { + Occupant::Same + } else { + Occupant::Corrupt + } +} + /// Read a raw object found at a legacy (typed) address. Besides bare /// `put_raw` bytes, that address may hold a blob envelope written by a /// `put` of the same bytes; both carry the same content, so both are valid. @@ -149,30 +181,34 @@ impl FsObjectStore { self.root.join(prefix).join(format!("{rest}.zst")) } - /// Raw objects (`put_raw`): `/raw//.zst`. A separate - /// namespace so bare bytes and typed envelopes can never claim the - /// same file. + /// Raw objects (`put_raw`): `/raw//.raw`. A separate + /// namespace, with its own suffix, so bare bytes and typed envelopes + /// can never claim the same file, even for a store rooted at `

/raw`. fn raw_path_for(&self, hash: &Hash) -> PathBuf { let (prefix, rest) = hash.shard(); self.root .join("raw") .join(prefix) - .join(format!("{rest}.zst")) + .join(format!("{rest}.raw")) } - /// Write `bytes` at `path` unless the path already holds the same - /// content. `is_same` judges whatever is stored there exactly as the - /// matching read would; a missing, unreadable or different occupant - /// (a legacy squatter or corruption) is replaced atomically. - fn write_unless_same( + /// Write `bytes` at `path` unless it is already occupied. `classify` + /// judges the occupant exactly as the matching read would: the same + /// content is a dedup hit, a valid object of another kind is refused + /// (nothing is written), and a missing, unreadable or corrupt occupant + /// is replaced atomically. + fn write_unless_occupied( &self, + hash: &Hash, path: &Path, bytes: &[u8], - is_same: impl FnOnce(Vec) -> bool, + classify: impl FnOnce(Vec) -> Occupant, ) -> Result<()> { if let Ok(stored) = self.read_at(path) { - if is_same(stored) { - return Ok(()); + match classify(stored) { + Occupant::Same => return Ok(()), + Occupant::Different => return Err(Error::AddressOccupied { hash: *hash }), + Occupant::Corrupt => {} } } self.write_at(path, bytes) @@ -223,8 +259,8 @@ impl ObjectStore for FsObjectStore { let bytes = serde_json::to_vec(object)?; let hash = object.hash(); let path = self.path_for(&hash); - self.write_unless_same(&path, &bytes, |stored| { - decode_typed(&hash, stored).is_ok_and(|existing| existing == *object) + self.write_unless_occupied(&hash, &path, &bytes, |stored| { + classify_typed(&hash, stored, object) })?; Ok(hash) } @@ -244,7 +280,7 @@ impl ObjectStore for FsObjectStore { let hash = Hash::of(bytes); let path = self.raw_path_for(&hash); - self.write_unless_same(&path, bytes, |stored| Hash::of(&stored) == hash)?; + self.write_unless_occupied(&hash, &path, bytes, |stored| classify_raw(&hash, &stored))?; Ok(hash) } @@ -629,4 +665,37 @@ mod tests { assert_eq!(store.get_raw(&hash).unwrap(), b); } + + // A valid object of another kind owns the address (a legacy Blob whose + // bytes are canonical Tree JSON): the Tree put is refused and the Blob + // stays readable. + #[test] + fn typed_put_refuses_a_valid_other_kind_occupant() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let tree = Tree::new(); + let tree_json = serde_json::to_vec(&tree).unwrap(); + let hash = tree.hash(); + let envelope = serde_json::to_vec(&Object::Blob(Blob::new(tree_json.clone()))).unwrap(); + store.write_at(&store.path_for(&hash), &envelope).unwrap(); + + match store.put(&Object::Tree(tree)) { + Err(Error::AddressOccupied { hash: h }) => assert_eq!(h, hash), + other => panic!("expected AddressOccupied, got {other:?}"), + } + match store.get(&hash).unwrap() { + Object::Blob(got) => assert_eq!(got.bytes, tree_json), + other => panic!("expected the Blob to survive, got {other:?}"), + } + } + + // Stores rooted at `

` and `

/raw` must not share a file. + #[test] + fn raw_paths_cannot_collide_with_typed_paths_of_a_raw_root() { + let dir = tempfile::tempdir().unwrap(); + let outer = FsObjectStore::open(dir.path()).unwrap(); + let inner = FsObjectStore::open(dir.path().join("raw")).unwrap(); + let h = Hash::of(b"x"); + assert_ne!(outer.raw_path_for(&h), inner.path_for(&h)); + } } diff --git a/crates/agenticd/src/wire_error.rs b/crates/agenticd/src/wire_error.rs index 127ca6a5..9a3a2f78 100644 --- a/crates/agenticd/src/wire_error.rs +++ b/crates/agenticd/src/wire_error.rs @@ -101,6 +101,9 @@ fn classify_core_error(err: &CoreError) -> (ErrorClass, &'static str, bool) { // Structural rejection of untrusted blob bytes; the caller must // change the content, so retrying can never succeed. CoreError::AmbiguousBlob { .. } => (ErrorClass::Validation, "ambiguous_blob", false), + // Same structural rejection: a valid object of another kind + // already owns this address, so the write can never succeed. + CoreError::AddressOccupied { .. } => (ErrorClass::Validation, "address_occupied", false), // Structural rejection of an untrusted branch name (path // traversal defence). The caller must change the name; retrying // the identical request can never succeed. @@ -435,6 +438,26 @@ mod tests { } } + #[test] + fn core_address_occupied_classifies_as_validation() { + let core_err: anyhow::Error = anyhow::Error::new(agentic_core::Error::AddressOccupied { + hash: agentic_core::Hash::of(b"x"), + }); + match map_anyhow_to_response_error(core_err) { + Response::Error { + class, + code, + retryable, + .. + } => { + assert_eq!(class, ErrorClass::Validation); + assert_eq!(code, "address_occupied"); + assert!(!retryable); + } + _ => panic!("expected Response::Error"), + } + } + #[test] fn core_secret_detected_classifies_as_validation() { // ADR-0013: SecretDetected is a structural rejection — the From 69799ce4006b902b33a4cd8e87de3afa8b100caf Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Fri, 20 Nov 2026 09:40:23 +0200 Subject: [PATCH 5/8] Treat unreadable occupants as errors and give each object write its own temp file The filesystem store replaced any occupant it failed to read, so a permission error on an existing object was handled like corruption and the object was overwritten. Only bytes that were read but fail to decode or verify count as corrupt now; any other read failure fails the put and leaves the file alone. Writes also shared one temp name per object path, which two stores with nested roots could collide on. Each write now creates its own temp file in the target directory, named with the process id and a counter, and removes it if the write fails. --- crates/agentic-core/src/store.rs | 107 +++++++++++++++++++++++++++---- 1 file changed, 95 insertions(+), 12 deletions(-) diff --git a/crates/agentic-core/src/store.rs b/crates/agentic-core/src/store.rs index 61046cf9..9089f767 100644 --- a/crates/agentic-core/src/store.rs +++ b/crates/agentic-core/src/store.rs @@ -15,7 +15,9 @@ use crate::object::{Blob, Commit, Object, ObjectKind, Tree}; use crate::scanner::{Allowlist, ScanPolicy, Scanner}; use crate::{Error, Result}; +use std::io::Write; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; /// Fail if `computed` doesn't match the content address `declared`. @@ -195,8 +197,10 @@ impl FsObjectStore { /// Write `bytes` at `path` unless it is already occupied. `classify` /// judges the occupant exactly as the matching read would: the same /// content is a dedup hit, a valid object of another kind is refused - /// (nothing is written), and a missing, unreadable or corrupt occupant - /// is replaced atomically. + /// (nothing is written), and a missing or corrupt occupant is replaced + /// atomically. "Corrupt" means the bytes were read but do not decode or + /// verify; a read that fails for any other reason (for example EACCES) + /// is an error, not licence to replace. fn write_unless_occupied( &self, hash: &Hash, @@ -204,12 +208,20 @@ impl FsObjectStore { bytes: &[u8], classify: impl FnOnce(Vec) -> Occupant, ) -> Result<()> { - if let Ok(stored) = self.read_at(path) { - match classify(stored) { - Occupant::Same => return Ok(()), - Occupant::Different => return Err(Error::AddressOccupied { hash: *hash }), - Occupant::Corrupt => {} + match std::fs::read(path) { + Ok(compressed) => { + let occupant = match zstd::stream::decode_all(&compressed[..]) { + Ok(stored) => classify(stored), + Err(_) => Occupant::Corrupt, + }; + match occupant { + Occupant::Same => return Ok(()), + Occupant::Different => return Err(Error::AddressOccupied { hash: *hash }), + Occupant::Corrupt => {} + } } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e.into()), } self.write_at(path, bytes) } @@ -218,12 +230,31 @@ impl FsObjectStore { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } - // Write to a temp file in the same directory and atomically rename, - // so a crash mid-write never leaves a torn object in the store. - let tmp = path.with_extension("tmp"); + // Write to a uniquely named temp file in the same directory and + // atomically rename, so a crash mid-write never leaves a torn object + // in the store and no other writer (including a store whose root + // nests this one) can share the temp file. + static NEXT_TMP: AtomicU64 = AtomicU64::new(0); + let name = path.file_name().unwrap_or_default().to_string_lossy(); + let tmp = path.with_file_name(format!( + "{name}.{}.{}.tmp", + std::process::id(), + NEXT_TMP.fetch_add(1, Ordering::Relaxed) + )); let compressed = zstd::stream::encode_all(bytes, 3)?; - std::fs::write(&tmp, &compressed)?; - std::fs::rename(&tmp, path)?; + let written = (|| { + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp)?; + file.write_all(&compressed)?; + drop(file); + std::fs::rename(&tmp, path) + })(); + if let Err(e) = written { + let _ = std::fs::remove_file(&tmp); + return Err(e.into()); + } Ok(()) } @@ -698,4 +729,56 @@ mod tests { let h = Hash::of(b"x"); assert_ne!(outer.raw_path_for(&h), inner.path_for(&h)); } + + // An occupant that cannot be read (here: no permission) is neither a + // dedup hit nor corruption; the put fails and the file is untouched. + #[cfg(unix)] + #[test] + fn unreadable_occupant_fails_the_put_and_is_left_alone() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let b = b"some prompt".to_vec(); + let hash = Hash::of(&b); + let path = store.path_for(&hash); + store.write_at(&path, b"occupant").unwrap(); + let before = std::fs::read(&path).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o000)).unwrap(); + if std::fs::read(&path).is_ok() { + eprintln!("running as root: permissions not enforced, skipping"); + return; + } + + let result = store.put(&Object::Blob(Blob::new(b))); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(matches!(result, Err(Error::Io(_))), "{result:?}"); + assert_eq!(std::fs::read(&path).unwrap(), before); + } + + // Writers never share a temp file, so a foreign file at the old shared + // temp name survives, and a failed write leaves no temp file behind. + #[test] + fn write_at_uses_private_temp_files() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let hash = Hash::of(b"x"); + let path = store.path_for(&hash); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let foreign = path.with_extension("tmp"); + std::fs::write(&foreign, b"someone else's temp").unwrap(); + + store.write_at(&path, b"payload").unwrap(); + assert_eq!(std::fs::read(&foreign).unwrap(), b"someone else's temp"); + assert_eq!(store.read_at(&path).unwrap(), b"payload"); + + // Rename onto a non-empty directory fails; the temp file must go. + let blocked = store.path_for(&Hash::of(b"y")); + std::fs::create_dir_all(blocked.join("child")).unwrap(); + assert!(store.write_at(&blocked, b"payload").is_err()); + let leftovers: Vec<_> = std::fs::read_dir(blocked.parent().unwrap()) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect(); + assert_eq!(leftovers.len(), 1, "temp file left behind: {leftovers:?}"); + } } From b170e025e107ecf0ffea81eed5d8b5c3b9b569fb Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Fri, 20 Nov 2026 09:52:41 +0200 Subject: [PATCH 6/8] Retry taken temp names instead of deleting them An AlreadyExists on the temp file belonged to another writer, and the cleanup removed it. Retry with the next name and clean up only a file this call created. --- crates/agentic-core/src/store.rs | 63 +++++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/crates/agentic-core/src/store.rs b/crates/agentic-core/src/store.rs index 9089f767..4d978539 100644 --- a/crates/agentic-core/src/store.rs +++ b/crates/agentic-core/src/store.rs @@ -20,6 +20,9 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; +/// Counter for [`FsObjectStore::write_at`] temp file names. +static NEXT_TMP: AtomicU64 = AtomicU64::new(0); + /// Fail if `computed` doesn't match the content address `declared`. /// /// The single chokepoint for the content-addressed integrity guarantee. @@ -234,19 +237,38 @@ impl FsObjectStore { // atomically rename, so a crash mid-write never leaves a torn object // in the store and no other writer (including a store whose root // nests this one) can share the temp file. - static NEXT_TMP: AtomicU64 = AtomicU64::new(0); let name = path.file_name().unwrap_or_default().to_string_lossy(); - let tmp = path.with_file_name(format!( - "{name}.{}.{}.tmp", - std::process::id(), - NEXT_TMP.fetch_add(1, Ordering::Relaxed) - )); let compressed = zstd::stream::encode_all(bytes, 3)?; - let written = (|| { - let mut file = std::fs::OpenOptions::new() + // Retry on a taken name (another process with the same pid in a + // different namespace); only a file this call created is ever removed. + let mut opened = None; + for _ in 0..16 { + let tmp = path.with_file_name(format!( + "{name}.{}.{}.tmp", + std::process::id(), + NEXT_TMP.fetch_add(1, Ordering::Relaxed) + )); + match std::fs::OpenOptions::new() .write(true) .create_new(true) - .open(&tmp)?; + .open(&tmp) + { + Ok(file) => { + opened = Some((tmp, file)); + break; + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(e.into()), + } + } + let Some((tmp, mut file)) = opened else { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!("no free temp file name next to {}", path.display()), + ) + .into()); + }; + let written = (|| { file.write_all(&compressed)?; drop(file); std::fs::rename(&tmp, path) @@ -781,4 +803,27 @@ mod tests { .collect(); assert_eq!(leftovers.len(), 1, "temp file left behind: {leftovers:?}"); } + + #[test] + fn write_at_skips_and_preserves_an_occupied_temp_name() { + let dir = tempfile::tempdir().unwrap(); + let store = FsObjectStore::open(dir.path()).unwrap(); + let path = store.path_for(&Hash::of(b"z")); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let name = path.file_name().unwrap().to_string_lossy().into_owned(); + let pid = std::process::id(); + // Occupy the next candidate names this process would pick. + let start = NEXT_TMP.load(Ordering::Relaxed); + let planted: Vec<_> = (start..start + 4) + .map(|n| path.with_file_name(format!("{name}.{pid}.{n}.tmp"))) + .collect(); + for p in &planted { + std::fs::write(p, b"other writer").unwrap(); + } + store.write_at(&path, b"payload").unwrap(); + for p in &planted { + assert_eq!(std::fs::read(p).unwrap(), b"other writer"); + } + assert_eq!(store.read_at(&path).unwrap(), b"payload"); + } } From 962a516b8384ee18faa6982c15104c5a8ebe9363 Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Fri, 20 Nov 2026 09:56:05 +0200 Subject: [PATCH 7/8] Refuse legacy ambiguous blobs before rollback side effects and record the namespace decision A store written before this change can hold a prompt, tool or model blob whose bytes are canonical Tree or Commit JSON. Rollback re-stages those blobs when it forward-records, and the new ambiguity check rejects them, but only after migrations, memory restore and prompt writes had changed live state. Rollback now loads and checks everything it will re-stage right after validating the target, dry runs included, and refuses with AmbiguousBlob before any side effect. The design is recorded in ADR-0018. GcsObjectStore::has now checks both cache namespaces before any network request. The temp-name collision test for FsObjectStore::write_at takes an injectable name counter, so it forces the collision instead of depending on the shared counter. --- README.md | 2 +- crates/agentic-core/src/gcs_store.rs | 30 ++++- crates/agentic-core/src/lib.rs | 2 +- crates/agentic-core/src/store.rs | 29 +++-- crates/agenticd/src/rollback/mod.rs | 33 +++++- crates/agenticd/tests/rollback_in_memory.rs | 104 +++++++++++++++++- .../adr/0018-separate-raw-object-namespace.md | 76 +++++++++++++ 7 files changed, 255 insertions(+), 21 deletions(-) create mode 100644 docs/adr/0018-separate-raw-object-namespace.md diff --git a/README.md b/README.md index d1649ae5..9f3c6f6d 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ git.agentic/ │ │ ├── roadmap.md 12-week build narrative; repo went public 2026-05-22 │ │ └── demo-scenario.md the canonical demo │ ├── adr/ -│ │ └── 0001 … 0017 — accepted architectural decision records +│ │ └── 0001 … 0018 — accepted architectural decision records │ └── architecture/ │ ├── overview.md system diagram and component boundaries │ └── snapshot-model.md the technical heart diff --git a/crates/agentic-core/src/gcs_store.rs b/crates/agentic-core/src/gcs_store.rs index da6954f4..b264513b 100644 --- a/crates/agentic-core/src/gcs_store.rs +++ b/crates/agentic-core/src/gcs_store.rs @@ -582,9 +582,13 @@ impl ObjectStore for GcsObjectStore { } fn has(&self, hash: &Hash) -> bool { - [Ns::Raw, Ns::Typed] - .into_iter() - .any(|ns| self.cache_path(hash, ns).exists() || self.remote_exists(hash, ns)) + // Both cache namespaces first, so a cached object answers without + // any network request; only then ask the bucket. + let namespaces = [Ns::Raw, Ns::Typed]; + namespaces + .iter() + .any(|&ns| self.cache_path(hash, ns).exists()) + || namespaces.iter().any(|&ns| self.remote_exists(hash, ns)) } } @@ -1083,4 +1087,24 @@ mod tests { assert_eq!(mock.requests().len(), 2, "{:?}", mock.requests()); assert!(!store.cache_path(&h, Ns::Raw).exists()); } + + // A cached object (either namespace) answers `has` with no request. + #[test] + fn has_checks_both_caches_before_the_network() { + let dir = tempdir().unwrap(); + let mock = Mock::start(|_| Resp::status(500)); + let store = mock.store(dir.path()); + let typed = Hash::of(b"typed"); + let raw = Hash::of(b"raw"); + store + .cache_write_compressed(&typed, Ns::Typed, &zst(b"t")) + .unwrap(); + store + .cache_write_compressed(&raw, Ns::Raw, &zst(b"r")) + .unwrap(); + + assert!(store.has(&typed)); + assert!(store.has(&raw)); + assert!(mock.requests().is_empty(), "{:?}", mock.requests()); + } } diff --git a/crates/agentic-core/src/lib.rs b/crates/agentic-core/src/lib.rs index 933f9c95..a16f5f3c 100644 --- a/crates/agentic-core/src/lib.rs +++ b/crates/agentic-core/src/lib.rs @@ -26,7 +26,7 @@ pub use gcs_store::{GcsObjectStore, DEFAULT_GCS_ENDPOINT}; pub use hash::Hash; pub use object::{Attestation, Blob, Commit, Object, ObjectKind, Tree, TypedRef}; pub use refs::{HeadRef, Refs, RefsSnapshot}; -pub use store::{FsObjectStore, ObjectStore}; +pub use store::{check_blob_unambiguous, FsObjectStore, ObjectStore}; /// Crate-level error type. Individual modules contribute their own variants. #[derive(Debug, thiserror::Error)] diff --git a/crates/agentic-core/src/store.rs b/crates/agentic-core/src/store.rs index 4d978539..e3589ad4 100644 --- a/crates/agentic-core/src/store.rs +++ b/crates/agentic-core/src/store.rs @@ -4,6 +4,8 @@ //! - Objects are zstd-compressed on disk. //! - First two hex chars of the BLAKE3 hash form the shard directory. //! - Filename is the remaining 62 hex chars plus `.zst`. +//! - Raw objects (`put_raw`) live apart, at `raw//.raw`; see +//! `docs/adr/0018-separate-raw-object-namespace.md`. //! //! `FsObjectStore` is the local-filesystem implementation; `GcsObjectStore` //! (see `gcs.rs`) is the remote backend introduced for the sidecar @@ -48,7 +50,7 @@ pub(crate) fn check_integrity(declared: &Hash, computed: Hash) -> Result<()> { /// `Commit`. Those objects hash `json(inner)`, so such a blob would share a /// content address with the typed object while being stored under a /// different envelope; whichever write came first would win the address. -pub(crate) fn check_blob_unambiguous(bytes: &[u8]) -> Result<()> { +pub fn check_blob_unambiguous(bytes: &[u8]) -> Result<()> { let canonical_tree = serde_json::from_slice::(bytes) .ok() .is_some_and(|t| serde_json::to_vec(&t).is_ok_and(|c| c == bytes)); @@ -230,6 +232,12 @@ impl FsObjectStore { } fn write_at(&self, path: &Path, bytes: &[u8]) -> Result<()> { + Self::write_at_with(path, bytes, || NEXT_TMP.fetch_add(1, Ordering::Relaxed)) + } + + /// [`Self::write_at`] with an injectable temp-name counter, so tests can + /// force a collision deterministically. + fn write_at_with(path: &Path, bytes: &[u8], mut next: impl FnMut() -> u64) -> Result<()> { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } @@ -243,11 +251,7 @@ impl FsObjectStore { // different namespace); only a file this call created is ever removed. let mut opened = None; for _ in 0..16 { - let tmp = path.with_file_name(format!( - "{name}.{}.{}.tmp", - std::process::id(), - NEXT_TMP.fetch_add(1, Ordering::Relaxed) - )); + let tmp = path.with_file_name(format!("{name}.{}.{}.tmp", std::process::id(), next())); match std::fs::OpenOptions::new() .write(true) .create_new(true) @@ -812,18 +816,23 @@ mod tests { std::fs::create_dir_all(path.parent().unwrap()).unwrap(); let name = path.file_name().unwrap().to_string_lossy().into_owned(); let pid = std::process::id(); - // Occupy the next candidate names this process would pick. - let start = NEXT_TMP.load(Ordering::Relaxed); - let planted: Vec<_> = (start..start + 4) + // Occupy the first four names the (fixed, deterministic) counter yields. + let planted: Vec<_> = (0..4) .map(|n| path.with_file_name(format!("{name}.{pid}.{n}.tmp"))) .collect(); for p in &planted { std::fs::write(p, b"other writer").unwrap(); } - store.write_at(&path, b"payload").unwrap(); + let mut counter = 0u64; + FsObjectStore::write_at_with(&path, b"payload", || { + counter += 1; + counter - 1 + }) + .unwrap(); for p in &planted { assert_eq!(std::fs::read(p).unwrap(), b"other writer"); } assert_eq!(store.read_at(&path).unwrap(), b"payload"); + assert_eq!(counter, 5, "names 0..4 were taken, 4 was the first free"); } } diff --git a/crates/agenticd/src/rollback/mod.rs b/crates/agenticd/src/rollback/mod.rs index 0075972e..10cf2b83 100644 --- a/crates/agenticd/src/rollback/mod.rs +++ b/crates/agenticd/src/rollback/mod.rs @@ -40,7 +40,7 @@ use std::path::PathBuf; use agentic_core::approval::{verify_token, ApprovalRejection}; use agentic_core::commit::{stage_and_commit, CommitInputs}; use agentic_core::refs::HeadRef; -use agentic_core::{Commit, Hash}; +use agentic_core::{check_blob_unambiguous, Commit, Hash}; use agentic_memory::adapter::SnapshotHandle; use agentic_memory::MemoryAdapter; use anyhow::{anyhow, Context}; @@ -263,6 +263,21 @@ pub async fn execute( preflight_prompts(&state, &args.repo, &prompts_hash)?; } + // Load everything the forward-record step will re-stage, and refuse + // before any side effect (migration, memory restore, prompt write; dry + // runs included) if a blob would be rejected as ambiguous. A pre-upgrade + // store can hold a prompt or tool blob whose bytes are canonical Tree or + // Commit JSON; staging it now fails with `AmbiguousBlob`, which must not + // happen after live state has already been rolled back. + let prompts_payload = read_text_blobs(&state, target.prompts)?; + let tools_payload = read_text_blobs(&state, target.tools)?; + let model_text = read_model_text(&state, &target)?; + ensure_unambiguous("prompts", &prompts_payload)?; + ensure_unambiguous("tools", &tools_payload)?; + if let Some(model) = &model_text { + check_blob_unambiguous(model.as_bytes()).context("target model blob")?; + } + let mut plan: Vec = Vec::new(); let branch = match state.refs.read_head()? { @@ -427,10 +442,6 @@ pub async fn execute( // Build a fresh Commit whose dimensions match the target but whose // parent is the current branch tip. History shows C → C' even though // C' contains A's state. - let prompts_payload = read_text_blobs(&state, target.prompts)?; - let tools_payload = read_text_blobs(&state, target.tools)?; - let model_text = read_model_text(&state, &target)?; - let inputs = CommitInputs { author: "agentic-rollback".to_string(), message: format!("Rollback to {}", target_hash.short()), @@ -464,6 +475,18 @@ pub async fn execute( }) } +/// Refuse a tree of blobs that forward-record would re-stage as ambiguous. +fn ensure_unambiguous( + what: &str, + blobs: &std::collections::BTreeMap>, +) -> anyhow::Result<()> { + for (name, bytes) in blobs { + check_blob_unambiguous(bytes) + .with_context(|| format!("target {what} entry {name:?} cannot be re-staged"))?; + } + Ok(()) +} + /// Reject target Commits whose `(memory_snapshot, schema_version)` pair is /// in a shape the daemon should not have produced. /// diff --git a/crates/agenticd/tests/rollback_in_memory.rs b/crates/agenticd/tests/rollback_in_memory.rs index d06bc60a..47de6443 100644 --- a/crates/agenticd/tests/rollback_in_memory.rs +++ b/crates/agenticd/tests/rollback_in_memory.rs @@ -10,7 +10,7 @@ use std::sync::Arc; use agentic_core::refs::Refs; -use agentic_core::{FsObjectStore, ObjectStore}; +use agentic_core::{FsObjectStore, Hash, Object, ObjectStore, Tree, TypedRef}; use agentic_memory::in_memory::InMemoryAdapter; use agentic_memory::MemoryAdapter; use agentic_proto::CommitInput; @@ -774,3 +774,105 @@ async fn rollback_refuses_invalid_prompt_name_before_side_effects() { ) .await; } + +// --------------------------------------------------------------------------- +// Pre-upgrade stores: a prompt blob whose bytes are canonical Tree JSON. +// --------------------------------------------------------------------------- + +/// Store `bytes` the way a pre-upgrade store did: bare bytes at the typed +/// path (`//.zst`), which the current code reads as a +/// blob. `put_raw` already produces that exact compressed file, under +/// `raw/`, so copy it into the legacy place. +fn plant_legacy_blob(objects: &std::path::Path, store: &dyn ObjectStore, bytes: &[u8]) -> Hash { + let hash = store + .put_raw(agentic_core::ObjectKind::Blob, bytes) + .unwrap(); + let (a, b) = hash.shard(); + let from = objects.join("raw").join(&a).join(format!("{b}.raw")); + let to = objects.join(&a).join(format!("{b}.zst")); + std::fs::create_dir_all(to.parent().unwrap()).unwrap(); + std::fs::copy(from, to).unwrap(); + hash +} + +#[tokio::test] +async fn rollback_refuses_legacy_ambiguous_blob_before_any_side_effect() { + let (state, adapter, baseline, _dir) = setup_gate_state(false).await; + let agentic_dir = state.repo_root.join(".agentic"); + + // A target commit whose prompts tree holds a legacy blob that is + // exactly the canonical JSON of a Tree. + let tree_json = serde_json::to_vec(&Tree::new()).unwrap(); + let blob_hash = plant_legacy_blob( + &agentic_dir.join("objects"), + state.store.as_ref(), + &tree_json, + ); + let mut prompts = Tree::new(); + prompts.insert( + "system.md", + TypedRef { + kind: agentic_core::ObjectKind::Blob, + hash: blob_hash, + }, + ); + let prompts_hash = state.store.put(&Object::Tree(prompts)).unwrap(); + let baseline_hash = state.refs.resolve(&baseline).unwrap().unwrap(); + let Object::Commit(mut target) = state.store.get(&baseline_hash).unwrap() else { + panic!("baseline is a commit"); + }; + target.prompts = Some(prompts_hash); + target.message = "legacy ambiguous prompt".into(); + let target_hash = state.store.put(&Object::Commit(target)).unwrap(); + + // Contaminate live state so a rollback that proceeded would be visible. + adapter.apply_migration("002_bump").await; + adapter + .insert_rows( + "messages", + vec![serde_json::json!({"id": 99, "body": "bad"})], + ) + .await; + let schema_dir = agentic_dir.join("schema"); + std::fs::create_dir_all(&schema_dir).unwrap(); + std::fs::write( + schema_dir.join("002_bump.down.sql"), + "-- no-op for fixture\n", + ) + .unwrap(); + let tip = state.refs.read_branch("main").unwrap(); + + for dry_run in [true, false] { + let err = rollback::execute( + Arc::clone(&state), + RollbackArgs { + target: target_hash.to_hex(), + dry_run, + accept_data_loss: false, + approval_token: None, + repo: state.repo_root.clone(), + }, + None, + ) + .await + .expect_err("a legacy ambiguous blob must be refused"); + assert!( + err.chain().any(|e| matches!( + e.downcast_ref::(), + Some(agentic_core::Error::AmbiguousBlob { .. }) + )), + "expected AmbiguousBlob (dry_run={dry_run}), got: {err:#}" + ); + + assert_eq!( + adapter.current_schema_version().await.unwrap(), + "002_bump", + "schema must not be reversed (dry_run={dry_run})" + ); + let rows = adapter.rows_of("messages").await; + assert_eq!(rows.len(), 1, "memory must not be restored"); + assert_eq!(rows[0]["body"], "bad"); + assert_eq!(state.refs.read_branch("main").unwrap(), tip); + assert!(!state.repo_root.join("prompts").exists()); + } +} diff --git a/docs/adr/0018-separate-raw-object-namespace.md b/docs/adr/0018-separate-raw-object-namespace.md new file mode 100644 index 00000000..015254b4 --- /dev/null +++ b/docs/adr/0018-separate-raw-object-namespace.md @@ -0,0 +1,76 @@ +# ADR-0018: Separate Namespace for Raw Objects + +Status: Accepted +Owner: Toni Bergholm +Date: 2026-11-20 +Amends: ADR-0002 (object store layout) + +## Context + +The object store keeps two kinds of content under one address space. +Typed objects (`put`, `get`) are stored as a JSON envelope tagged with their kind. +Raw objects (`put_raw`, `get_raw`), used for memory segments and segment manifests, are stored as the bare bytes. +Blobs hash their bytes directly, so a Blob and a raw object with the same bytes have the same address. +Trees and Commits hash `json(inner)` but are stored in the tagged envelope, so a Blob whose bytes equal that JSON shares an address with the Tree or Commit. + +Whoever wrote first owned the address. +The filesystem store skipped a write when the path existed, and the GCS store uploaded unconditionally and could overwrite. +A client that controls prompt blob bytes can predict segment bytes, because they derive from rows it wrote. +It could therefore occupy a raw object's address with a blob envelope. +A later honest `put_raw` then did nothing, and `get_raw` failed its integrity check. +The commit landed but could never be restored. + +## Decision 1: raw objects live in their own namespace + +Raw objects are stored apart from typed objects, with a distinct suffix, so the two can never share a path or object name, even for stores whose roots or prefixes nest. + +- Filesystem: `/raw//.raw`. Typed objects stay at `//.zst`. +- GCS: object name `/raw//.raw.zst`, with the local cache at `/raw//.raw.zst`. Typed objects keep their current names. + +`get_raw` falls back to the legacy shared path when the raw path is missing, so existing stores keep reading. +A legacy address may hold bare raw bytes or a blob envelope of the same bytes, and both are accepted. +`get` accepts bare raw bytes at a typed address as a Blob with the same content. +Every read is still verified against its address. +`has` is true if either namespace holds the object. + +## Decision 2: reject ambiguous blobs + +A Blob whose bytes are exactly the canonical JSON of a Tree or a Commit is rejected with `Error::AmbiguousBlob`. +That is the only way a Blob can share an address with a typed object. +The daemon maps it to a validation error (`ambiguous_blob`, not retryable), the same class as `SecretDetected`. + +## Decision 3: verify an occupied address, never trust it + +A put that finds its address occupied reads the occupant exactly as the matching read would, and classifies it. + +- Same content: a dedup hit, nothing is written. +- A valid object of another kind: the put fails with `Error::AddressOccupied` (a validation error on the wire, `address_occupied`, not retryable). Nothing is written or cached. +- Corrupt, meaning the bytes were read but do not decode or verify: the occupant is replaced. + +On the filesystem the replacement is atomic (unique temp file, then rename). A read failure other than "not found" is an error, not corruption. +On GCS every upload sends `ifGenerationMatch=0`, so an existing object is never overwritten blindly. HTTP 412 means the name is taken, and the occupant is downloaded and classified. +A corrupt occupant is replaced with `ifGenerationMatch=`. If the download reports no generation, or the object changed meanwhile, the put fails. + +## Decision 4: the cache follows the bucket + +The GCS local cache is written only once the bucket is known to hold the verified object: after a successful upload, or after a 412 whose occupant classified as the same content. +A failed or refused put leaves no cache entry. +`get_raw` and `has` consult the hash-verified cache under both names before any network request. + +## Decision 5: rollback refuses legacy ambiguous blobs up front + +A pre-upgrade store can already hold a prompt, tool or model blob whose bytes are canonical Tree or Commit JSON. +Rollback forward-records by re-staging those blobs, which would now fail with `AmbiguousBlob` after migrations, memory restore and prompt writes had already changed live state. +Rollback therefore loads everything it will re-stage right after validating the target, and refuses with `AmbiguousBlob` before any side effect, dry runs included. + +## Consequences + +- A dedup hit on GCS costs two requests (the conditional upload and a download) where it used to cost one. A new object still costs one. +- Existing stores keep working without migration. New raw objects land in the new namespace, and old ones are read through the legacy fallback. +- Legacy ambiguous blobs stay in the store. They are readable, but a rollback to a commit that references one is refused with a clear error before anything changes. +- A legacy bare raw object whose bytes equal canonical Tree or Commit JSON could still block a later put of that Tree or Commit. This is not realistic for segments and manifests. + +## Alternatives considered + +Kind-domain-separated addressing, where the kind is part of the hash input, would remove the whole class of collision. +It changes every address and the Commit wire contract, so it needs its own ADR and a store migration, and it is out of scope here. From 91ef7dcc35f1df560e7dccde9c15ab280d4f648c Mon Sep 17 00:00:00 2001 From: Toni Bergholm Date: Fri, 20 Nov 2026 10:04:09 +0200 Subject: [PATCH 8/8] State the legacy squat case and its repair in ADR-0018 --- docs/adr/0018-separate-raw-object-namespace.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/adr/0018-separate-raw-object-namespace.md b/docs/adr/0018-separate-raw-object-namespace.md index 015254b4..8b1b6292 100644 --- a/docs/adr/0018-separate-raw-object-namespace.md +++ b/docs/adr/0018-separate-raw-object-namespace.md @@ -68,7 +68,8 @@ Rollback therefore loads everything it will re-stage right after validating the - A dedup hit on GCS costs two requests (the conditional upload and a download) where it used to cost one. A new object still costs one. - Existing stores keep working without migration. New raw objects land in the new namespace, and old ones are read through the legacy fallback. - Legacy ambiguous blobs stay in the store. They are readable, but a rollback to a commit that references one is refused with a clear error before anything changes. -- A legacy bare raw object whose bytes equal canonical Tree or Commit JSON could still block a later put of that Tree or Commit. This is not realistic for segments and manifests. +- A legacy occupant that is a valid object of another kind blocks a later put at its address with a non-retryable `address_occupied`. The realistic case is a pre-upgrade prompt or tool blob whose bytes are the canonical JSON of a predictable Tree: every later commit that produces that Tree fails until the occupant is removed. A bare raw segment or manifest cannot equal Tree or Commit JSON, so it does not cause this. +- Repair: the error names the address. An operator who confirms the occupant is not referenced by a commit they need deletes that one object file (or GCS object) and retries the commit. ## Alternatives considered