diff --git a/.env.example b/.env.example index 81c60824d..75fce06c5 100644 --- a/.env.example +++ b/.env.example @@ -24,11 +24,26 @@ DATABASE_URL=postgresql://gitlawb:changeme@localhost:5432/gitlawb # ── Database pool & startup resilience ──────────────────────────────────── # Maximum connections in the PostgreSQL pool. A cap, not a floor — # connections open lazily. Size against the DB server's max_connections, -# remembering admin tooling opens its own pool. Each concurrent write pins one -# connection for its whole duration (the connection-affine advisory lock), so the -# node REJECTS at boot any value below GITLAWB_MAX_CONCURRENT_GIT_PUSHES + 8 -# headroom — keep this comfortably above that (default 48 for pushes 32). +# remembering admin tooling opens its own pool. GITLAWB_DB_MAX_CONNECTIONS=48 +# Maximum connections in the DEDICATED advisory-lock pool, separate from the +# pool above. Every in-flight repo write pins one connection here for its whole +# duration, so this is a hard ceiling on simultaneous writes node-wide: size it +# to expected peak concurrent writers, not small. Keeping it separate is what +# stops a push burst from starving ordinary request handlers. Budget +# (GITLAWB_DB_MAX_CONNECTIONS + this) per node against the server's +# max_connections, times node count, plus admin tooling. +GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS=40 +# Upper bound, in seconds, on any object-storage transfer that runs while a +# per-repo write lock is HELD (the archive download inside acquire_write and the +# upload inside release). These were free before the lock's connection was +# pinned to the guard; now an unbounded stall holds a lock-pool slot, and enough +# stalls deny every write on the node. The bound applies PER SPAN and there are +# two (the acquire-side refresh, which covers the existence check and download +# together, and the release-side upload), so worst-case slot occupancy is about +# twice this value plus the git work between them. Read it together with the pool +# size above and with GITLAWB_GIT_SERVICE_TIMEOUT_SECS. +GITLAWB_LOCK_HELD_TRANSFER_TIMEOUT_SECS=300 # Seconds a request waits for a pool connection before failing with 503. GITLAWB_DB_ACQUIRE_TIMEOUT_SECS=5 # Upper bound on each startup connect+migrate attempt, in seconds. Keep it diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 92d129803..849ea0259 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -3758,9 +3758,13 @@ mod tests { // deterministically. let endpoint = crate::test_support::silent_http_endpoint().await; let tigris = - crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) - .await; - state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir, + Some(tigris), + pool, + std::time::Duration::from_secs(300), + ); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; let mut cfg = (*state.config).clone(); cfg.git_acquire_timeout_secs = 1; @@ -3831,9 +3835,13 @@ mod tests { // (endpoint-pinned test client, no AWS_* env reads). let endpoint = crate::test_support::silent_http_endpoint().await; let tigris = - crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) - .await; - state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir, + Some(tigris), + pool, + std::time::Duration::from_secs(300), + ); let mut cfg = (*state.config).clone(); cfg.git_acquire_timeout_secs = 1; state.config = Arc::new(cfg); @@ -7956,9 +7964,18 @@ mod tests { // (endpoint-pinned test client, no AWS_* env reads). let endpoint = crate::test_support::silent_http_endpoint().await; let tigris = - crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) - .await; - state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir, + Some(tigris), + pool, + std::time::Duration::from_secs(300), + ); + state + .db + .upsert_mirror_repo("z6f3budget", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); let mut cfg = (*state.config).clone(); cfg.ipfs_request_budget_secs = 1; cfg.git_acquire_timeout_secs = 2; diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 0eacfa724..5150c2e09 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -61,20 +61,30 @@ pub async fn create_issue( let json_str = serde_json::to_string(&issue) .map_err(|e| AppError::BadRequest(format!("serialization error: {e}")))?; - // Shed 503 + Retry-After on an exhausted write-lock POOL instead of a generic - // git 500 (#173 F1). This path holds no admission permit, so it reaches the pool - // unthrottled; reuse the push handler's mapping so the two cannot drift. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) .await .map_err(|e| crate::api::repos::acquire_write_app_error(&e, &repo))?; - let disk_path = guard.path().to_path_buf(); - let create_result = git_issues::create_issue(&disk_path, &issue_id, &json_str); + let create_result = git_issues::create_issue(guard.path(), &issue_id, &json_str); - // Always release the advisory lock — even on error; upload to Tigris only on success. - guard.release(create_result.is_ok()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let deadline = std::time::Instant::now() + git_timeout; + let git_bin = state.git_bin.clone(); + let issue_id_for_comp = issue_id.clone(); + + let release_result = if create_result.is_ok() { + guard + .release_compensating(true, move |path| { + git_issues::delete_issue_ref(&git_bin, path, &issue_id_for_comp, deadline) + }) + .await + .into_result() + } else { + guard.release(false).await.into_result() + }; + release_result?; create_result.map_err(|e| AppError::Git(e.to_string()))?; @@ -225,6 +235,8 @@ pub async fn close_issue( State(state): State, Extension(auth): Extension, Path((owner, repo, issue_id)): Path<(String, String, String)>, + headers: axum::http::HeaderMap, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, ) -> Result> { let record = state .db @@ -232,8 +244,161 @@ pub async fn close_issue( .await? .ok_or_else(|| AppError::RepoNotFound(format!("{owner}/{repo}")))?; - // Same capacity shed as create_issue above (#173 F1): an exhausted write-lock - // pool is a 503 + Retry-After, not a 500 git error. + // READ-GATE before any snapshot work or rate charging. The author fallback below + // needs the issue blob, which needs the repo tree, so authorship cannot be established + // without a download; but a caller who cannot even READ the repo must be + // stopped here, cheaply, before any Tigris transfer or extraction happens. + // Without this, any signed non-owner could issue parallel close requests for + // arbitrary issue ids and drive unbounded downloads and blocking extraction + // (a disposable-identity DoS), because the route has no other pre-authorization. + // + // Rate limiting runs AFTER this gate so a 429 cannot distinguish a hidden repo + // from a missing one (INV-12 read-denial status contract). + // + // mirror-rows-handled: a repo row synced from a peer is stored public and + // carries none of the owner's visibility rules, so for such a row this check + // can only return allow. That is deliberate rather than overlooked, and it is + // the same verdict every other read gate in this API reaches for one. Refusing + // instead would deny the repo's real owner and the issue's real author on any + // node whose only copy of the repo is a synced one, which is an ordinary state + // here, and it would not buy the protection it appears to, because a synced + // row's recorded owner comes from the peer that sent it. The expensive work + // this check guards is bounded ahead of it by the per-IP limiter and the read + // pool slot taken below, and the authoritative owner-or-author decision still + // runs below and again under the write lock. + { + // mirror-rows-handled: synced mirror rows carry no owner rules; read gate only. + let rules = state.db.list_visibility_rules(&record.id).await?; + let caller = auth.0.as_str(); + if crate::visibility::visibility_check( + &rules, + record.is_public, + &record.owner_did, + Some(caller), + "/", + ) == crate::visibility::Decision::Deny + { + return Err(AppError::RepoNotFound(format!("{owner}/{repo}"))); + } + } + + // Per-IP flood brake, layered on the same shared limiter and trusted-proxy + // policy as the push advertisement. The pre-lock snapshot downloads the + // whole archive and runs a blocking extraction, so an unlimited route would + // let disposable identities drive unbounded transfer/CPU/disk with parallel + // close requests for arbitrary issue ids. Applied after the read gate so a + // denied reader still sees 404, not 429. + if let Some(key) = crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust) { + if !state.close_issue_rate_limiter.check(&key).await { + tracing::warn!(repo = %repo, key = %key, "close_issue rate limited"); + return Err(AppError::TooManyRequests( + "rate limit exceeded — try again later".into(), + )); + } + } + + // AUTHORIZE BEFORE ACQUIRING. The per-repo advisory lock genuinely excludes + // now, so taking it first would hand any caller with read access a way to hold + // that lock on demand and be refused afterwards, while a legitimate writer + // burned its retry budget against it. On a public repo that is every + // permissionless identity. The lock must not be reachable by a caller who is + // about to be refused the write. + let is_owner = crate::api::require_repo_owner(&record, &auth.0).is_ok(); + if !is_owner { + // Cap concurrent snapshot work on the shared read pool. The hourly rate + // bucket above is not a concurrent-work brake; without this, parallel close + // attempts from read-capable callers could each drive a full archive + // download and blocking extraction before the author denial below. + let _read_permit = state + .git_read_semaphore + .clone() + .try_acquire_owned() + .map_err(|_| { + tracing::warn!( + repo = %repo, + "close_issue snapshot refused — git read pool at capacity" + ); + AppError::Overloaded("git service at capacity, retry shortly".into()) + })?; + + // Not the owner, so the author fallback decides it, and the author lives in + // the issue's git-JSON blob rather than a DB column. + // + // Read it WITHOUT the write lock, from a NON-MUTATING SNAPSHOT. The + // justification is NOT that authorship is immutable — it is not: + // `refs/gitlawb/**` is pushable, so a forged author blob can be pushed + // (tracked separately; it is what makes this fallback only as trustworthy + // as push authorization). The justification is that this read is only a + // PRE-CHECK, deciding whether to take the lock at all. It is NOT the + // authorization decision: `acquire_write` re-downloads the archive after + // locking, so the tree that gets mutated is routinely not this one, and the + // authoritative owner-or-author check runs again under the guard below. + // Refusing here early just keeps a caller who is already visibly + // unauthorized from reaching the lock. + // + // `read_snapshot`, not `acquire_fresh`: acquire's fast path returns as soon + // as the directory exists and never contacts object storage, so on a node + // with a stale copy the author's own issue would be invisible and the + // cannot-establish-authorship arm below would 403 a legitimate author. + // read_snapshot refreshes the same way, but unpacks into a throwaway temp + // dir instead of publishing into the live repo path — an unlocked + // pre-check must not delete or swap the directory under a concurrent + // guarded write on the same path. + // Pre-lock authorization only: bound with the read acquire budget, not the + // under-lock transfer timeout a guarded write may hold for minutes. + let snapshot_bound_secs = state.config.git_acquire_timeout_secs; + let snapshot = tokio::time::timeout( + std::time::Duration::from_secs(snapshot_bound_secs), + state + .repo_store + .read_snapshot(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!( + repo = %repo, + bound_secs = snapshot_bound_secs, + "close_issue snapshot exceeded the read acquire bound — shedding as a retryable refusal" + ); + AppError::RepoUnavailable + })??; + let snapshot_path = snapshot.path().to_path_buf(); + + let author_did: Option = match git_issues::get_issue(&snapshot_path, &issue_id) { + Ok(Some(raw)) => serde_json::from_str::(&raw) + .ok() + .and_then(|i| i.author), + // Cannot establish authorship, so fail closed. Deliberately 403 rather + // than 404 for a non-owner: a caller who is not authorized to write + // should not learn from this route whether the issue exists. Both arms + // below return None; they are split only so a read failure is visible + // to operators, since a genuinely absent issue and an unreadable one + // are the same answer to the client but not the same event. + Ok(None) => None, + Err(e) => { + tracing::warn!( + repo = %repo, + issue = %issue_id, + err = %e, + "get_issue failed during close_issue authorship pre-check" + ); + None + } + }; + let is_author = author_did + .as_deref() + .is_some_and(|a| crate::api::did_matches(&auth.0, a)); + if !is_author { + return Err(AppError::Forbidden( + "only the repo owner or the issue author can close this issue".into(), + )); + } + } + + // Authorized. Only now is the lock taken. + // Propagate rather than stringify: AppError's From downcasts to + // sqlx::Error so a pool timeout or a database outage surfaces as a retryable + // 503. Calling .to_string() first destroys that and reports both as a 500. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) @@ -241,37 +406,56 @@ pub async fn close_issue( .map_err(|e| crate::api::repos::acquire_write_app_error(&e, &repo))?; let disk_path = guard.path().to_path_buf(); - // Owner OR issue author may close. The author lives in the issue's git-JSON - // blob (not a DB column); a None author (legacy issues) falls back to - // owner-only. Read it under the write guard, before mutating. - let author_did: Option = match git_issues::get_issue(&disk_path, &issue_id) { - Ok(Some(raw)) => serde_json::from_str::(&raw) - .ok() - .and_then(|i| i.author), + // Re-read under the guard and RE-AUTHORIZE against what we read, rather than + // only confirming the issue still exists. The pre-lock read decided whether to + // take the lock; it cannot be the authorization decision, because acquire_write + // re-downloads the archive after locking, so this is frequently a different tree + // than the one the author was read from. Checking existence alone would leave the + // whole decision resting on the earlier read of a tree we are no longer looking + // at. The blob is already in hand here, so this costs a deserialize. + match git_issues::get_issue(&disk_path, &issue_id) { + Ok(Some(raw)) => { + let author_now: Option = serde_json::from_str::(&raw) + .ok() + .and_then(|i| i.author); + let is_author_now = author_now + .as_deref() + .is_some_and(|a| crate::api::did_matches(&auth.0, a)); + if !is_owner && !is_author_now { + // Consumed, NOT propagated, and that is deliberate at all three + // `release(false)` sites below. These release without + // publishing, so there is nothing for the store to refuse, and + // mapping the outcome here would let a 503 shadow the + // authorization answer this route exists to give. + let _ = guard.release(false).await; + return Err(AppError::Forbidden( + "only the repo owner or the issue author can close this issue".into(), + )); + } + } Ok(None) => { - guard.release(false).await; - return Err(AppError::NotFound(format!("issue {issue_id} not found"))); + let _ = guard.release(false).await; + // The owner keeps the informative 404; a non-owner must not learn from + // this route whether the issue exists, matching the pre-check above. + return Err(if is_owner { + AppError::NotFound(format!("issue {issue_id} not found")) + } else { + AppError::Forbidden( + "only the repo owner or the issue author can close this issue".into(), + ) + }); } Err(e) => { - guard.release(false).await; + let _ = guard.release(false).await; return Err(AppError::Git(e.to_string())); } - }; - let is_owner = crate::api::require_repo_owner(&record, &auth.0).is_ok(); - let is_author = author_did - .as_deref() - .is_some_and(|a| crate::api::did_matches(&auth.0, a)); - if !is_owner && !is_author { - guard.release(false).await; - return Err(AppError::Forbidden( - "only the repo owner or the issue author can close this issue".into(), - )); } let close_result = git_issues::close_issue(&disk_path, &issue_id); // Always release the advisory lock — even on error; upload to Tigris only on success. - guard.release(close_result.is_ok()).await; + // Same short-circuit as create_issue, and before the 200 body below. + guard.release(close_result.is_ok()).await.into_result()?; let updated = close_result .map_err(|e| AppError::Git(e.to_string()))? @@ -285,12 +469,344 @@ pub async fn close_issue( Ok(Json(issue)) } -/// #173 F1 follow-up: the two issue write paths reach `acquire_write` holding NO -/// admission permit (unlike the push handler, which is capped by the git-push -/// semaphore), so they are the callers most likely to meet an exhausted write-lock -/// POOL under load. An exhausted pool is a capacity signal, so both must shed -/// 503 + Retry-After (`AppError::Overloaded`) the way the push handler does, not -/// report the generic 500 git error that says nothing about retrying. +#[cfg(test)] +mod tests { + use super::*; + use sqlx::PgPool; + + /// U7: once the advisory lock actually excludes, taking it BEFORE authorizing + /// turns close_issue into a wedge primitive. Any caller with repo read access + /// (on a public repo, any permissionless identity) could take the per-repo + /// write lock on demand and be refused the write afterwards, while the owner's + /// push burned its retry budget against a lock held by someone with no write + /// authorization. + /// + /// The observable: hold the lock from an independent session, then call the + /// handler as a stranger. If it authorizes first it refuses immediately; if it + /// acquires first it sits in the 60-attempt retry loop and the deadline fires. + #[sqlx::test] + async fn stranger_is_refused_without_waiting_on_the_write_lock(pool: PgPool) { + use sqlx::Connection; + let opts = (*pool.connect_options()).clone(); + let state = crate::test_support::test_state(pool.clone()).await; + + let owner = "did:key:z6MkU7Owner"; + state + .db + .upsert_mirror_repo("z6MkU7Owner", "u7repo", "/tmp/u7repo", None, true) + .await + .expect("seed repo"); + let record = state + .db + .get_repo("z6MkU7Owner", "u7repo") + .await + .expect("get_repo") + .expect("repo exists"); + + // An independent session holds the repo's write lock for the whole call. + let key = crate::git::repo_store::advisory_lock_key_for_test( + &record.owner_did.replace([':', '/'], "_"), + &record.name, + ); + let mut holder = sqlx::PgConnection::connect_with(&opts).await.unwrap(); + let held: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut holder) + .await + .unwrap(); + assert!( + held.0, + "the test must hold the lock for this to mean anything" + ); + let _ = owner; + + let stranger = crate::auth::AuthenticatedDid("did:key:z6MkU7Stranger".to_string()); + let outcome = tokio::time::timeout( + std::time::Duration::from_secs(3), + close_issue( + axum::extract::State(state.clone()), + axum::Extension(stranger), + axum::extract::Path(( + "z6MkU7Owner".to_string(), + "u7repo".to_string(), + "1".to_string(), + )), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(Some("203.0.113.64:5000".parse().unwrap())), + ), + ) + .await; + + let refused = outcome.expect( + "a caller with no write authorization must be refused WITHOUT waiting on the \ + write lock; hitting this deadline means the handler tried to acquire first, \ + which is the wedge primitive", + ); + assert!( + matches!(refused, Err(AppError::Forbidden(_))), + "expected 403 Forbidden for a stranger, got {:?}", + refused.err().map(|e| format!("{e:?}")) + ); + } + + /// The read-gate's blind spot, pinned rather than left to be rediscovered. + /// + /// A repo row synced from a peer is stored public and carries none of the + /// owner's visibility rules, so the gate's own inputs can only produce allow + /// for it. The gate above therefore does not carry this class of row, and the + /// test that does cover it (`non_reader_is_refused_before_the_snapshot`) seeds + /// a locally created repo, which cannot observe this: a passing test there is + /// not coverage here. + /// + /// Two things are asserted, and the second is why the first is acceptable. + /// The gate's verdict for such a row is allow for an arbitrary caller, and the + /// handler still refuses that caller afterwards, because the decision that + /// matters is the owner-or-author check rather than this one. If a later change + /// makes the gate the load-bearing decision for this route, the first assertion + /// breaks and this comment is where to start. + #[sqlx::test] + async fn a_synced_row_is_not_gated_by_its_own_visibility(pool: PgPool) { + let state = crate::test_support::test_state(pool.clone()).await; + + // Only a synced row exists for this repo, with no locally created twin. + state + .db + .upsert_mirror_repo("z6MkSyncOwner", "syncrepo", "/tmp/syncrepo", None, false) + .await + .expect("seed synced repo"); // false = not quarantined, the ordinary case + let record = state + .db + .get_repo("z6MkSyncOwner", "syncrepo") + .await + .expect("get_repo") + .expect("repo exists") + .clone(); + assert!( + record.id.contains('/'), + "this test is only meaningful against a synced row; got id {}", + record.id + ); + + // The gate's two inputs, and what they force. + let rules = state + .db + .list_visibility_rules(&record.id) + .await + .expect("list rules"); + assert!(rules.is_empty(), "a synced row carries no rules of its own"); + assert!(record.is_public, "a synced row is stored public"); + assert_eq!( + crate::visibility::visibility_check( + &rules, + record.is_public, + &record.owner_did, + Some("did:key:z6MkSyncStranger"), + "/", + ), + crate::visibility::Decision::Allow, + "the gate can only allow for a synced row, which is the property the \ + handler's mirror-rows-handled note records", + ); + + // So the refusal has to come from the decision that is actually load-bearing. + let stranger = crate::auth::AuthenticatedDid("did:key:z6MkSyncStranger".to_string()); + let outcome = close_issue( + axum::extract::State(state.clone()), + axum::Extension(stranger), + axum::extract::Path(( + "z6MkSyncOwner".to_string(), + "syncrepo".to_string(), + "1".to_string(), + )), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(Some("203.0.113.99:5000".parse().unwrap())), + ) + .await; + assert!( + outcome.is_err(), + "a stranger must still be refused on a synced row, gate or no gate", + ); + let body = format!("{:?}", outcome.err().unwrap()); + assert!( + !body.contains("syncrepo/") && !body.to_lowercase().contains("issue body"), + "the refusal must not leak repo contents: {body}", + ); + } + + /// The read-gate added for the pre-lock snapshot: a caller who cannot READ + /// the repo (private repo, no rule granting them access) must be refused with + /// a not-found BEFORE any snapshot download or extraction happens. The + /// observable is the refusal itself; the cheaper part (no Tigris work) is + /// structural (the gate precedes the snapshot call in the handler). + #[sqlx::test] + async fn non_reader_is_refused_before_the_snapshot(pool: PgPool) { + let state = crate::test_support::test_state(pool.clone()).await; + let now = chrono::Utc::now(); + state + .db + .create_repo(&crate::db::RepoRecord { + id: uuid::Uuid::new_v4().to_string(), + name: "priv-close".to_string(), + owner_did: "z6MkT3Owner".to_string(), + description: None, + is_public: false, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: "/tmp/priv-close".to_string(), + forked_from: None, + machine_id: None, + }) + .await + .expect("seed private repo"); + + let stranger = crate::auth::AuthenticatedDid("did:key:z6MkT3Stranger".to_string()); + let res = close_issue( + axum::extract::State(state.clone()), + axum::Extension(stranger), + axum::extract::Path(( + "z6MkT3Owner".to_string(), + "priv-close".to_string(), + "1".to_string(), + )), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(Some("203.0.113.67:5000".parse().unwrap())), + ) + .await; + assert!( + matches!(res, Err(AppError::RepoNotFound(_))), + "a non-reader must be refused as not-found, got {:?}", + res.err().map(|e| format!("{e:?}")) + ); + } + + async fn seed_repo_with_issue( + state: &crate::state::AppState, + owner_slug: &str, + owner_did: &str, + repo: &str, + issue_id: &str, + author_did: &str, + ) -> std::path::PathBuf { + state + .db + .upsert_mirror_repo(owner_slug, repo, "/unused", None, true) + .await + .expect("seed repo row"); + // Seed at the path the HANDLER will resolve. upsert_mirror_repo stores the + // bare slug in owner_did, and close_issue resolves from record.owner_did, so + // seeding from the full did:key would create the repo in a different + // directory and the handler would find nothing. + let record = state + .db + .get_repo(owner_slug, repo) + .await + .expect("get_repo") + .expect("seeded repo exists"); + let _ = owner_did; + let path = state + .repo_store + .acquire(&record.owner_did, &record.name) + .await + .expect("resolve disk path"); + let _ = std::fs::remove_dir_all(&path); + crate::git::store::init_bare(&path).expect("init bare repo"); + // Must deserialize as a real IssueRecord: `created_at` and `status` are + // required, and a parse failure would silently drop the author (the + // `.ok()` on from_str), which reads as a 403 rather than as a broken fixture. + let json = serde_json::to_string(&IssueRecord { + id: issue_id.to_string(), + title: "seeded".to_string(), + body: Some(String::new()), + author: Some(author_did.to_string()), + created_at: chrono::Utc::now().to_rfc3339(), + status: "open".to_string(), + signed_payload: None, + }) + .expect("serialize seeded issue"); + crate::git::issues::create_issue(&path, issue_id, &json).expect("seed issue blob"); + path + } + + /// INV-21(c) positive twin 1: the OWNER can still close. The reorder moved the + /// owner check above the lock, so this is the arm most likely to have broken, + /// and the deny test alone could not see it. + /// + /// The issue is seeded with a THIRD party as its author, deliberately. Seeding + /// the owner as their own author made this test unable to fail: with the owner + /// check disabled, the author fallback granted the close anyway and the test + /// stayed green. Only the owner arm can grant here now. + #[sqlx::test] + async fn owner_can_still_close_after_the_reorder(pool: PgPool) { + let state = crate::test_support::test_state(pool.clone()).await; + let owner_did = "did:key:z6MkT1Owner"; + seed_repo_with_issue( + &state, + "z6MkT1Owner", + owner_did, + "t1repo", + "1", + "did:key:z6MkT1Stranger", + ) + .await; + + let res = close_issue( + axum::extract::State(state.clone()), + axum::Extension(crate::auth::AuthenticatedDid(owner_did.to_string())), + axum::extract::Path(( + "z6MkT1Owner".to_string(), + "t1repo".to_string(), + "1".to_string(), + )), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(Some("203.0.113.65:5000".parse().unwrap())), + ) + .await; + assert!( + res.is_ok(), + "the owner must still be able to close: {:?}", + res.err().map(|e| format!("{e:?}")) + ); + } + + /// INV-21(c) positive twin 2: the non-owner AUTHOR can still close, through both + /// the pre-lock check and the re-assertion under the guard. + /// + /// It does NOT cover the acquire-vs-acquire_fresh distinction, despite that being + /// the reason the call changed. `RepoStore::for_testing` hardcodes `tigris: None`, + /// which makes `acquire` and `acquire_fresh` identical in every test here, so + /// reverting that line leaves this green. Separating them needs an object-storage + /// seam, which is out of scope for this change and tracked separately. Claiming + /// the coverage here would be worse than admitting the gap. + #[sqlx::test] + async fn issue_author_who_is_not_the_owner_can_still_close(pool: PgPool) { + let state = crate::test_support::test_state(pool.clone()).await; + let owner_did = "did:key:z6MkT2Owner"; + let author_did = "did:key:z6MkT2Author"; + seed_repo_with_issue(&state, "z6MkT2Owner", owner_did, "t2repo", "1", author_did).await; + + let res = close_issue( + axum::extract::State(state.clone()), + axum::Extension(crate::auth::AuthenticatedDid(author_did.to_string())), + axum::extract::Path(( + "z6MkT2Owner".to_string(), + "t2repo".to_string(), + "1".to_string(), + )), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(Some("203.0.113.66:5000".parse().unwrap())), + ) + .await; + assert!( + res.is_ok(), + "the issue author, who is NOT the repo owner, must still be able to close: {:?}", + res.err().map(|e| format!("{e:?}")) + ); + } +} + +/// #173 F1 follow-up: issue write paths reach `acquire_write` with no admission +/// permit, so an exhausted write-lock pool must shed 503 + Retry-After. #[cfg(test)] mod lock_pool_shed_tests { use super::*; @@ -314,22 +830,17 @@ mod lock_pool_shed_tests { } } - /// State whose repo store draws write locks from a ONE-connection pool with a - /// short checkout timeout, so a single held guard exhausts it promptly rather - /// than at the pool default. async fn one_connection_lock_pool_state(pool: &PgPool) -> AppState { let mut state = crate::test_support::test_state(pool.clone()).await; state.repo_store = crate::git::repo_store::RepoStore::new( std::path::PathBuf::from("/tmp/gitlawb-issues-lockpool"), None, crate::git::repo_store::build_lock_pool(pool, 1, std::time::Duration::from_secs(1)), + std::time::Duration::from_secs(300), ); state } - /// The shed must be a real 503 carrying Retry-After, not just an internal enum - /// variant: assert on the rendered response so a remapping of `Overloaded` is - /// caught here too. fn assert_sheds_503_with_retry_after(err: AppError, what: &str) { let resp = err.into_response(); assert_eq!( @@ -346,10 +857,6 @@ mod lock_pool_shed_tests { ); } - /// RED-before/GREEN-after for `create_issue`. Both directions: the shed while the - /// only lock-pool connection is held by a guard on a DIFFERENT repo (so this is - /// pool capacity, not advisory-lock contention on this repo), and the must-not - /// case once that connection is back. #[sqlx::test] async fn create_issue_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { let owner = "did:key:zISSUECREATELOCKPOOLAAAAAAAAAAAAAAAAAAAA"; @@ -380,9 +887,7 @@ mod lock_pool_shed_tests { let err = shed.expect_err("an exhausted lock pool must fail the call"); assert_sheds_503_with_retry_after(err, "create_issue"); - // MUST-NOT: with the pool free again the call is not shed as capacity (it - // fails later on the nonexistent on-disk repo, which is a git 500). - held.release(false).await; + let _ = held.release(false).await; let admitted = create_issue( State(state.clone()), Extension(AuthenticatedDid(owner.to_string())), @@ -401,7 +906,38 @@ mod lock_pool_shed_tests { ); } - /// RED-before/GREEN-after for `close_issue`, same two directions. + #[sqlx::test] + async fn close_issue_read_pool_exhaustion_sheds_before_snapshot(pool: PgPool) { + use std::sync::Arc; + let owner = "did:key:zISSUECLOSEREADPOOLBBBBBBBBBBBBBBBBBBBBB"; + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state + .db + .create_repo(&seed_repo(owner, "read-cap")) + .await + .expect("seed repo"); + + let shed = close_issue( + State(state.clone()), + Extension(AuthenticatedDid( + "did:key:zISSUECLOSEREADSTRANGER".to_string(), + )), + Path(( + owner.to_string(), + "read-cap".to_string(), + "deadbeef".to_string(), + )), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(None), + ) + .await; + assert!( + matches!(shed, Err(AppError::Overloaded(_))), + "an exhausted read pool must shed before snapshot work; got {shed:?}" + ); + } + #[sqlx::test] async fn close_issue_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { let owner = "did:key:zISSUECLOSELOCKPOOLBBBBBBBBBBBBBBBBBBBBB"; @@ -426,12 +962,14 @@ mod lock_pool_shed_tests { "lp-close".to_string(), "deadbeef".to_string(), )), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(None), ) .await; let err = shed.expect_err("an exhausted lock pool must fail the call"); assert_sheds_503_with_retry_after(err, "close_issue"); - held.release(false).await; + let _ = held.release(false).await; let admitted = close_issue( State(state.clone()), Extension(AuthenticatedDid(owner.to_string())), @@ -440,6 +978,8 @@ mod lock_pool_shed_tests { "lp-close".to_string(), "deadbeef".to_string(), )), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(None), ) .await; assert!( @@ -448,4 +988,244 @@ mod lock_pool_shed_tests { admitted.err() ); } + + async fn assert_retryable_repo_acquire(err: AppError, what: &str, code: &str) { + let resp = err.into_response(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "{what}: a transient acquire refusal must shed 503, not a 500 git error" + ); + let body = axum::body::to_bytes(resp.into_body(), 64 * 1024) + .await + .expect("body"); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains(code), + "{what}: the 503 must carry the {code} code, got {body}" + ); + } + + /// Advisory-lock contention on a non-push mutation must map through the shared + /// `acquire_write_app_error` classifier to `repo_busy`, not a 500 git_error. + #[sqlx::test] + async fn create_issue_contention_sheds_repo_busy_not_500(pool: PgPool) { + let owner = "did:key:zISSUECREATEBUSYAAAAAAAAAAAAAAAAAAAAA"; + let repo_name = "busy-create"; + let state = crate::test_support::test_state(pool.clone()).await; + state + .db + .create_repo(&seed_repo(owner, repo_name)) + .await + .expect("seed repo"); + + let held = state + .repo_store + .acquire_write(owner, repo_name) + .await + .expect("the first writer takes the advisory lock"); + + let shed = create_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), repo_name.to_string())), + Json(CreateIssueRequest { + title: "t".to_string(), + body: None, + signed_payload: None, + }), + ) + .await; + let err = shed.expect_err("contention must refuse the second writer"); + assert_retryable_repo_acquire(err, "create_issue", "repo_busy").await; + + let _ = held.release(false).await; + } + + /// A refused under-lock refresh on a non-push mutation must map to + /// `repo_unavailable`, not expose the storage error as a 500 git_error. + #[sqlx::test] + async fn create_issue_unavailable_sheds_repo_unavailable_not_500(pool: PgPool) { + use axum::response::IntoResponse; + + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any(|method: axum::http::Method| async move { + if method == axum::http::Method::HEAD { + let mut resp = axum::http::StatusCode::OK.into_response(); + resp.headers_mut() + .insert("etag", axum::http::HeaderValue::from_static("\"gen-1\"")); + resp + } else { + axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let owner = "did:key:zISSUEUNAVAILAAAAAAAAAAAAAAAAAAAAAAA"; + let repo_name = "unavail-create"; + let mut state = crate::test_support::test_state(pool.clone()).await; + state.repo_store = crate::git::repo_store::RepoStore::for_testing_with_tigris( + std::path::PathBuf::from("/tmp/gitlawb-issue-unavail"), + crate::git::repo_store::build_lock_pool(&pool, 2, std::time::Duration::from_secs(1)), + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint), + ); + state + .db + .create_repo(&seed_repo(owner, repo_name)) + .await + .expect("seed repo"); + + let shed = create_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), repo_name.to_string())), + Json(CreateIssueRequest { + title: "t".to_string(), + body: None, + signed_payload: None, + }), + ) + .await; + let err = shed.expect_err("a failed archive download must refuse the write"); + assert_retryable_repo_acquire(err, "create_issue", "repo_unavailable").await; + + server.abort(); + } + + /// A denied reader on a private repo must see 404 even when the caller's rate + /// bucket is exhausted. Rate limiting runs after the read gate. + #[sqlx::test] + async fn close_issue_rate_limit_runs_after_the_read_gate(pool: PgPool) { + use std::net::SocketAddr; + use std::time::Duration; + + let owner = "did:key:zCLOSERATEOWNERAAAAAAAAAAAAAAAAAAAAAAA"; + let stranger = "did:key:zCLOSERATESTRANGERBBBBBBBBBBBBBBBBBB"; + let mut state = crate::test_support::test_state(pool).await; + state.close_issue_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let mut repo = seed_repo(owner, "priv-close"); + repo.is_public = false; + state.db.create_repo(&repo).await.expect("seed repo"); + + let peer: SocketAddr = "203.0.113.88:7000".parse().unwrap(); + assert!( + state + .close_issue_rate_limiter + .check(&peer.ip().to_string()) + .await, + "exhaust the close_issue bucket before close_issue" + ); + + let res = close_issue( + State(state), + Extension(AuthenticatedDid(stranger.to_string())), + Path((owner.to_string(), "priv-close".to_string(), "1".to_string())), + axum::http::HeaderMap::new(), + crate::rate_limit::PeerAddr(Some(peer)), + ) + .await; + + assert!( + matches!(res, Err(AppError::RepoNotFound(_))), + "a non-reader must see 404, not 429, even when rate limited: {:?}", + res + ); + } + + /// close_issue and receive-pack must not share one per-IP bucket. + #[sqlx::test] + async fn close_issue_rate_limit_does_not_drain_push_bucket(pool: PgPool) { + use std::net::SocketAddr; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool).await; + state.close_issue_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.99:7000".parse().unwrap(); + let key = peer.ip().to_string(); + assert!( + state.close_issue_rate_limiter.check(&key).await, + "exhaust only the close_issue bucket" + ); + assert!( + state.push_rate_limiter.check(&key).await, + "push traffic must keep its own bucket after close_issue is exhausted" + ); + } + + /// A refused publish must roll back the local issue ref so a retry does not + /// mint a second id for the same logical filing attempt. + #[sqlx::test] + async fn create_issue_rolls_back_local_ref_when_publish_refuses(pool: PgPool) { + use axum::response::IntoResponse; + + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any(|method: axum::http::Method| async move { + if method == axum::http::Method::HEAD { + let mut resp = axum::http::StatusCode::OK.into_response(); + resp.headers_mut() + .insert("etag", axum::http::HeaderValue::from_static("\"gen-1\"")); + resp + } else { + axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let owner = "did:key:zISSUEROLLBACKAAAAAAAAAAAAAAAAAAAAAAA"; + let repo_name = "rollback-create"; + let mut state = crate::test_support::test_state(pool.clone()).await; + let disk = tempfile::TempDir::new().unwrap(); + crate::git::store::init_bare(&disk.path().join("repo.git")).expect("bare repo"); + let mut seeded = seed_repo(owner, repo_name); + seeded.disk_path = disk.path().join("repo.git").to_string_lossy().to_string(); + state.db.create_repo(&seeded).await.expect("seed repo"); + state.repo_store = crate::git::repo_store::RepoStore::for_testing_with_tigris( + disk.path().to_path_buf(), + crate::git::repo_store::build_lock_pool(&pool, 2, std::time::Duration::from_secs(1)), + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint), + ); + + let shed = create_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), repo_name.to_string())), + Json(CreateIssueRequest { + title: "t".to_string(), + body: None, + signed_payload: None, + }), + ) + .await; + assert!( + shed.is_err(), + "a failed archive upload must refuse the write" + ); + + let issues = git_issues::list_issues(&disk.path().join("repo.git")).expect("list issues"); + assert!( + issues.is_empty(), + "the local issue ref must be rolled back after a refused publish, got {issues:?}" + ); + + server.abort(); + } } diff --git a/crates/gitlawb-node/src/api/pulls.rs b/crates/gitlawb-node/src/api/pulls.rs index 6255ef246..95c0b5064 100644 --- a/crates/gitlawb-node/src/api/pulls.rs +++ b/crates/gitlawb-node/src/api/pulls.rs @@ -209,9 +209,6 @@ pub async fn merge_pr( return Err(AppError::BadRequest(format!("PR is already {}", pr.status))); } - // Shed 503 + Retry-After on an exhausted write-lock POOL instead of a generic - // git 500 (#173 F1). Merging holds no admission permit, so it reaches the pool - // unthrottled; reuse the push handler's mapping so the two cannot drift. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) @@ -228,7 +225,10 @@ pub async fn merge_pr( ); // Always release the advisory lock — even on error; upload to Tigris only on success. - guard.release(merge_result.is_ok()).await; + // Short-circuit on a refused publish before the PR is marked merged and + // before the webhook fires. Both are irreversible announcements of a merge + // commit that only exists on this node's disk. + guard.release(merge_result.is_ok()).await.into_result()?; let merge_sha = merge_result.map_err(|e| AppError::Git(e.to_string()))?; @@ -428,12 +428,8 @@ pub async fn list_comments( Ok(Json(serde_json::json!({ "comments": comments }))) } -/// #173 F1 follow-up: `merge_pr` reaches `acquire_write` holding NO admission permit -/// (unlike the push handler, which is capped by the git-push semaphore), so it is one -/// of the callers most likely to meet an exhausted write-lock POOL under load. An -/// exhausted pool is a capacity signal, so the merge must shed 503 + Retry-After -/// (`AppError::Overloaded`) the way the push handler does, not report the generic -/// 500 git error that says nothing about retrying. +/// #173 F1 follow-up: merge holds no admission permit, so an exhausted write-lock +/// pool must shed 503 + Retry-After. #[cfg(test)] mod lock_pool_shed_tests { use super::*; @@ -457,20 +453,15 @@ mod lock_pool_shed_tests { } } - /// RED-before/GREEN-after for `merge_pr`. Both directions: the shed while the only - /// lock-pool connection is held by a guard on a DIFFERENT repo (so this is pool - /// capacity, not advisory-lock contention on this repo), and the must-not case - /// once that connection is back. #[sqlx::test] async fn merge_pr_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { let owner = "did:key:zMERGELOCKPOOLOWNERAAAAAAAAAAAAAAAAAAAAA"; let mut state = crate::test_support::test_state(pool.clone()).await; - // One lock-pool connection with a short checkout timeout, so a single held - // guard exhausts it promptly rather than at the pool default. state.repo_store = crate::git::repo_store::RepoStore::new( std::path::PathBuf::from("/tmp/gitlawb-pulls-lockpool"), None, crate::git::repo_store::build_lock_pool(&pool, 1, std::time::Duration::from_secs(1)), + std::time::Duration::from_secs(300), ); let repo = seed_repo(owner, "lp-merge"); @@ -524,9 +515,7 @@ mod lock_pool_shed_tests { "merge_pr: a capacity shed must tell the client when to retry" ); - // MUST-NOT: with the pool free again the merge is not shed as capacity (it - // fails later on the nonexistent on-disk repo, which is a git 500). - held.release(false).await; + let _ = held.release(false).await; let admitted = merge_pr( State(state.clone()), Extension(AuthenticatedDid(owner.to_string())), diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index d4b8ef7d4..ed028ab2f 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -3,6 +3,7 @@ use axum::http::StatusCode; use axum::response::Response; use axum::Json; use bytes::Bytes; +use std::path::PathBuf; use std::sync::Arc; use crate::auth::{caller_authorized_to_push, AuthenticatedDid}; @@ -692,8 +693,17 @@ pub async fn git_info_refs( git_permit(&state.git_read_semaphore)? }; - // For receive-pack (push), download the latest from Tigris so the client - // sees the same refs that acquire_write() will operate on. + // For receive-pack (push), read the latest from Tigris so the client sees + // the same refs that acquire_write() will operate on. A NON-MUTATING + // snapshot, not `acquire_fresh`: the advertisement runs WITHOUT the advisory + // lock, and acquire_fresh downloads and publishes into the live repo path + // (removing the existing directory and renaming the extract into place), so + // an unlocked advertisement could delete or swap the directory under a + // concurrent guarded write. In the worst ordering the guarded write has + // finished but `release` has not compressed the tree, and the guarded + // release uploads the replaced old tree with its still-valid ETag and + // reports success, losing the accepted write. The snapshot unpacks into a + // throwaway temp dir that is served from and then removed. // // Bound the acquire under `git_acquire_timeout_secs`: the concurrency permit is // already held above, and `git_service_timeout_secs` only starts once git spawns, @@ -703,28 +713,46 @@ pub async fn git_info_refs( // so the shed frees the slot; return a bounded 503. let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); let acquire_fut = async { - if service == "git-receive-pack" { + let res = if service == "git-receive-pack" { state .repo_store - .acquire_fresh(&record.owner_did, &record.name) + .read_snapshot(&record.owner_did, &record.name) .await + .map(|s| (s.path().to_path_buf(), Some(s))) } else { state .repo_store .acquire(&record.owner_did, &record.name) .await - } + .map(|p| (p, None)) + }; + res.map_err(|e| { + if is_expected_transient_acquire_failure(&e) { + tracing::warn!(repo = %name, service = %service, err = %e, "repo acquire failed"); + } else { + tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); + } + // This closure bypasses the `From` chain, so a typed + // refusal would otherwise be stringified into a 500 `git_error`. Route + // just that one case through `From` and leave every other failure on + // exactly today's behavior: this call site also serves the read path via + // `acquire`, whose error vocabulary is out of scope here. + if e.is::() { + AppError::from(e) + } else { + AppError::Git(e.to_string()) + } + }) }; - let disk_path = tokio::time::timeout(acquire_deadline, acquire_fut) + // The snapshot (if any) is kept alive for the whole handler scope below: its + // Drop removes the temp dir it was unpacked into, so dropping it here would + // delete the directory `info_refs` is about to serve from. + let (disk_path, _snapshot_keepalive) = tokio::time::timeout(acquire_deadline, acquire_fut) .await .map_err(|_elapsed| { tracing::warn!(repo = %name, service = %service, "repo acquire timed out; shedding with 503"); AppError::Overloaded("git service acquisition timed out, retry shortly".into()) - })? - .map_err(|e| { - tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); - AppError::Git(e.to_string()) - })?; + })??; // Move the admission permits into the guard so they release only after the spawned // git process group is confirmed reaped, on complete/timeout/disconnect — not the @@ -978,6 +1006,7 @@ pub(crate) mod drain_faults { pub(crate) rules_read_failures_left: usize, pub(crate) repo_read_attempts: usize, pub(crate) rules_read_attempts: usize, + pub(crate) reread_exhausted: bool, } fn table() -> &'static Mutex> { @@ -1008,6 +1037,25 @@ pub(crate) mod drain_faults { .unwrap_or_default() } + /// Whether the drain re-read loop exhausted its retry budget for `repo_id`. + pub(crate) fn reread_exhausted(repo_id: &str) -> bool { + table() + .lock() + .unwrap() + .get(repo_id) + .map(|c| c.reread_exhausted) + .unwrap_or(false) + } + + pub(crate) fn mark_reread_exhausted(repo_id: &str) { + table() + .lock() + .unwrap() + .entry(repo_id.to_string()) + .or_default() + .reread_exhausted = true; + } + /// Production-path hook: count one repo re-read attempt, return whether it must fail. pub(crate) fn take_repo_read(repo_id: &str) -> bool { let mut map = table().lock().unwrap(); @@ -1129,6 +1177,8 @@ async fn drain_refresh_state(ctx: &EncryptTaskCtx) -> DrainRefresh { "coalesced drain: re-read failed on every attempt; the coalesced push's \ pin/encrypt pass is dropped (no reconciliation sweep re-derives it)" ); + #[cfg(test)] + drain_faults::mark_reread_exhausted(&ctx.repo_id); DrainRefresh::Failed } @@ -1516,11 +1566,12 @@ async fn pin_and_encrypt_objects( /// Map an `acquire_write` failure to the right `AppError`. An exhausted repo write-lock /// POOL is a capacity signal, not a broken repo, so it sheds 503 + Retry-After the same /// way the admission caps around it do; it used to fall into the generic git 500, which -/// tells the client nothing about retrying (#173 F1). Anything else stays a git error. +/// tells the client nothing about retrying (#173 F1). Lock contention and a refused +/// under-lock refresh are transient and map to fixed-body 503s via [`RepoBusy`] and +/// [`RepoUnavailable`]; only genuine untyped git failures stay on the 500 path. /// -/// Shared with the non-push `acquire_write` callers (`api/issues.rs`, `api/pulls.rs`) -/// rather than copied: those hold no admission permit, so they meet an exhausted pool -/// first, and a second copy of this mapping would be free to drift from the push path. +/// Shared with every `acquire_write` caller (`receive-pack`, `api/issues.rs`, +/// `api/pulls.rs`) so the write-acquisition contract cannot drift between routes. pub(crate) fn acquire_write_app_error(err: &anyhow::Error, repo: &str) -> AppError { if err .downcast_ref::() @@ -1528,6 +1579,16 @@ pub(crate) fn acquire_write_app_error(err: &anyhow::Error, repo: &str) -> AppErr { tracing::warn!(repo = %repo, err = %err, "write-lock pool exhausted; shedding with 503"); AppError::Overloaded("git write locks at capacity, retry shortly".into()) + } else if is_expected_transient_acquire_failure(err) { + tracing::warn!(repo = %repo, err = %err, "acquire_write failed"); + if err + .downcast_ref::() + .is_some() + { + AppError::RepoBusy + } else { + AppError::RepoUnavailable + } } else { tracing::error!(repo = %repo, err = %err, "acquire_write failed"); AppError::Git(err.to_string()) @@ -1538,6 +1599,23 @@ pub(crate) fn acquire_write_app_error(err: &anyhow::Error, repo: &str) -> AppErr /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; /// callers add their own tracing. +/// Acquire failures that are ordinary and transient: lock contention +/// ([`RepoBusy`]) and an under-lock refresh that could not reach object storage +/// ([`RepoUnavailable`]). Both already log at their raise site and both map to a +/// retryable 503, so the handler layer logs them at warn rather than paging. +/// Best-effort, like the database startup classifier: anything this cannot +/// recognize counts as NOT transient and keeps its error-level log. +/// +/// [`RepoBusy`]: crate::git::repo_store::RepoBusy +/// [`RepoUnavailable`]: crate::git::repo_store::RepoUnavailable +fn is_expected_transient_acquire_failure(err: &anyhow::Error) -> bool { + err.downcast_ref::() + .is_some() + || err + .downcast_ref::() + .is_some() +} + fn git_service_app_error(err: &anyhow::Error) -> AppError { if err .downcast_ref::() @@ -2297,19 +2375,7 @@ pub async fn git_receive_pack( // would return 200 to the pusher before the durable copy lands, which is a larger // change to the client contract than the window it closes. let push_succeeded = receive_result.is_ok(); - if push_succeeded { - tokio::spawn(post_receive_replication_tail( - state.clone(), - record.clone(), - ref_updates.clone(), - disk_path.clone(), - auth.0.to_string(), - )); - } - - // Always release the advisory lock — even on error — to prevent stale locks - // from blocking subsequent pushes. Only upload to Tigris when the push - // succeeded; uploading a half-applied repo would propagate corruption. + let publish_durability = PublishDurabilitySlot::new(); // Reclaim the write lock from the shared cell (#173 F2). This is only reachable // once `receive_pack` has returned, so the admission guard's copy can only ever // DELAY release, never perform it early; on the disconnect path this line is not @@ -2319,7 +2385,30 @@ pub async fn git_receive_pack( .expect("repo write-lock mutex poisoned") .take() .expect("the write lock is only taken here, and only once"); - reclaimed.release(push_succeeded).await; + if push_succeeded { + // Arm on the guard's OWN publish stage, so a disconnect anywhere in + // `release` is classified by how far the publish actually got rather + // than by the mere fact that release was entered. + publish_durability.arm_for_release(reclaimed.publish_stage()); + tokio::spawn(post_receive_replication_tail( + state.clone(), + record.clone(), + ref_updates.clone(), + disk_path.clone(), + auth.0.to_string(), + Some(publish_durability.arc()), + )); + } + // Short-circuit on a refused publish BEFORE anything downstream observes + // the push. The pack is on local disk but not in object storage, so + // touching the repo, recording the push, bumping trust, issuing ref + // certificates or answering 200 would all be reporting a write no other + // node can read. + let outcome = reclaimed.release(push_succeeded).await; + if push_succeeded { + publish_durability.record(outcome).await; + } + outcome.into_result()?; // Clean path: clone (a) already dropped inside run_git_service when the receive-pack // group was reaped; clone (b) held here spanned the success-only Tigris upload that // ran inside release() above. Drop it now so a second same-repo push proceeds the @@ -2436,13 +2525,172 @@ pub async fn git_receive_pack( /// the per-repo-coalesced pin/encrypt task, and this push's own Pinata + announce /// task. Split out of `git_receive_pack` so the ordering the coalescing gate depends /// on is directly testable; the handler spawns it and returns. +/// Records the release-side publish outcome for the detached post-receive tail. +/// +/// The subtle case is the one this type exists for: the handler future is +/// DROPPED, so `release` never returns and `record` is never called. The slot +/// then has to classify an attempt whose outcome nobody will ever report, and +/// the only honest source for that is the publish STAGE — how far the attempt +/// had actually got when it was abandoned. +/// +/// Tracking "release started" instead was the defect. `release` awaits a blocking +/// compression before the conditional PUT is constructed, so a cancellation +/// there is a definite "no publication was attempted", yet every cancellation +/// after the flag was set recorded `UploadUnknowable` — which the tail accepted, +/// and then went on to do IPFS, Pinata, P2P, GraphQL, Arweave and peer work for +/// refs that existed only in a rejected local write. +struct PublishDurabilitySlot { + inner: Arc>>, + recorded: std::sync::atomic::AtomicBool, + /// The stage of the release this slot is armed for. `None` until the release + /// is about to be awaited, which is what makes a drop before that point leave + /// the slot empty and the tail fail closed. + stage: std::sync::Mutex>>, +} + +impl PublishDurabilitySlot { + fn new() -> Self { + Self { + inner: Arc::new(std::sync::Mutex::new(None)), + recorded: std::sync::atomic::AtomicBool::new(false), + stage: std::sync::Mutex::new(None), + } + } + + /// Arm the slot against the guard's publish stage, immediately before the + /// release is awaited. + fn arm_for_release(&self, stage: Arc) { + *self.stage.lock().expect("publish stage slot poisoned") = Some(stage); + } + + fn arc(&self) -> Arc>> { + Arc::clone(&self.inner) + } + + async fn record(&self, outcome: crate::git::repo_store::ReleaseOutcome) { + *self + .inner + .lock() + .expect("publish durability mutex poisoned") = Some(outcome); + self.recorded + .store(true, std::sync::atomic::Ordering::SeqCst); + } +} + +impl Drop for PublishDurabilitySlot { + fn drop(&mut self) { + if self.recorded.load(std::sync::atomic::Ordering::SeqCst) { + return; + } + let Some(stage) = self + .stage + .lock() + .expect("publish stage slot poisoned") + .clone() + else { + // Dropped before the release was even armed. Leave the slot empty so + // the tail fails closed on the wait below. + return; + }; + use crate::git::publish::PublishStage; + use crate::git::repo_store::ReleaseOutcome; + let outcome = match stage.get() { + // The store answered. Cancellation after that point loses the + // handler's response, not the write, so the tail may run. + // + // `NoBackend` joins it: with no object storage configured there is + // no publication to confirm and the pack on local disk is the + // durable copy, which is exactly what `ReleaseOutcome::Released` + // already means for a release that had nothing to upload. + PublishStage::Published { .. } | PublishStage::NoBackend => ReleaseOutcome::Released, + // THE FINDING. Compression was still running, so the conditional PUT + // was never constructed and no publication was ever attempted. This + // is a definite refusal, not an unknowable one, and the tail must + // skip every replication effect. + PublishStage::Idle | PublishStage::PreparingArchive | PublishStage::Refused => { + ReleaseOutcome::UploadFailed + } + // Genuinely in flight. Nothing here can reconcile it — a `Drop` impl + // cannot await a HEAD — so it stays unknowable, which the tail refuses + // because it requires `Released`. The write is still on local disk and + // the next writer's under-lock refresh resolves it. + PublishStage::PutDispatched { .. } | PublishStage::Ambiguous { .. } => { + ReleaseOutcome::UploadUnknowable + } + }; + let mut slot = self + .inner + .lock() + .expect("publish durability mutex poisoned"); + if slot.is_none() { + *slot = Some(outcome); + } + } +} + +/// May the detached replication tail run? +/// +/// ONLY on `Released`, which is the outcome that means the store acknowledged +/// this attempt (directly, or through the release-side reconciliation that +/// matched the attempt id against what the store holds). `UploadUnknowable` used +/// to pass here, which let the tail pin, announce and publish refs whose archive +/// may never have landed; an unresolved dispatch has to be reconciled to this +/// attempt's own generation before it counts as confirmation, and the place that +/// can do it is `release`, not this predicate. +async fn publish_durability_confirmed( + slot: &Option>>>, + wait: std::time::Duration, +) -> bool { + let Some(slot) = slot else { + return true; + }; + let confirmed = |outcome: Option| { + matches!( + outcome, + Some(crate::git::repo_store::ReleaseOutcome::Released) + ) + }; + let start = std::time::Instant::now(); + loop { + let outcome = *slot.lock().expect("publish durability mutex poisoned"); + if outcome.is_some() { + return confirmed(outcome); + } + if start.elapsed() >= wait { + // No outcome within the release-side transfer bound plus slack. Fail + // closed when the slot is still empty; an abandoned handler installs + // its stage's verdict via PublishDurabilitySlot::drop, so the tail + // resolves promptly rather than waiting out the full bound. + return confirmed(*slot.lock().expect("publish durability mutex poisoned")); + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } +} + async fn post_receive_replication_tail( state: AppState, record: RepoRecord, ref_updates: Vec, disk_path: std::path::PathBuf, did: String, + publish_durability: Option< + Arc>>, + >, ) { + let durability_wait = std::time::Duration::from_secs( + state + .config + .lock_held_transfer_timeout_secs + .saturating_add(5), + ); + if !publish_durability_confirmed(&publish_durability, durability_wait).await { + tracing::warn!( + repo = %record.id, + "skipping post-receive tail: publish durability was not confirmed" + ); + return; + } + // Replication enforcement (Phase 2): decide once per push whether the public // may read this repo at all and, if so, which blob OIDs must not leave the // node. `withheld == None` means this push pins nothing (private / mode A / @@ -2994,6 +3242,203 @@ pub async fn list_federated_repos( // ── Fork ────────────────────────────────────────────────────────────────── +/// Removes a fork's local mirror clone on every exit until disarmed after the +/// database row commits — but ONLY while that clone still belongs to the attempt +/// the guard was built for. +/// +/// The bare `remove_dir_all` this replaces authorized itself with the path alone. +/// Two attempts at the same logical fork share one path, so a failed attempt +/// unwinding late could remove the directory a successor had already published +/// and inserted a row for. +struct ForkCloneGuard { + path: Option, + attempt: crate::git::publish::PublishAttemptId, +} + +const FORK_CREATE_CONFIRM_ATTEMPTS: u32 = 5; + +/// Who owns the logical fork name once `create_repo` has reported an error. +/// +/// The distinction the previous `Option` could not draw is between +/// "my insert committed" and "somebody's insert committed". Only the first +/// entitles this request to answer 201 with that row, and only the third +/// entitles it to compensate. +enum ForkRowConfirmation { + /// THIS attempt's row is present: the insert committed and the error was a + /// transport-level report of a successful write. + Ours(Box), + /// A different attempt owns the name. This request's archive, disk path and + /// fork provenance do not belong to that row, so it must not be returned, + /// and that attempt's resources must not be compensated. + Foreign, + /// Nothing owns the name. + Absent, +} + +/// After `create_repo` fails, re-read the row with bounded retries so a +/// transport-level error does not skip archive compensation when the insert never +/// landed, and so a successful insert is not mistaken for a failure. +/// +/// Keyed on `record_id` FIRST. Owner/name identifies a namespace, not an attempt: +/// a concurrent ordinary create, a mirror registration or a retry can insert a +/// different row under the same logical name, and treating that as proof of our +/// own commit returned 201 carrying somebody else's row. +async fn confirm_fork_repo_row( + db: &crate::db::Db, + record_id: &str, + owner_short: &str, + fork_name: &str, +) -> anyhow::Result { + let mut delay = std::time::Duration::from_millis(25); + for attempt in 0..FORK_CREATE_CONFIRM_ATTEMPTS { + let looked_up = async { + if let Some(ours) = db.get_repo_by_id(record_id).await? { + return Ok::<_, anyhow::Error>(ForkRowConfirmation::Ours(Box::new(ours))); + } + Ok(match db.get_repo(owner_short, fork_name).await? { + Some(_) => ForkRowConfirmation::Foreign, + None => ForkRowConfirmation::Absent, + }) + } + .await; + match looked_up { + Ok(confirmation) => return Ok(confirmation), + Err(e) if attempt + 1 < FORK_CREATE_CONFIRM_ATTEMPTS => { + tracing::warn!( + fork = %fork_name, + attempt, + err = %e, + "fork create_repo confirmation lookup failed — retrying" + ); + tokio::time::sleep(delay).await; + delay = delay + .saturating_mul(2) + .min(std::time::Duration::from_secs(1)); + } + Err(e) => return Err(e), + } + } + unreachable!("loop returns on every attempt") +} + +/// When the confirmation lookup stays unavailable after bounded retries, keep +/// retrying in the background and compensate only after establishing that no row +/// exists for this fork name. +/// +/// The `None` observation is still racy on its own — a successor can commit +/// between the read and the deletions, and did, which let recovery for a failed +/// fork erase a succeeding attempt's repository. What makes it safe is that the +/// compensation it calls is now conditional on the object and the directory still +/// carrying THIS attempt's identity, so a successor that commits at any point +/// after the read keeps both. +#[allow(clippy::too_many_arguments)] +async fn schedule_fork_create_recovery( + db: std::sync::Arc, + repo_store: crate::git::repo_store::RepoStore, + record_id: String, + owner_short: String, + fork_name: String, + owner_did: String, + disk_path: std::path::PathBuf, + attempt: crate::git::publish::PublishAttemptId, +) { + let mut delay = std::time::Duration::from_millis(250); + for n in 0..12 { + match confirm_fork_repo_row(&db, &record_id, &owner_short, &fork_name).await { + Ok(ForkRowConfirmation::Ours(_)) => { + tracing::info!( + fork = %fork_name, + attempt = n, + "fork create_repo recovery found this attempt's row — no compensation" + ); + return; + } + Ok(ForkRowConfirmation::Foreign) => { + tracing::info!( + fork = %fork_name, + attempt = n, + "fork create_repo recovery found another attempt's row under this name — \ + compensating only what this attempt still owns" + ); + repo_store + .compensate_fork_archive(&owner_did, &fork_name, disk_path.as_path(), &attempt) + .await; + return; + } + Ok(ForkRowConfirmation::Absent) => { + tracing::warn!( + fork = %fork_name, + attempt = n, + "fork create_repo recovery confirmed no row — compensating orphan archive" + ); + repo_store + .compensate_fork_archive(&owner_did, &fork_name, disk_path.as_path(), &attempt) + .await; + return; + } + Err(e) if n + 1 < 12 => { + tracing::warn!( + fork = %fork_name, + attempt = n, + err = %e, + "fork create_repo recovery lookup failed — retrying" + ); + tokio::time::sleep(delay).await; + delay = delay + .saturating_mul(2) + .min(std::time::Duration::from_secs(30)); + } + Err(e) => { + tracing::error!( + fork = %fork_name, + err = %e, + "fork create_repo recovery gave up — orphan archive may remain until operator cleanup" + ); + return; + } + } + } +} + +impl ForkCloneGuard { + fn new( + path: crate::git::repo_store::ValidatedRepoDiskPath, + attempt: crate::git::publish::PublishAttemptId, + ) -> Self { + let path = path.into_path_buf(); + // Stamp the directory before anything can fail, so every later cleanup + // (this guard's Drop, the handler's compensation, the background + // recovery) has an ownership answer to consult. + crate::git::repo_store::claim_fork_disk_path(&path, &attempt); + Self { + path: Some(path), + attempt, + } + } + + fn path(&self) -> &std::path::Path { + self.path + .as_deref() + .expect("fork clone guard disarmed while still in use") + } + + fn disarm(&mut self) { + self.path = None; + } +} + +impl Drop for ForkCloneGuard { + fn drop(&mut self) { + if let Some(path) = self.path.take() { + crate::git::repo_store::remove_fork_clone_if_ours( + &path, + &self.attempt, + "fork attempt cleanup", + ); + } + } +} + #[derive(Debug, Deserialize)] pub struct ForkRepoRequest { pub name: Option, // defaults to source repo name @@ -3064,7 +3509,12 @@ pub async fn fork_repo( .await .map_err(|e| AppError::Git(e.to_string()))?; - let disk_path = store::repo_disk_path(&state.config.repos_dir, &forker_did, &fork_name); + let disk_path = crate::git::repo_store::validated_repo_disk_path( + &state.config.repos_dir, + &forker_did, + &fork_name, + ) + .map_err(|e| AppError::BadRequest(e.to_string()))?; // Clone the source repo as a mirror let output = std::process::Command::new("git") @@ -3084,15 +3534,109 @@ pub async fn fork_repo( ))); } - // Upload fork to Tigris - state + // ONE identity for the whole workflow: the DB row id this request will + // insert is also the attempt id stamped into the object's metadata and onto + // the disk clone. That is what lets confirmation ask "is my row there" rather + // than "is a row there", and lets every cleanup ask "is this still mine" + // rather than "does this name resolve". + let record_id = Uuid::new_v4().to_string(); + let attempt = crate::git::publish::PublishAttemptId::from_owned(record_id.clone()); + + let mut clone_guard = ForkCloneGuard::new(disk_path.clone(), attempt.clone()); + + // Upload fork to Tigris. Create-only: a refused precondition means an orphan + // archive already sits under this key (a failed create_repo or another + // writer), and proceeding would create a DB record whose archive is shadowed + // by bytes that are not this fork. Refuse rather than accept a fork other + // nodes would fetch as unrelated content. The clone guard removes the local + // mirror on every DEFINITE upload failure path. + if let Err(e) = state .repo_store - .release_after_write(&forker_did, &fork_name) - .await; + .release_after_write(&forker_did, &fork_name, &attempt) + .await + { + match e { + crate::git::tigris::UploadError::PreconditionLost { status } => { + // One reconciliation before refusing: an SDK-level retry, or a + // duplicated dispatch, can land THIS attempt's bytes and then see + // the create-only fence refuse the second copy. The stored object + // carrying our own attempt id says the publish succeeded, and + // refusing it would fence the fork name behind our own work. + if state + .repo_store + .fork_attempt_landed(&forker_did, &fork_name, &attempt) + .await + .unwrap_or(false) + { + tracing::info!( + fork = %fork_name, + status, + "fork create-only PUT was refused but the stored archive is this \ + attempt's own — continuing" + ); + } else { + tracing::warn!( + forker = %forker_did, + fork = %fork_name, + status, + "fork refused: an archive already exists under the fork's key" + ); + return Err(AppError::RepoExists(fork_name.clone())); + } + } + crate::git::tigris::UploadError::NotPublished(other) => { + // Proven not to have committed. Dropping the clone (on the + // guard's Drop) is safe, and the fork name is left free. + return Err(AppError::Git(format!("fork upload failed: {other:#}"))); + } + crate::git::tigris::UploadError::Ambiguous { source, .. } => { + // THE DURABLE FAILURE MODE. The create-only PUT can commit and + // the response can still be lost or unparseable. Compensating + // that as a definite failure dropped the only local clone and + // skipped the DB insert, and every retry then saw the orphan + // object and returned RepoExists — the fork name unusable until + // an operator intervened. + // + // Ask the store instead. The attempt id travelled with the bytes, + // so "did MY write land" is answerable even when "did my request + // succeed" is not. + match state + .repo_store + .fork_attempt_landed(&forker_did, &fork_name, &attempt) + .await + { + Ok(true) => { + tracing::warn!( + fork = %fork_name, + err = %source, + "fork upload lost its response but the stored archive is this \ + attempt's own — recovering the committed publish" + ); + } + // Not (yet) ours, or unreachable. The request may still be in + // flight, so nothing here may be destroyed: keep the clone, + // keep whatever is stored, and refuse retryably. Disarming the + // guard is what preserves the only local copy. + Ok(false) | Err(_) => { + clone_guard.disarm(); + tracing::warn!( + forker = %forker_did, + fork = %fork_name, + err = %source, + "fork upload outcome is unknowable — leaving the clone and any \ + stored object in place for reconciliation rather than \ + compensating a write that may have landed" + ); + return Err(AppError::RepoUnavailable); + } + } + } + } + } let now = Utc::now(); let record = crate::db::RepoRecord { - id: Uuid::new_v4().to_string(), + id: record_id.clone(), name: fork_name.clone(), owner_did: forker_did.clone(), description: source.description.clone(), @@ -3100,12 +3644,92 @@ pub async fn fork_repo( default_branch: source.default_branch.clone(), created_at: now, updated_at: now, - disk_path: disk_path.to_string_lossy().to_string(), + disk_path: clone_guard.path().to_string_lossy().to_string(), forked_from: Some(source.id.clone()), machine_id: state.machine_id.clone(), }; - state.db.create_repo(&record).await?; + if let Err(e) = state.db.create_repo(&record).await { + match confirm_fork_repo_row(&state.db, &record_id, forker_short, &fork_name).await { + Ok(ForkRowConfirmation::Ours(committed)) => { + clone_guard.disarm(); + tracing::warn!( + fork = %fork_name, + forker = %forker_did, + "fork create_repo returned an error but THIS attempt's row is present — treating as success" + ); + return Ok(( + StatusCode::CREATED, + Json(to_response(&committed, &state, 0)), + )); + } + Ok(ForkRowConfirmation::Foreign) => { + // A concurrent create, mirror registration or retry owns the + // name. Returning its row would hand this caller a repository + // whose archive, disk path and fork provenance are not the ones + // this request produced. Refuse, and compensate only what this + // attempt still owns — the guard and `compensate_fork_archive` + // both check ownership, so the successor's object and directory + // survive. + tracing::warn!( + fork = %fork_name, + forker = %forker_did, + "fork create_repo lost the name to another attempt — refusing rather than \ + claiming its row" + ); + clone_guard.disarm(); + state + .repo_store + .compensate_fork_archive(&forker_did, &fork_name, disk_path.as_path(), &attempt) + .await; + return Err(AppError::RepoExists(fork_name.clone())); + } + Ok(ForkRowConfirmation::Absent) => { + clone_guard.disarm(); + state + .repo_store + .compensate_fork_archive(&forker_did, &fork_name, disk_path.as_path(), &attempt) + .await; + return Err(e.into()); + } + Err(lookup_err) => { + clone_guard.disarm(); + let db = std::sync::Arc::clone(&state.db); + let repo_store = state.repo_store.clone(); + let owner_short = forker_short.to_string(); + let fork_name_cl = fork_name.clone(); + let owner_did = forker_did.clone(); + let disk_path_cl = disk_path.as_path().to_path_buf(); + let record_id_cl = record_id.clone(); + let attempt_cl = attempt.clone(); + tokio::spawn(async move { + schedule_fork_create_recovery( + db, + repo_store, + record_id_cl, + owner_short, + fork_name_cl, + owner_did, + disk_path_cl, + attempt_cl, + ) + .await; + }); + tracing::warn!( + fork = %fork_name, + create_err = %e, + lookup_err = %lookup_err, + "fork create_repo failed and confirmation lookup stayed unavailable — scheduled recovery" + ); + return Err(AppError::RepoUnavailable); + } + } + } + + clone_guard.disarm(); + // The row is committed: this directory is now the repository, not an + // attempt's staging area, and nothing may ever compensate it away. + crate::git::repo_store::release_fork_disk_claim(disk_path.as_path()); // Persist the proof so the fork carries it when it propagates to peers. if let Some(p) = verified_proof { @@ -3343,6 +3967,232 @@ mod tests { const OWNER_SHORT: &str = "z6MkpTHR8VNsBxYAAWHut2Geadd9jSwuBV8xRoAnwWsdvktH"; const STRANGER_DID: &str = "did:key:z6Mkffonly5tranger0000000000000000000000000000000"; + use crate::git::publish::{PublishAttemptId, PublishStage, PublishStageCell}; + use crate::git::repo_store::ReleaseOutcome; + + /// A slot armed against a stage cell the test drives directly, which is the + /// only way to reach the abandoned-handler arms without a live push. + fn armed_slot(stage: PublishStage) -> (PublishDurabilitySlot, Arc) { + let cell = Arc::new(PublishStageCell::new()); + cell.set(stage); + let slot = PublishDurabilitySlot::new(); + slot.arm_for_release(Arc::clone(&cell)); + (slot, cell) + } + + /// P1 FINDING 1, at the type level. A handler cancelled while the archive is + /// still being compressed has NOT attempted publication: the conditional PUT + /// is not constructed until compression returns. Recording that as + /// `UploadUnknowable` is what let the detached tail do IPFS, Pinata, P2P, + /// GraphQL, Arweave and peer work for refs that exist only in a local write + /// the store never saw. + #[test] + fn slot_drop_during_compression_records_a_definite_non_publication() { + let (slot, _cell) = armed_slot(PublishStage::PreparingArchive); + let arc = slot.arc(); + drop(slot); + assert_eq!( + *arc.lock().expect("publish durability mutex poisoned"), + Some(ReleaseOutcome::UploadFailed), + "a cancellation before the PUT was dispatched must be recorded as a definite \ + non-publication, not as unknowable durability" + ); + } + + /// The other side of the same boundary: once the request is on the wire, the + /// outcome genuinely is unknowable, and a `Drop` impl cannot await a HEAD to + /// resolve it. Unknowable is honest here — and the tail refuses it anyway, + /// because the tail requires `Released`. + #[test] + fn slot_drop_after_dispatch_records_unknowable() { + let (slot, _cell) = armed_slot(PublishStage::PutDispatched { + attempt: PublishAttemptId::new(), + }); + let arc = slot.arc(); + drop(slot); + assert_eq!( + *arc.lock().expect("publish durability mutex poisoned"), + Some(ReleaseOutcome::UploadUnknowable), + "a dispatched PUT may still commit, so the drop must not claim it definitely failed" + ); + } + + /// A disconnect AFTER the store acknowledged the publish loses the response, + /// not the write. The tail must still run: this is the case the stage model + /// gains over the old flag, which could only ever say "unknowable" here. + #[test] + fn slot_drop_after_the_store_acknowledged_records_released() { + let (slot, _cell) = armed_slot(PublishStage::Published { + attempt: PublishAttemptId::new(), + etag: Some("\"e1\"".to_string()), + }); + let arc = slot.arc(); + drop(slot); + assert_eq!( + *arc.lock().expect("publish durability mutex poisoned"), + Some(ReleaseOutcome::Released), + "a confirmed publish is durable regardless of what happened to the handler" + ); + } + + /// A node with no object storage configured has nothing to publish, so the + /// pack on local disk IS the durable copy and a disconnect must not withhold + /// the tail. Distinct from `Idle`, where a publish was possible and never + /// started; collapsing the two turns every Tigris-less deployment's pushes + /// into un-replicated ones. + #[test] + fn slot_drop_with_no_storage_backend_records_released() { + let (slot, _cell) = armed_slot(PublishStage::NoBackend); + let arc = slot.arc(); + drop(slot); + assert_eq!( + *arc.lock().expect("publish durability mutex poisoned"), + Some(ReleaseOutcome::Released), + "with no backend there is no publication to confirm, so the tail must run" + ); + } + + #[test] + fn publish_durability_slot_drop_leaves_empty_before_release_starts() { + let slot = PublishDurabilitySlot::new(); + let arc = slot.arc(); + drop(slot); + assert_eq!( + *arc.lock().expect("publish durability mutex poisoned"), + None, + "dropping before release starts must not synthesize durability for the tail" + ); + } + + #[test] + fn publish_durability_slot_drop_waits_for_contended_mutex() { + let (slot, _cell) = armed_slot(PublishStage::PutDispatched { + attempt: PublishAttemptId::new(), + }); + let arc = slot.arc(); + let holder = { + let arc = Arc::clone(&arc); + std::thread::spawn(move || { + let _guard = arc.lock().expect("publish durability mutex poisoned"); + std::thread::sleep(std::time::Duration::from_millis(100)); + }) + }; + std::thread::sleep(std::time::Duration::from_millis(10)); + drop(slot); + holder.join().expect("mutex holder thread"); + let outcome = *arc.lock().expect("publish durability mutex poisoned"); + assert_eq!( + outcome, + Some(ReleaseOutcome::UploadUnknowable), + "drop must block until it can install its verdict, not give up on try_lock" + ); + } + + /// The tail must resolve promptly on an abandoned handler rather than wait + /// out the whole transfer bound — it just resolves to a REFUSAL now. + #[tokio::test] + async fn publish_durability_confirmed_refuses_quickly_after_unrecorded_slot_drop() { + let start = std::time::Instant::now(); + let (slot, _cell) = armed_slot(PublishStage::PreparingArchive); + let arc = slot.arc(); + drop(slot); + let confirmed = + publish_durability_confirmed(&Some(arc), std::time::Duration::from_millis(50)).await; + assert!( + !confirmed, + "a cancellation before dispatch must not admit the replication tail" + ); + assert!( + start.elapsed() < std::time::Duration::from_millis(200), + "the tail must not wait out the full transfer bound once the slot carries a verdict" + ); + } + + #[tokio::test] + async fn publish_durability_confirmed_fails_closed_when_release_never_records() { + let slot = Arc::new(std::sync::Mutex::new(None)); + let confirmed = + publish_durability_confirmed(&Some(slot), std::time::Duration::from_millis(30)).await; + assert!( + !confirmed, + "an empty slot after the bounded wait must not admit the tail; only an installed \ + Released outcome may" + ); + } + + /// P1 FINDING 1, at the gate. The tail replicates to IPFS, Pinata, P2P, + /// GraphQL, Arweave and peers, all of which publish refs to other nodes. Only + /// a CONFIRMED publish may license that. `UploadUnknowable` used to pass here, + /// which is what carried an unattempted (and an unreconciled) write into the + /// network. + #[tokio::test] + async fn publish_durability_confirmed_accepts_only_released() { + let slot = Arc::new(std::sync::Mutex::new(Some(ReleaseOutcome::Released))); + assert!( + publish_durability_confirmed(&Some(slot), std::time::Duration::from_millis(5)).await + ); + + for refused in [ + ReleaseOutcome::UploadUnknowable, + ReleaseOutcome::UploadFailed, + ReleaseOutcome::Fenced, + ] { + let slot = Arc::new(std::sync::Mutex::new(Some(refused))); + assert!( + !publish_durability_confirmed(&Some(slot), std::time::Duration::from_millis(5)) + .await, + "{refused:?} is not a confirmed publish and must not admit the tail" + ); + } + } + + #[test] + fn fork_clone_guard_removes_its_own_mirror_on_drop() { + let root = tempfile::TempDir::new().unwrap(); + let validated = crate::git::repo_store::validated_repo_disk_path( + root.path(), + "did:key:testfork", + "fork", + ) + .expect("test fork path must validate"); + std::fs::create_dir_all(validated.as_path()).unwrap(); + { + let _guard = ForkCloneGuard::new(validated.clone(), PublishAttemptId::new()); + assert!(validated.exists()); + } + assert!( + !validated.exists(), + "dropping the fork clone guard must remove the mirror directory it stamped" + ); + } + + /// P1 FINDING 3, the filesystem half. Two attempts at one logical fork share + /// a disk path. A late-unwinding attempt must not remove the directory a + /// SUCCESSOR now owns — the successor has already published its archive and + /// returned 201 for it. + #[test] + fn fork_clone_guard_leaves_a_successors_mirror_alone() { + let root = tempfile::TempDir::new().unwrap(); + let validated = crate::git::repo_store::validated_repo_disk_path( + root.path(), + "did:key:testfork", + "fork", + ) + .expect("test fork path must validate"); + std::fs::create_dir_all(validated.as_path()).unwrap(); + + let loser = ForkCloneGuard::new(validated.clone(), PublishAttemptId::new()); + // The successor claims the path while the loser is still unwinding. + let successor = PublishAttemptId::new(); + crate::git::repo_store::claim_fork_disk_path(validated.as_path(), &successor); + + drop(loser); + assert!( + validated.exists(), + "a failed attempt must not delete the directory a successor now owns" + ); + } + #[test] fn upload_pack_request_finalizes_only_with_done_pktline() { let want = "0032want 1111111111111111111111111111111111111111\n"; @@ -3525,6 +4375,55 @@ mod tests { assert!(git_permit(&sem).is_ok()); } + #[test] + fn is_expected_transient_matches_both_typed_refusals() { + // The real raise shape wraps the marker in a `.context()` layer naming the + // owner slug and repo, so the downcast has to survive that wrapping. + let busy = anyhow::Error::new(crate::git::repo_store::RepoBusy) + .context("another write is in progress for alice/demo"); + assert!(is_expected_transient_acquire_failure(&busy)); + + let unavailable = anyhow::Error::new(crate::git::repo_store::RepoUnavailable) + .context("could not read the archive HEAD for alice/demo"); + assert!(is_expected_transient_acquire_failure(&unavailable)); + } + + #[test] + fn is_expected_transient_rejects_unrelated_failures() { + // Anything the classifier cannot recognize keeps paging at error level. + let other = anyhow::anyhow!("disk on fire"); + assert!(!is_expected_transient_acquire_failure(&other)); + } + + #[test] + fn acquire_write_app_error_maps_transient_markers_to_retryable_503() { + let busy = anyhow::Error::new(crate::git::repo_store::RepoBusy) + .context("another write is in progress for alice/demo"); + assert!(matches!( + acquire_write_app_error(&busy, "demo"), + AppError::RepoBusy + )); + + let unavailable = anyhow::Error::new(crate::git::repo_store::RepoUnavailable) + .context("could not read the archive HEAD for alice/demo"); + assert!(matches!( + acquire_write_app_error(&unavailable, "demo"), + AppError::RepoUnavailable + )); + + let pool = anyhow::Error::new(crate::git::repo_store::LockPoolBusy); + assert!(matches!( + acquire_write_app_error(&pool, "demo"), + AppError::Overloaded(_) + )); + + let other = anyhow::anyhow!("disk on fire"); + assert!(matches!( + acquire_write_app_error(&other, "demo"), + AppError::Git(_) + )); + } + fn repo_owned_by(owner_did: &str) -> crate::db::RepoRecord { let now = chrono::Utc::now(); crate::db::RepoRecord { @@ -6255,6 +7154,7 @@ mod tests { repos_dir.path().to_path_buf(), None, crate::git::repo_store::build_lock_pool(&pool, 4, std::time::Duration::from_secs(5)), + std::time::Duration::from_secs(300), ); let mut cfg = (*state.config).clone(); // Long enough that the git-service timeout is never what ends this push; the @@ -6302,9 +7202,7 @@ mod tests { let mut fut = Box::pin(git_receive_pack( State(state.clone()), Path((owner.to_string(), name.to_string())), - Extension(crate::auth::AuthenticatedDid( - "did:key:z6MkDisconnectWriteLockProofDidAAAAAAAA".to_string(), - )), + Extension(crate::auth::AuthenticatedDid("did:key:z6disc".to_string())), crate::rate_limit::PeerAddr(Some( "203.0.113.81:5000".parse::().unwrap(), )), @@ -6429,6 +7327,7 @@ mod tests { repos_dir.path().to_path_buf(), None, crate::git::repo_store::build_lock_pool(&pool, 4, std::time::Duration::from_secs(5)), + std::time::Duration::from_secs(300), ); state .db @@ -6453,9 +7352,7 @@ mod tests { git_receive_pack( State(state.clone()), Path((owner.to_string(), name.to_string())), - Extension(crate::auth::AuthenticatedDid( - "did:key:z6MkPushSuccessReleaseProofDidAAAAAAAA".to_string(), - )), + Extension(crate::auth::AuthenticatedDid("did:key:z6succ".to_string())), crate::rate_limit::PeerAddr(Some( "203.0.113.83:5000".parse::().unwrap(), )), @@ -6522,6 +7419,7 @@ mod tests { std::path::PathBuf::from("/tmp/gitlawb-lockpool-shed"), None, crate::git::repo_store::build_lock_pool(&pool, 1, std::time::Duration::from_secs(1)), + std::time::Duration::from_secs(300), ); state .db @@ -6529,7 +7427,7 @@ mod tests { .await .unwrap(); - let did = "did:key:z6MkLockPoolShedProofDidAAAAAAAAAAAAAAAAAA"; + let did = "did:key:z6lockpool"; let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); // Occupy the only lock-pool connection with a write on an UNRELATED repo. @@ -6556,7 +7454,7 @@ mod tests { // MUST-NOT: with the pool free again, the push is not shed as capacity (it fails // later on the nonexistent on-disk repo, which is a git error, not Overloaded). - held.release(false).await; + let _ = held.release(false).await; let admitted = git_receive_pack( State(state.clone()), Path((owner.to_string(), name.to_string())), @@ -9631,6 +10529,43 @@ mod tests { const F2A_PUSHER: &str = "did:key:z6MkF2aPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + /// When publish durability is fenced, the tail must not run walks or take + /// coalescing keys before returning. + #[cfg(unix)] + #[sqlx::test] + async fn post_receive_tail_skips_all_work_when_publish_fenced(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let (state, rec) = f2a_state(pool, &git_bin, "z6f2afence", "fence-repo", false).await; + let slot = Arc::new(std::sync::Mutex::new(Some( + crate::git::repo_store::ReleaseOutcome::Fenced, + ))); + post_receive_replication_tail( + state.clone(), + rec, + f2a_update("refs/heads/main", &c1), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + Some(slot), + ) + .await; + assert_eq!( + f2a_walks(&log), + 0, + "a fenced publish must not run the replication walk; log:\n{}", + f2a_log(&log) + ); + assert_eq!( + state.encrypt_inflight.len(), + 0, + "a fenced publish must not take the per-repo coalescing key" + ); + } + /// Scenario 1 (the finding). A second rapid push to the same repo coalesces /// WITHOUT running the withheld walk. Asserted on the walk's git children, not /// on the `Coalesced` outcome: with `try_begin` below the walk (the pre-fix @@ -9660,6 +10595,7 @@ mod tests { f2a_update("refs/heads/main", &c2), repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, ) .await; let after_first = f2a_walks(&log); @@ -9680,6 +10616,7 @@ mod tests { f2a_update("refs/heads/second", &c1), repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, ) .await; @@ -9753,6 +10690,7 @@ mod tests { f2a_update("refs/heads/main", &c2), repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, ) .await; @@ -9834,6 +10772,7 @@ mod tests { f2a_update("refs/heads/main", &c2), repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, )); f2a_wait_for(|| started.exists(), "the admitted push's walk to start").await; @@ -9947,6 +10886,7 @@ mod tests { }], repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, ) .await; @@ -10177,6 +11117,344 @@ mod tests { ); } + // ── #285 P1 finding 1: cancellation before the PUT is dispatched ─────── + + /// A minimal object-store stub that COUNTS PUTs and answers every HEAD 404. + /// + /// Not a semantics mock: the whole claim of the tests below is that no PUT + /// ever arrives, so the only thing it has to do faithfully is notice one. + async fn p3_put_counting_store() -> ( + String, + Arc, + tokio::task::JoinHandle<()>, + ) { + let puts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any({ + let puts = Arc::clone(&puts); + move |method: axum::http::Method, _body: axum::body::Bytes| { + let puts = Arc::clone(&puts); + async move { + if method == axum::http::Method::PUT { + puts.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + return axum::http::StatusCode::OK; + } + // Nothing is stored, so the acquire-side refresh takes the + // create-only arm and downloads nothing. + axum::http::StatusCode::NOT_FOUND + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + (endpoint, puts, server) + } + + /// A push handler whose release-side publish parks INSIDE the blocking + /// compression, which is the window the finding is about: the conditional PUT + /// is not constructed until compression returns, so a handler cancelled here + /// definitely never attempted publication. + /// + /// Path-scoped for the same reason as `p2_parked_release_state`: the tail's + /// withheld walk is the observable, and without a path-scoped rule + /// `replication_withheld_set` takes the no-walk shortcut and spawns no git. + /// + /// The bare repo is created directly rather than through `repo_store.init`, + /// which would spawn a background create-only upload of its own and pollute + /// the PUT count this test reads. + #[cfg(unix)] + async fn p3_compression_gated_state( + pool: sqlx::PgPool, + tmp: &std::path::Path, + owner: &str, + name: &str, + gate: Arc, + ) -> ( + AppState, + std::path::PathBuf, + Arc, + tokio::task::JoinHandle<()>, + ) { + let log = tmp.join("git.log"); + let git_bin = f2a_logging_git(tmp, &log); + let repos_dir = tmp.join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let (endpoint, puts, server) = p3_put_counting_store().await; + + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_bin = git_bin; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.clone(), + Some( + crate::git::tigris::TigrisClient::for_testing_with_endpoint( + "test-bucket", + &endpoint, + ) + .with_compress_gate(gate), + ), + pool.clone(), + std::time::Duration::from_secs(300), + ); + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{owner}-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkP3TailReaderAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + let bare = + crate::git::repo_store::validated_repo_disk_path(&repos_dir, &rec.owner_did, name) + .expect("test repo path"); + crate::git::store::init_bare(&bare).expect("a bare repo on disk"); + (state, log, puts, server) + } + + /// #285 P1 FINDING 1 (RED-before/GREEN-after). A client disconnect while the + /// release-side archive is still being COMPRESSED must not count as publish + /// durability. + /// + /// `TigrisClient::upload` awaits `spawn_blocking(compress_repo)` and does not + /// construct, let alone send, the conditional PUT until that returns. So a + /// handler dropped in this window definitively never attempted publication — + /// and the detached tail must do NOTHING: no IPFS/Pinata pin, no P2P + /// announcement, no GraphQL publication, no Arweave work, no peer + /// notification, all for refs that exist only in a local write the store + /// never saw. + /// + /// Load-bearing: with the slot recording `UploadUnknowable` for every + /// cancellation after release starts (the pre-fix shape), the tail is + /// admitted and its withheld walk's `for-each-ref` appears in the git log + /// (RED). Reading the publish STAGE instead classifies this as a definite + /// non-publication and the walk never runs (GREEN). + /// + /// The existing `receive_pack_tail_survives_a_disconnect_during_release` + /// parks at the pre-unlock point, which is AFTER the upload, so it cannot + /// reach this boundary at all. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_cancelled_during_compression_publishes_nothing_and_runs_no_tail( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + // Shut: every compression from this store parks until the gate opens. + let gate = Arc::new(crate::git::tigris::BlockingGate::shut()); + let (state, log, puts, server) = + p3_compression_gated_state(pool, tmp.path(), "z6p3comp", "c1", Arc::clone(&gate)).await; + + let mut fut = Box::pin(p2_push(&state, "z6p3comp", "c1")); + let mut ran = false; + for _ in 0..1000 { + let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + assert!( + step.is_err(), + "the handler must park inside the release-side compression, not return" + ); + if p2_logged(&log, "receive-pack") { + ran = true; + break; + } + } + assert!(ran, "the push must reach receive-pack"); + // Settle: the handler is now inside `release`, blocked on the gate with + // no PUT constructed. + for _ in 0..10 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + } + assert_eq!( + puts.load(std::sync::atomic::Ordering::SeqCst), + 0, + "compression has not finished, so no PUT can have been built yet" + ); + + // THE DISCONNECT, inside the compression window. + drop(fut); + + // Give the tail every chance to misbehave. Pre-fix it is admitted the + // instant the slot's Drop installs its verdict, so this window is far + // more than enough for the walk to show up. + tokio::time::sleep(std::time::Duration::from_millis(1500)).await; + + assert!( + !p2_logged(&log, "for-each-ref"), + "RED: a push cancelled before its PUT was even constructed still ran the \ + replication tail. Nothing was published, so pinning, announcing and \ + replicating these refs advertises a write no other node can read. git log:\n{}", + f2a_log(&log) + ); + assert_eq!( + puts.load(std::sync::atomic::Ordering::SeqCst), + 0, + "no PUT may have reached the store at any point" + ); + + // Release the parked blocking task so the runtime can tear down cleanly. + gate.open(); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + server.abort(); + } + + /// THE CONTROL, and what makes the assertion above attributable. Same setup, + /// same handler, but nothing holds the gate: the publish completes, the store + /// sees the PUT, and the tail DOES run its walk. + /// + /// Without this, a green negative would prove only that the walk never runs + /// in this harness, not that the CANCELLATION is what stops it. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_that_completes_its_publish_still_runs_the_tail(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + // Open from the start: this control must publish for real. + let gate = Arc::new(crate::git::tigris::BlockingGate::shut()); + gate.open(); + let (state, log, puts, server) = + p3_compression_gated_state(pool, tmp.path(), "z6p3ctrl", "c1", gate).await; + + p2_push(&state, "z6p3ctrl", "c1") + .await + .expect("an uncontended push must succeed"); + + assert_eq!( + puts.load(std::sync::atomic::Ordering::SeqCst), + 1, + "the release must have published exactly once" + ); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !p2_logged(&log, "for-each-ref") { + assert!( + std::time::Instant::now() < deadline, + "a confirmed publish must admit the replication tail; git log:\n{}", + f2a_log(&log) + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + + server.abort(); + } + + // ── #285 P1 finding 3: fork confirmation is bound to the attempt ─────── + + /// A repo row under `owner/name` owned by some OTHER attempt. + async fn p3_seed_foreign_row( + state: &AppState, + owner_did: &str, + name: &str, + ) -> crate::db::RepoRecord { + let now = Utc::now(); + let rec = crate::db::RepoRecord { + id: Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner_did.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/unused/{name}"), + forked_from: None, + machine_id: None, + }; + state + .db + .create_repo(&rec) + .await + .expect("seed the other row"); + rec + } + + /// #285 P1 FINDING 3, interleaving (a): another row COMMITS UNDER THE NAME + /// before this attempt's confirmation runs. + /// + /// The ordering is the barrier: the successor's insert completes, and only + /// then does the failed attempt look up. Keyed on owner/name, that lookup + /// answered "a row exists" and the request returned 201 carrying the + /// successor's row — even though its own uploaded archive, disk path and fork + /// provenance belong to no row at all. Keyed on `record.id` it cannot. + #[sqlx::test] + async fn fork_confirmation_never_claims_a_concurrent_attempts_row(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let owner = "did:key:z6MkForkIdentityAAAAAAAAAAAAAAAAAAAAAAAA"; + let short = crate::db::normalize_owner_key(owner); + let name = "contested-fork"; + + // Interleaving: the successor commits first. + let successor = p3_seed_foreign_row(&state, owner, name).await; + + // Then this attempt, whose own insert did NOT land, confirms. + let mine = Uuid::new_v4().to_string(); + let confirmation = confirm_fork_repo_row(&state.db, &mine, short, name) + .await + .expect("the lookup itself succeeds"); + match confirmation { + ForkRowConfirmation::Foreign => {} + ForkRowConfirmation::Ours(row) => panic!( + "the failed attempt claimed the successor's row {} as its own commit", + row.id + ), + ForkRowConfirmation::Absent => { + panic!("a row for this name does exist, so Absent would license compensation") + } + } + assert_ne!(successor.id, mine); + } + + /// The must-do direction: when this attempt's OWN insert did commit, the + /// confirmation has to recognize it, or a transport-level error on a + /// successful write would turn into a spurious failure plus compensation of + /// a live repository. + #[sqlx::test] + async fn fork_confirmation_recognizes_this_attempts_own_committed_row(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let owner = "did:key:z6MkForkIdentityBBBBBBBBBBBBBBBBBBBBBBBB"; + let short = crate::db::normalize_owner_key(owner); + let name = "my-fork"; + + let mine = p3_seed_foreign_row(&state, owner, name).await; + match confirm_fork_repo_row(&state.db, &mine.id, short, name) + .await + .expect("lookup") + { + ForkRowConfirmation::Ours(row) => assert_eq!(row.id, mine.id), + other => panic!( + "this attempt's own row must confirm as Ours, got {}", + match other { + ForkRowConfirmation::Foreign => "Foreign", + ForkRowConfirmation::Absent => "Absent", + ForkRowConfirmation::Ours(_) => unreachable!(), + } + ), + } + } + + /// And the third arm: nothing owns the name, which is the only state that + /// licenses compensation at all. + #[sqlx::test] + async fn fork_confirmation_reports_absent_when_nothing_owns_the_name(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let short = "z6MkForkIdentityCCCCCCCCCCCCCCCCCCCCCCCC"; + assert!(matches!( + confirm_fork_repo_row(&state.db, &Uuid::new_v4().to_string(), short, "nobody-here") + .await + .expect("lookup"), + ForkRowConfirmation::Absent + )); + } + /// Scenario 5 (trap 3, fail-closed). On a repo whose withheld walk is failing, a /// coalesced push must not publish. Before the gate moved, every push on such a /// repo got `announce = false` from its own walk; a coalesced push has no walk, so @@ -10225,6 +11503,7 @@ mod tests { }], repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, ) .await; @@ -10280,6 +11559,7 @@ mod tests { f2a_update("refs/heads/main", &c1), repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, ) .await; tokio::time::sleep(std::time::Duration::from_millis(300)).await; @@ -10321,6 +11601,7 @@ mod tests { }], repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, ) .await; @@ -10418,6 +11699,7 @@ mod tests { }], repo.path().to_path_buf(), F2A_PUSHER.to_string(), + None, ) .await; diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 27b67786e..6d81dbd4e 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -517,6 +517,7 @@ mod tests { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + close_issue_rate_limiter: RateLimiter::new(120, Duration::from_secs(3600)), ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 1fbf4376e..fc7dfc987 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -252,11 +252,9 @@ pub struct Config { /// Maximum connections in the PostgreSQL pool. This is a cap, not a floor /// (connections open lazily). Size against the database server's - /// max_connections, remembering admin tooling opens its own pool. Each - /// concurrent write pins one pooled connection for its whole duration (the - /// advisory lock in `repo_store::acquire_write` is connection-affine), so this - /// must exceed `max_concurrent_git_pushes` by `DB_POOL_APP_HEADROOM` or slow - /// pushes starve every other DB path — enforced by `Config::validate`. + /// max_connections, remembering admin tooling opens its own pool. Writes pin + /// the dedicated lock pool, not this one; this pool still needs enough headroom + /// for ordinary request handlers and metadata paths (`DB_POOL_APP_HEADROOM`). #[arg( long, env = "GITLAWB_DB_MAX_CONNECTIONS", @@ -265,6 +263,46 @@ pub struct Config { )] pub db_max_connections: u32, + /// Maximum connections in the dedicated advisory-lock pool, which is separate + /// from the main pool above. + /// + /// Size this against the expected peak number of concurrent distinct-repo + /// writers, NOT small. Every in-flight repo write pins one connection here for + /// its whole duration (the write, its metadata tail, and the bounded archive + /// upload), so this value is a hard ceiling on simultaneous writes node-wide. + /// Keeping it separate from GITLAWB_DB_MAX_CONNECTIONS is what stops a push + /// burst from starving ordinary request handlers; the cost is that + /// (main pool + lock pool) must fit inside the database server's + /// max_connections, times the number of nodes, plus admin tooling. + #[arg( + long, + env = "GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS", + default_value_t = 40, + value_parser = clap::value_parser!(u32).range(1..) + )] + pub db_lock_pool_max_connections: u32, + + /// Upper bound, in seconds, on any single object-storage transfer that runs + /// while the per-repo advisory lock is HELD. + /// + /// Two bounded spans exist, and the bound applies per span. The acquire-side + /// refresh in `acquire_write` covers the existence HEAD and the download + /// together under ONE budget (it runs after the lock is taken and before the + /// guard is constructed), and the archive upload in `release` gets its own. + /// Worst-case slot occupancy is therefore about twice this value plus the git + /// work between them, not one times this value. Both used to be free, because the lock's + /// connection was returned to the pool immediately; now that a write guard + /// pins a lock-pool connection for its whole lifetime, an unbounded transfer + /// holds that slot, and enough stalled transfers deny every write on the node. + /// This is the bound that keeps a stall from becoming an outage. + #[arg( + long, + env = "GITLAWB_LOCK_HELD_TRANSFER_TIMEOUT_SECS", + default_value_t = 300, + value_parser = clap::value_parser!(u64).range(1..) + )] + pub lock_held_transfer_timeout_secs: u64, + /// Maximum time a request waits for a pool connection before failing with /// 503, in seconds. Bounds queueing when the database is slow or down. #[arg( @@ -359,15 +397,14 @@ pub struct Config { /// error rather than a boot-time panic). /// /// CONNECTION BUDGET. A push holds a Postgres connection from the node's separate - /// advisory-lock pool for the whole receive-pack, and that pool is sized from this - /// knob (this value + 8, clamped to 64 in `main.rs`). The node's total ceiling is - /// therefore `db_max_connections` (default 48) + the lock pool (default 40), i.e. - /// 88 by default, and at most `db_max_connections` + 64. Size BOTH against the - /// database server's `max_connections`: `db_max_connections`' own doc predates the - /// lock pool and no longer covers most of the node's connections. The +8 headroom - /// is shared with the three non-push `acquire_write` callers (`api/issues.rs` x2, - /// `api/pulls.rs`). Raising this knob past the clamp does NOT buy more lock-pool - /// connections; pushes beyond it wait briefly and then shed a 503 + Retry-After. + /// advisory-lock pool (`GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS`, default 40) for + /// the whole receive-pack. That pool must be at least this value plus + /// `DB_LOCK_POOL_NON_PUSH_HEADROOM` so admitted pushes and non-push mutations + /// can each pin a lock-pool connection for their whole duration. Size BOTH + /// pools against the database server's `max_connections`: the main pool + /// (`GITLAWB_DB_MAX_CONNECTIONS`, default 48) serves ordinary handlers, and the + /// lock pool serves writes. Raising this knob does not raise the lock pool; + /// set `GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS` explicitly. #[arg( long, env = "GITLAWB_MAX_CONCURRENT_GIT_PUSHES", @@ -725,25 +762,37 @@ impl Config { /// the pool must clear the concurrent-write cap by at least this margin. pub const DB_POOL_APP_HEADROOM: u32 = 8; + /// Lock-pool slots reserved for non-push `acquire_write` callers (issue create, + /// close, merge) so a saturated push budget cannot turn ordinary mutations into + /// lock-pool exhaustion on unrelated repositories. + pub const DB_LOCK_POOL_NON_PUSH_HEADROOM: u32 = 8; + /// Cross-field boot validation. Single-field ranges are enforced by clap; this /// catches combinations that ship a denial-of-service under otherwise-valid /// values. Call once at startup and fail fast on `Err`. pub fn validate(&self) -> Result<(), String> { - // A write pins one pooled connection for its whole duration (the - // connection-affine advisory lock in repo_store::acquire_write), and - // concurrent writes are capped at max_concurrent_git_pushes. If the pool - // does not exceed that cap by DB_POOL_APP_HEADROOM, a burst of slow pushes - // drains every connection and every other DB path 503s. (#174 F1) - let floor = (self.max_concurrent_git_pushes as u64) + (Self::DB_POOL_APP_HEADROOM as u64); - if (self.db_max_connections as u64) < floor { + // Concurrent git writes pin the dedicated lock pool for their whole + // duration (connection-affine advisory lock in acquire_write). The main + // pool no longer carries that occupancy. + let lock_floor = self + .max_concurrent_git_pushes + .saturating_add(Self::DB_LOCK_POOL_NON_PUSH_HEADROOM as usize); + if (self.db_lock_pool_max_connections as usize) < lock_floor { return Err(format!( - "GITLAWB_DB_MAX_CONNECTIONS ({}) must be at least max_concurrent_git_pushes ({}) \ - + {} headroom = {}: each concurrent write pins one pooled connection for its whole \ - duration, so a smaller pool lets a burst of slow pushes starve every other DB path.", - self.db_max_connections, + "GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS ({}) must be at least \ + max_concurrent_git_pushes ({}) plus {} for non-push mutations \ + that share the lock pool", + self.db_lock_pool_max_connections, self.max_concurrent_git_pushes, - Self::DB_POOL_APP_HEADROOM, - floor + Self::DB_LOCK_POOL_NON_PUSH_HEADROOM + )); + } + let main_floor = Self::DB_POOL_APP_HEADROOM as u64; + if (self.db_max_connections as u64) < main_floor { + return Err(format!( + "GITLAWB_DB_MAX_CONNECTIONS ({}) must be at least {} for non-git \ + database paths", + self.db_max_connections, main_floor )); } Ok(()) @@ -754,6 +803,25 @@ impl Config { mod tests { use super::*; + #[test] + fn lock_pool_size_defaults_to_40_and_rejects_zero() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).db_lock_pool_max_connections, + 40 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--db-lock-pool-max-connections", "8"]) + .db_lock_pool_max_connections, + 8 + ); + // A zero-sized lock pool would deny every write, so clap must reject it + // rather than let a node boot into a state where no repo can be written. + assert!( + Config::try_parse_from(["gitlawb-node", "--db-lock-pool-max-connections", "0"]) + .is_err() + ); + } + #[test] fn git_service_timeout_defaults_to_600_and_rejects_zero() { assert_eq!( @@ -1363,41 +1431,61 @@ mod tests { ); } - /// #174 F1: a connection-affine write lock pins a pooled connection per - /// concurrent write, so the pool must clear `max_concurrent_git_pushes` by - /// `DB_POOL_APP_HEADROOM` or a push burst starves every other DB path. - /// `validate()` must reject an under-sized pool at boot. + /// #174 F1: concurrent git writes pin the dedicated lock pool, not the main + /// pool. `validate()` must reject an under-sized lock pool at boot. #[test] fn db_pool_must_clear_the_git_push_cap() { - // Shipped defaults validate (48 >= 32 + 8). + // Shipped defaults validate (lock pool 40 >= pushes 32 + non-push headroom 8). Config::parse_from(["gitlawb-node"]) .validate() .expect("default config must validate"); - // An under-sized pool relative to the push cap is rejected (20 < 32 + 8). - let under = Config::parse_from([ + // An under-sized lock pool relative to the push cap is rejected. + let under_lock = Config::parse_from([ "gitlawb-node", - "--db-max-connections", - "20", + "--db-lock-pool-max-connections", + "16", "--max-concurrent-git-pushes", "32", ]); assert!( - under.validate().is_err(), - "db_max_connections 20 below max_concurrent_git_pushes 32 + headroom must be rejected" + under_lock.validate().is_err(), + "db_lock_pool_max_connections below max_concurrent_git_pushes + headroom must be rejected" ); - // Exactly at the floor validates (40 == 32 + 8). - let at_floor = Config::parse_from([ + // Exactly at the push cap without non-push headroom is rejected. + let push_only = Config::parse_from([ + "gitlawb-node", + "--db-lock-pool-max-connections", + "32", + "--max-concurrent-git-pushes", + "32", + ]); + assert!( + push_only.validate().is_err(), + "db_lock_pool_max_connections equal to max_concurrent_git_pushes must be rejected" + ); + + // Main pool can be smaller than pushes + headroom when the lock pool carries writes. + let split = Config::parse_from([ "gitlawb-node", "--db-max-connections", + "16", + "--db-lock-pool-max-connections", "40", "--max-concurrent-git-pushes", "32", ]); assert!( - at_floor.validate().is_ok(), - "db_max_connections at the floor (pushes + headroom) must validate" + split.validate().is_ok(), + "a small main pool with a large lock pool must validate" + ); + + // Main pool still needs the app headroom floor. + let under_main = Config::parse_from(["gitlawb-node", "--db-max-connections", "4"]); + assert!( + under_main.validate().is_err(), + "db_max_connections below DB_POOL_APP_HEADROOM must be rejected" ); } diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index cc2cf0bd7..171e08e28 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -265,11 +265,6 @@ pub struct Db { } impl Db { - /// Access the underlying Postgres connection pool. - pub fn pool(&self) -> &PgPool { - &self.pool - } - #[cfg(test)] pub fn for_testing(pool: PgPool) -> Self { Self { pool } @@ -310,6 +305,46 @@ impl Db { Ok(db) } + /// Build the dedicated pool that advisory-lock connections come from. + /// + /// Deliberately **lazy**: connections open on first use rather than at boot. + /// The main pool has to connect eagerly because it runs migrations, which is + /// why it needs `connect_db_with_retry`'s backoff and degraded-server + /// handoff. This pool has no startup work at all, so an eager connect would + /// only add a new way for the process to fail to boot, and would need a + /// second copy of that retry machinery to be safe. Being lazy removes the + /// failure mode instead of handling it: if Postgres is unreachable when the + /// first write arrives, that write fails on the pool's own acquire timeout, + /// the same way any other database-backed request already does. + /// + /// Kept separate from the main pool so a burst of lock-holding connections + /// cannot starve ordinary request handlers. See + /// `GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS` for the sizing tradeoff. + pub fn lock_pool( + database_url: &str, + max_connections: u32, + acquire_timeout: Duration, + ) -> Result { + info!( + max_connections, + acquire_timeout_secs = acquire_timeout.as_secs(), + "creating dedicated advisory-lock pool (lazy)" + ); + PgPoolOptions::new() + .max_connections(max_connections) + // Explicit, and load-bearing rather than cosmetic: `RepoWriteGuard::Drop` + // has a no-runtime branch that calls `PoolConnection::leak()` and relies + // on the husk's own drop doing nothing. With `min_connections > 0` that + // drop still spawns a pool-replenish task, and spawning without a runtime + // panics inside Drop, which aborts the process during unwind. It is 0 by + // default, so this line exists to keep a future tuning change from + // silently re-arming that panic. + .min_connections(0) + .acquire_timeout(acquire_timeout) + .connect_lazy(database_url) + .context("creating advisory-lock pool") + } + /// Cheap liveness probe against the pool, for readiness checks: one /// `SELECT 1` that fails fast when the database is unreachable. pub async fn ping(&self) -> Result<()> { diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index 474408e59..cb8f77115 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -69,6 +69,15 @@ pub enum AppError { #[error("server overloaded: {0}")] Overloaded(String), + #[error("repository is busy")] + RepoBusy, + + #[error("repository is temporarily unavailable")] + RepoUnavailable, + + #[error("repository write was fenced by a concurrent publish")] + RepoWriteFenced, + #[error("database error: {0}")] Db(#[from] sqlx::Error), @@ -110,7 +119,31 @@ impl From for AppError { fn from(err: anyhow::Error) -> Self { match err.downcast::() { Ok(sql) => AppError::Db(sql), - Err(err) => AppError::Internal(err), + // Lock contention is transient and ordinary, so it must not land as a + // 500. The internal message names the owner slug and repo, so the + // variant carries nothing: the detail stays in the log at the raise + // site and the client gets a fixed retryable body. + Err(err) => match err.downcast::() { + Ok(_) => AppError::Overloaded("git write locks at capacity, retry shortly".into()), + Err(err) => match err.downcast::() { + Ok(_) => AppError::RepoBusy, + // Same reasoning one rung down: a refused under-lock refresh is a + // transient storage condition, and its internal message names the + // owner slug and repo, so the variant carries nothing. + Err(err) => match err.downcast::() { + Ok(_) => AppError::RepoUnavailable, + // And one more rung: a publish the store refused twice is + // transient in the same way, and the retry is the client's + // to make. The variant carries nothing for the same reason + // as the two above. + Err(err) => match err.downcast::() + { + Ok(_) => AppError::RepoWriteFenced, + Err(err) => AppError::Internal(err), + }, + }, + }, + }, } } } @@ -184,6 +217,29 @@ impl IntoResponse for AppError { // 504, distinct from the 500 git_error and from the read-gate's 404 / // the auth 401, so the client can tell a deadline from a failure. AppError::Timeout(msg) => (StatusCode::GATEWAY_TIMEOUT, "git_timeout", msg.clone()), + // 503 with a FIXED body: the caller should retry, and must not be told + // which repo is contended or for how long. + AppError::RepoBusy => ( + StatusCode::SERVICE_UNAVAILABLE, + "repo_busy", + "repository is busy — retry".into(), + ), + // 503 with a FIXED body for the same reason: the caller should retry, and + // must not be told which repo could not be refreshed or why. + AppError::RepoUnavailable => ( + StatusCode::SERVICE_UNAVAILABLE, + "repo_unavailable", + "repository is temporarily unavailable, retry".into(), + ), + // 503 with a FIXED body again, and its own code: the caller should + // retry, but the condition is not contention, so a client that + // distinguishes them should be able to. The body must not say which + // repo lost its publish or to whom. + AppError::RepoWriteFenced => ( + StatusCode::SERVICE_UNAVAILABLE, + "repo_write_fenced", + "repository changed underneath this write, retry".into(), + ), AppError::Db(e) if db_unavailable(e) => ( StatusCode::SERVICE_UNAVAILABLE, DB_UNAVAILABLE_CODE, @@ -244,7 +300,11 @@ impl IntoResponse for AppError { // here rather than in bespoke early returns, keeping each variant handled once. if matches!( self, - AppError::Overloaded(_) | AppError::SearchIncomplete { .. } + AppError::Overloaded(_) + | AppError::SearchIncomplete { .. } + | AppError::RepoBusy + | AppError::RepoUnavailable + | AppError::RepoWriteFenced ) { resp.headers_mut().insert( axum::http::header::RETRY_AFTER, @@ -362,4 +422,31 @@ mod tests { let resp = err.into_response(); assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); } + + #[test] + fn lock_pool_busy_via_anyhow_from_is_503_overloaded() { + let err: AppError = anyhow::Error::new(crate::git::repo_store::LockPoolBusy).into(); + let resp = err.into_response(); + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + resp.headers().get("retry-after").unwrap().to_str().unwrap(), + "1" + ); + } + + #[test] + fn repo_busy_unavailable_and_fenced_advertise_retry_after() { + for err in [ + AppError::RepoBusy, + AppError::RepoUnavailable, + AppError::RepoWriteFenced, + ] { + let resp = err.into_response(); + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + resp.headers().get("retry-after").unwrap().to_str().unwrap(), + "1" + ); + } + } } diff --git a/crates/gitlawb-node/src/git/issues.rs b/crates/gitlawb-node/src/git/issues.rs index 730983020..b20f171c9 100644 --- a/crates/gitlawb-node/src/git/issues.rs +++ b/crates/gitlawb-node/src/git/issues.rs @@ -61,6 +61,27 @@ pub fn create_issue(repo_path: &Path, issue_id: &str, json: &str) -> Result<()> Ok(()) } +/// Remove a single issue ref while the write guard still holds the advisory lock, +/// after a definite publish refusal. Best-effort: a failed delete leaves a +/// local-only orphan, which is still better than telling the client to retry into a +/// duplicate id. Not used on `UploadUnknowable`, where the PUT may still land. +pub fn delete_issue_ref( + git_bin: &str, + repo_path: &Path, + issue_id: &str, + deadline: std::time::Instant, +) -> Result<()> { + let ref_name = format!("refs/gitlawb/issues/{issue_id}"); + crate::git::visibility_pack::run_bounded_git( + git_bin, + &["update-ref", "-d", &ref_name], + repo_path, + b"", + deadline, + )?; + Ok(()) +} + /// List all issue refs and return their JSON content. pub fn list_issues(repo_path: &Path) -> Result> { // List all refs under refs/gitlawb/issues/ @@ -278,6 +299,27 @@ mod tests { assert!(result.unwrap_err().to_string().contains("ambiguous")); } + #[test] + fn delete_issue_ref_removes_a_created_ref() { + let dir = TempDir::new().unwrap(); + init_repo(&dir); + let full_id = "eee88888-0000-0000-0000-000000000000"; + create_issue( + dir.path(), + full_id, + r#"{"id":"eee88888-0000-0000-0000-000000000000","status":"open"}"#, + ) + .unwrap(); + delete_issue_ref( + "git", + dir.path(), + full_id, + std::time::Instant::now() + std::time::Duration::from_secs(30), + ) + .unwrap(); + assert_eq!(resolve_issue_id(dir.path(), full_id).unwrap(), None); + } + #[test] fn test_close_issue_via_prefix() { let dir = TempDir::new().unwrap(); diff --git a/crates/gitlawb-node/src/git/mod.rs b/crates/gitlawb-node/src/git/mod.rs index 59e34c843..0c880b42f 100644 --- a/crates/gitlawb-node/src/git/mod.rs +++ b/crates/gitlawb-node/src/git/mod.rs @@ -1,4 +1,5 @@ pub mod issues; +pub mod publish; pub mod push_delta; pub mod repo_store; pub mod smart_http; diff --git a/crates/gitlawb-node/src/git/publish.rs b/crates/gitlawb-node/src/git/publish.rs new file mode 100644 index 000000000..d1222a670 --- /dev/null +++ b/crates/gitlawb-node/src/git/publish.rs @@ -0,0 +1,358 @@ +//! The publication boundary: WHO wrote, and HOW FAR that write got. +//! +//! Four lifecycle defects on this branch shared one root cause. A resource was +//! tracked by a coarse state — "release started", "the path exists", "a row with +//! this name exists" — when the safety question was about the IDENTITY and the +//! PUBLICATION STAGE of one specific write attempt. A tail replicated refs from +//! an attempt that had not dispatched a PUT; a read served a tree whose +//! generation nobody had confirmed; a fork claimed a concurrent attempt's row and +//! deleted a successor's object; a response-loss failure was compensated as if it +//! proved non-publication. +//! +//! This module is the vocabulary that makes those questions answerable, and it is +//! deliberately free of any object-storage type: +//! +//! - [`PublishAttemptId`] — minted before the request is built, carried with the +//! bytes as user metadata, read back off the store to decide whether what is +//! published is THIS attempt's work. +//! - [`PublishStage`] / [`PublishStageCell`] — how far one attempt got, observable +//! from outside the future that is performing it, so a CANCELLED attempt can +//! still be classified. +//! - [`UploadError`] — what the client KNOWS about dispatch and commit, not just +//! which HTTP status came back. Destructive compensation is licensed only by +//! [`UploadError::proves_not_published`]. +//! +//! # Porting note (#79) +//! +//! PR #79 (`feat/storage-abstraction`) deletes `git/tigris.rs` and replaces it +//! with a `BlobStore`/`RepoArchive` layer. Everything in this file is +//! backend-agnostic on purpose and is meant to survive that swap unchanged: a new +//! backend supplies its own error classifier (the one in `tigris.rs` is a single +//! private function) and keeps these types as the contract its callers read. + +use std::sync::Mutex; + +/// A durable identity for ONE publish attempt. +/// +/// The point is reconciliation. A conditional PUT whose response was lost leaves +/// the client unable to say whether its bytes committed; an attempt id stored +/// alongside those bytes turns that into a question the store can answer, because +/// "is the published object mine?" is decidable where "did my request succeed?" +/// is not. +/// +/// Fork creation passes the DB row id it is about to insert, so the object, the +/// on-disk clone and the database row are all stamped with the same attempt +/// identity and every cleanup can be made conditional on still owning all three. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PublishAttemptId(String); + +impl PublishAttemptId { + /// A fresh identity for an attempt that has nothing else to be named after. + pub fn new() -> Self { + Self(uuid::Uuid::new_v4().to_string()) + } + + /// Name the attempt after a caller-owned identity — fork creation uses the + /// `record.id` it is about to insert, which is what ties the object back to + /// the exact row rather than to the logical owner/name. + pub fn from_owned(id: impl Into) -> Self { + Self(id.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Default for PublishAttemptId { + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Display for PublishAttemptId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// The object user-metadata key an attempt id travels in. +/// +/// S3 lowercases user-metadata keys, so this must already be lowercase or the +/// read back would never match what was written. +pub const ATTEMPT_METADATA_KEY: &str = "gitlawb-attempt"; + +/// The precondition an upload is fenced on. +/// +/// Object storage is the only place a fence can hold. Dropping the future of an +/// in-flight PUT does not cancel the request the server is already processing, +/// so no amount of local locking stops an abandoned writer's bytes from landing +/// after a successor has published. A conditional PUT the store itself refuses +/// is what actually stops it. +#[derive(Clone, Debug)] +pub enum UploadPrecondition { + /// Publish only if the stored object is still the generation we observed. + IfMatch(String), + /// Publish only if nothing is stored under the key yet. + IfAbsent, + /// No fence. Last writer wins. + Unconditional, +} + +/// What the store holds under a key right now. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StoredGeneration { + /// The generation a later conditional upload can fence itself on. + pub etag: Option, + /// The attempt that published it, when the object carries our metadata. + /// `None` for anything written by something that does not stamp attempts + /// (an operator upload, a pre-attempt-id archive). + pub attempt: Option, +} + +impl StoredGeneration { + /// Is what the store holds the work of `attempt`? + /// + /// This is the whole reconciliation primitive. It is deliberately a + /// three-way question collapsed to a boolean only at the point of use: an + /// object with no attempt metadata answers `false`, because "somebody else's + /// bytes" and "bytes nobody stamped" license exactly the same caution. + pub fn belongs_to(&self, attempt: &PublishAttemptId) -> bool { + self.attempt.as_deref() == Some(attempt.as_str()) + } +} + +/// The receipt of a publish the store ACKNOWLEDGED. +/// +/// Only produced on a response the client actually read, so holding one is proof +/// of publication in a way that "the upload future returned" is not. +#[derive(Clone, Debug)] +pub struct UploadReceipt { + pub attempt: PublishAttemptId, + pub etag: Option, +} + +/// How far ONE publish attempt got. +/// +/// Observable through [`PublishStageCell`] from outside the future doing the +/// work, which is the property the whole design turns on: a handler future that +/// is DROPPED never returns an outcome, so the only way to classify a cancelled +/// attempt is to read the stage it had reached when it was abandoned. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PublishStage { + /// Nothing has been attempted. + Idle, + /// No object-storage backend is configured, so there is no publication to + /// confirm and the local write IS the durable copy. Distinct from `Idle`, + /// which means an attempt was possible and had not started: a cancellation + /// under `NoBackend` loses nothing, while a cancellation under `Idle` means + /// a publish that could have happened never did. + NoBackend, + /// The archive is being built. The request has not been constructed, let + /// alone sent. A cancellation here is a DEFINITE "no publication was + /// attempted" — not an ambiguous in-flight PUT. + PreparingArchive, + /// The request is on the wire. Its fate is unknown until the store answers, + /// and a cancellation here does NOT stop the server processing it. + PutDispatched { attempt: PublishAttemptId }, + /// The store acknowledged the write. This is the ONLY stage that licenses + /// replication, a 2xx, or treating the local tree as durable. + Published { + attempt: PublishAttemptId, + etag: Option, + }, + /// The store definitively did not publish this attempt (a refused + /// precondition, or a failure that proves the bytes never committed). + Refused, + /// The attempt may or may not have committed and has not been reconciled. + /// Destructive compensation is NEVER licensed from here. + Ambiguous { attempt: PublishAttemptId }, +} + +impl PublishStage { + /// Did this attempt reach a state where the store may hold its bytes? + /// + /// True from dispatch onward. `false` is what makes deleting the attempt's + /// object or local tree safe. + pub fn may_have_published(&self) -> bool { + matches!( + self, + PublishStage::PutDispatched { .. } + | PublishStage::Published { .. } + | PublishStage::Ambiguous { .. } + ) + } + + /// The attempt whose fate is unresolved, when there is one. This is what a + /// reconciliation HEAD is compared against. + pub fn unresolved_attempt(&self) -> Option<&PublishAttemptId> { + match self { + PublishStage::PutDispatched { attempt } | PublishStage::Ambiguous { attempt } => { + Some(attempt) + } + _ => None, + } + } +} + +/// A [`PublishStage`] an in-flight upload writes and an outside observer reads. +/// +/// `std::sync::Mutex` rather than an async lock on purpose: every write is a +/// field assignment with no await inside, and the reader that matters most runs +/// inside a `Drop` impl, where an async lock cannot be awaited at all. +#[derive(Debug)] +pub struct PublishStageCell(Mutex); + +impl PublishStageCell { + pub fn new() -> Self { + Self::seeded(PublishStage::Idle) + } + + pub fn seeded(stage: PublishStage) -> Self { + Self(Mutex::new(stage)) + } + + pub fn set(&self, stage: PublishStage) { + *self.0.lock().expect("publish stage mutex poisoned") = stage; + } + + pub fn get(&self) -> PublishStage { + self.0.lock().expect("publish stage mutex poisoned").clone() + } +} + +impl Default for PublishStageCell { + fn default() -> Self { + Self::new() + } +} + +/// What the client knows about whether a request it could not complete ever +/// reached the wire. +/// +/// The classifier that produces this is the ONLY backend-aware code in the +/// publish path; a pluggable store (#79) supplies its own and every caller below +/// keeps working off [`UploadError`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DispatchKnowledge { + /// The request could not be built or sent. The store cannot hold these bytes. + NeverSent, + /// The request may have been sent, and may have committed. + MaybeSent, +} + +/// Why an upload did not publish, split by what it leaves the caller ENTITLED TO +/// DO — not by which HTTP status came back. +/// +/// The split exists because the previous shape ("precondition lost" vs "some +/// other error") made every caller treat a lost response as proof of failure. +/// Smithy timeout, dispatch and response errors all allow that the request was +/// sent and committed: a server can accept a complete conditional PUT and lose +/// or corrupt the response before the client observes success. Compensating that +/// as a definite failure deletes state that IS published. +#[derive(Debug, thiserror::Error)] +pub enum UploadError { + /// The store explicitly refused the fence (412, or 409 under create-only). + /// DEFINITE: this attempt did not publish, and a successor did. + #[error("upload precondition lost (HTTP {status})")] + PreconditionLost { status: u16 }, + /// The attempt provably never committed: it failed before the request could + /// be dispatched, or the store answered a definite client-side refusal. + /// Destructive compensation is safe here and ONLY here. + #[error("upload did not reach the store: {0:#}")] + NotPublished(#[source] anyhow::Error), + /// The request may have been dispatched and may have committed; the client + /// never learned which. The attempt id is carried so the caller can ask the + /// store rather than guess. + #[error("upload outcome is unknowable (attempt {attempt}): {source:#}")] + Ambiguous { + attempt: PublishAttemptId, + #[source] + source: anyhow::Error, + }, +} + +impl UploadError { + /// May this caller destroy state that would be needed if the write HAD + /// landed — delete the object, drop the only local clone, invalidate the + /// cache? + /// + /// Only a proven non-publication says yes. Every new variant must default to + /// `false`, which is why this is a match on the safe arms rather than a + /// negation of the unsafe one. + pub fn proves_not_published(&self) -> bool { + matches!( + self, + UploadError::PreconditionLost { .. } | UploadError::NotPublished(_) + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_definite_outcomes_license_destructive_compensation() { + assert!(UploadError::PreconditionLost { status: 412 }.proves_not_published()); + assert!( + UploadError::NotPublished(anyhow::anyhow!("no route to host")).proves_not_published() + ); + assert!( + !UploadError::Ambiguous { + attempt: PublishAttemptId::new(), + source: anyhow::anyhow!("response body truncated"), + } + .proves_not_published(), + "a response-loss failure must never license deleting state the write may own" + ); + } + + #[test] + fn a_stage_may_have_published_only_from_dispatch_onward() { + let attempt = PublishAttemptId::new(); + assert!(!PublishStage::Idle.may_have_published()); + assert!(!PublishStage::NoBackend.may_have_published()); + assert!( + !PublishStage::PreparingArchive.may_have_published(), + "compression has not constructed a request, let alone sent one" + ); + assert!(!PublishStage::Refused.may_have_published()); + assert!(PublishStage::PutDispatched { + attempt: attempt.clone() + } + .may_have_published()); + assert!(PublishStage::Ambiguous { + attempt: attempt.clone() + } + .may_have_published()); + assert!(PublishStage::Published { + attempt, + etag: None + } + .may_have_published()); + } + + #[test] + fn an_unstamped_object_never_belongs_to_an_attempt() { + let attempt = PublishAttemptId::new(); + assert!(StoredGeneration { + etag: Some("\"e\"".into()), + attempt: Some(attempt.as_str().to_string()), + } + .belongs_to(&attempt)); + assert!( + !StoredGeneration { + etag: Some("\"e\"".into()), + attempt: None, + } + .belongs_to(&attempt), + "bytes nobody stamped must not be claimed by this attempt" + ); + assert!(!StoredGeneration { + etag: Some("\"e\"".into()), + attempt: Some("someone-else".into()), + } + .belongs_to(&attempt)); + } +} diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 458207466..9878ab289 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -10,123 +10,121 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; use anyhow::{Context, Result}; -use sqlx::pool::PoolConnection; use sqlx::postgres::PgPoolOptions; -use sqlx::{PgPool, Postgres}; +use sqlx::PgPool; use tokio::sync::Mutex; use tracing::{debug, info, warn}; use super::store; -use super::tigris::TigrisClient; +use super::tigris::{ + AttemptDelete, PublishAttemptId, PublishStage, PublishStageCell, TigrisClient, UploadError, + UploadPrecondition, +}; /// Centralized repo storage: local disk cache + optional Tigris backend. #[derive(Clone)] pub struct RepoStore { repos_dir: PathBuf, tigris: Option, - /// Dedicated Postgres pool for repo write advisory locks, built by - /// `build_lock_pool` (see there for why it is separate and why it carries an - /// `after_release` hook). Never use this for ordinary queries. + /// Dedicated Postgres pool that advisory-lock connections come from, kept + /// separate from the pool serving ordinary request handlers. Each write guard + /// pins one connection here for its whole lifetime, so a push burst consumes + /// this pool rather than starving application queries. Sized by + /// `GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS`. lock_pool: PgPool, + /// Bound on any object-storage transfer that runs while the lock is HELD. + lock_held_transfer_timeout: Duration, + /// Wall-clock cap on WAITING for the lock. A field rather than a bare const so + /// the busy path can be driven in a test without a 90s wait. + lock_acquire_deadline: Duration, /// Tracks repos already confirmed to exist in Tigris — avoids redundant /// HEAD checks and background uploads for repos we've already migrated. migrated: Arc>>, - /// Test-only stall injected at the head of `acquire_write`'s Tigris phase, - /// i.e. AFTER the advisory lock is taken and BEFORE the guard exists. That - /// window is exactly where the outer `tokio::time::timeout` in - /// `api/repos.rs` can drop the future (#173). `TigrisClient` takes its - /// endpoint from process-wide AWS env vars and has no injectable seam, so - /// this flag is the smallest way to hold a real `acquire_write` open in that - /// window and cancel it there. - #[cfg(test)] - tigris_stall: Option, - /// Test-only counter of how many times a write guard from this store REACHED the - /// Tigris upload site in `release` (the point past the `success` check, where a - /// configured client would be uploaded to). It counts the decision, not a network - /// call: `TigrisClient` takes its endpoint from process-wide AWS env vars and has no - /// injectable seam, so every test runs with `tigris: None` and a counter inside the - /// `Some` arm could never move. Reaching the site is the property under test anyway: - /// an interrupted push must not publish a half-applied repo, and the disconnect path - /// must therefore never get here (#173 F2). - /// - /// Per store rather than a process global, so cases running in parallel do not see - /// each other's uploads, and an `Arc` rather than a `thread_local` because the guard - /// is released from a detached task on another worker thread. Same test-only counter - /// idiom as `ipfs_pin::note_legacy_repair_read`. - #[cfg(test)] - upload_site_reached: Arc, /// Test-only seam: armed here, copied into every `RepoWriteGuard` this store /// hands out, so a test that only holds the `AppState` (not the guard) can /// still park `release` at its pre-unlock point. See /// `RepoWriteGuard::test_pre_unlock_gate`. Never set outside tests. #[cfg(test)] pre_unlock_gate: Option>, + /// Per store rather than a process global, so parallel tests do not see each + /// other's uploads. See [`RepoStore::tigris_upload_site_reached`]. + #[cfg(test)] + upload_site_reached: Arc, } impl RepoStore { - /// Derives its own lock pool from `pool`, so callers that only have the main - /// pool (tests, `for_testing` sites in other modules) still get the - /// `after_release` semantics `acquire_write` depends on. #[cfg(test)] - pub fn for_testing(repos_dir: PathBuf, pool: PgPool) -> Self { - Self::new( + pub fn for_testing(repos_dir: PathBuf, lock_pool: PgPool) -> Self { + Self { repos_dir, - None, - build_lock_pool(&pool, 8, Duration::from_secs(5)), - ) + tigris: None, + lock_pool, + lock_held_transfer_timeout: Duration::from_secs(300), + lock_acquire_deadline: LOCK_ACQUIRE_DEADLINE, + migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), + pre_unlock_gate: None, + upload_site_reached: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + } } - /// Test-only: every guard from this store parks in `release` right before the - /// `pg_advisory_unlock` await, until `gate` is notified. Dropping the future - /// while it is parked reproduces a client disconnect inside `release`. + /// Same as [`RepoStore::for_testing`] but with Tigris enabled, so the paths + /// that only run when a backend is configured are reachable in a test. #[cfg(test)] - pub fn with_pre_unlock_gate(mut self, gate: Arc) -> Self { - self.pre_unlock_gate = Some(gate); - self + pub fn for_testing_with_tigris( + repos_dir: PathBuf, + lock_pool: PgPool, + tigris: TigrisClient, + ) -> Self { + Self::new(repos_dir, Some(tigris), lock_pool, Duration::from_secs(300)) } - /// Test-only: the dedicated advisory-lock pool this store runs its write locks - /// on. `for_testing` DERIVES it from the pool it is handed (see `build_lock_pool`), - /// so a test that wants to observe what happened to a guard's connection has to - /// look here, not at the pool it passed in. + /// Shorten the lock-acquire deadline so the busy path is reachable in a test + /// without waiting out the production default. #[cfg(test)] - pub(crate) fn lock_pool(&self) -> &PgPool { - &self.lock_pool + pub fn with_lock_acquire_deadline(mut self, deadline: Duration) -> Self { + self.lock_acquire_deadline = deadline; + self } - /// Test-only: see `tigris_stall`. + /// Test-only: every guard from this store parks in `release` right before the + /// `pg_advisory_unlock` await, until `gate` is notified. Dropping the future + /// while it is parked reproduces a client disconnect inside `release`. #[cfg(test)] - pub fn with_tigris_stall(mut self, stall: Duration) -> Self { - self.tigris_stall = Some(stall); + pub fn with_pre_unlock_gate(mut self, gate: Arc) -> Self { + self.pre_unlock_gate = Some(gate); self } /// Test-only: how many write guards from this store have reached the Tigris upload - /// site. See [`RepoStore::upload_site_reached`]. + /// site. See [`RepoWriteGuard::release`]. #[cfg(test)] pub fn tigris_upload_site_reached(&self) -> usize { self.upload_site_reached .load(std::sync::atomic::Ordering::SeqCst) } - /// `lock_pool` must come from `build_lock_pool`; a plain pool leaks advisory - /// locks on cancellation. - pub fn new(repos_dir: PathBuf, tigris: Option, lock_pool: PgPool) -> Self { + pub fn new( + repos_dir: PathBuf, + tigris: Option, + lock_pool: PgPool, + lock_held_transfer_timeout: Duration, + ) -> Self { Self { repos_dir, tigris, lock_pool, + lock_held_transfer_timeout, + lock_acquire_deadline: LOCK_ACQUIRE_DEADLINE, migrated: Arc::new(Mutex::new(HashSet::new())), #[cfg(test)] - tigris_stall: None, + pre_unlock_gate: None, #[cfg(test)] upload_site_reached: Arc::new(std::sync::atomic::AtomicUsize::new(0)), - #[cfg(test)] - pre_unlock_gate: None, } } @@ -135,10 +133,23 @@ impl RepoStore { /// spawned to lazily migrate it (on-demand migration for pre-Tigris repos). /// Returns the local path to the bare repo. pub async fn acquire(&self, owner_did: &str, repo_name: &str) -> Result { - let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; + // Validate at the sink CodeQL watches (`rust/path-injection`), not only inside + // `local_path()`, so `exists` and every later filesystem touch share one barrier. + let local_path = validated_repo_disk_path(&self.repos_dir, owner_did, repo_name)?; + let owner_slug = owner_did.replace([':', '/'], "_"); - // Fast path: repo exists locally + // Fast path: repo exists locally. if local_path.exists() { + // ...but existence is not the cache contract. A tree left by a write + // whose PUT outcome was never resolved carries this node's refs with + // no confirmed generation behind them, and serving it would publish a + // possibly-refused write to every reader for as long as the directory + // survives. Reconcile the quarantine FIRST, before the migration + // bookkeeping and before the path is handed out. + if let Some(marker) = read_quarantine(&local_path) { + self.reconcile_quarantine(&owner_slug, repo_name, &local_path, &marker) + .await?; + } // Lazy migration: if Tigris is enabled and we haven't confirmed this // repo is in Tigris yet, check and upload in the background. if let Some(ref tigris) = self.tigris { @@ -158,11 +169,34 @@ impl RepoStore { } Ok(false) => { info!(repo = %name, "migrating local repo to tigris"); - if let Err(e) = tigris.upload(&slug, &name, &path).await { - warn!(repo = %name, err = %e, "lazy migration to tigris failed"); - return; + // Create-only. This backfill was decided on a + // negative existence check that is already + // stale, so a refusal means someone else + // published this key in between and dropping + // our bytes is the correct outcome. An + // unconditional PUT here would overwrite their + // archive, which is the exact bug this fence + // exists to close. + match tigris + .upload(&slug, &name, &path, UploadPrecondition::IfAbsent) + .await + { + Ok(_) => { + info!(repo = %name, "lazy migration to tigris complete"); + } + // Logged apart from the warn arm below so a + // refusal, which is the fence working, does + // not read as a storage failure. The key is + // populated either way, so this still + // counts as migrated. + Err(UploadError::PreconditionLost { status }) => { + info!(repo = %name, status, "lazy migration dropped: another writer already published this repo"); + } + Err(e) => { + warn!(repo = %name, err = %e, "lazy migration to tigris failed"); + return; + } } - info!(repo = %name, "lazy migration to tigris complete"); } Err(e) => { warn!(repo = %name, err = %e, "tigris existence check failed"); @@ -173,7 +207,7 @@ impl RepoStore { }); } } - return Ok(local_path); + return Ok(local_path.into_path_buf()); } // Try downloading from Tigris @@ -181,7 +215,7 @@ impl RepoStore { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "cache miss — downloading from tigris"); tigris - .download(&owner_slug, repo_name, &local_path) + .download(&owner_slug, repo_name, &local_path, None) .await .context("downloading repo from tigris")?; // Mark as migrated since we just downloaded it @@ -189,44 +223,149 @@ impl RepoStore { .lock() .await .insert(format!("{owner_slug}/{repo_name}")); - return Ok(local_path); + return Ok(local_path.into_path_buf()); } } // Not found anywhere — return path anyway; caller will get a meaningful // error from git when the path doesn't exist. - Ok(local_path) + Ok(local_path.into_path_buf()) + } + + /// Decide whether a quarantined live tree may be served. + /// + /// The ONLY thing that lifts a quarantine is the store confirming it holds + /// the attempt that left it — which is decidable, because the attempt id + /// travelled with the bytes. Everything else refuses with a retryable + /// `RepoUnavailable`. + /// + /// Deleting the tree instead is NOT a safe alternative and is deliberately + /// not done here: the unresolved PUT may have landed, in which case this + /// directory can be the only local copy of it. Refusing costs a 503 that a + /// later write (whose under-lock refresh re-downloads the confirmed archive + /// and clears the marker) or a successful reconciliation resolves; deleting + /// costs the data. + async fn reconcile_quarantine( + &self, + owner_slug: &str, + repo_name: &str, + local_path: &ValidatedRepoDiskPath, + marker: &QuarantineMarker, + ) -> Result<()> { + let refuse = || { + Err(anyhow::Error::new(RepoUnavailable).context(format!( + "local tree for {owner_slug}/{repo_name} is quarantined: its publish outcome \ + is unresolved and could not be reconciled against object storage" + ))) + }; + let (Some(tigris), Some(attempt)) = ( + self.tigris.as_ref(), + marker.attempt.as_deref().map(PublishAttemptId::from_owned), + ) else { + // No backend to ask, or no attempt to ask about. Either way nothing + // can confirm this tree, and a read that cannot be confirmed must + // not be served as an ordinary success. + warn!( + repo = %repo_name, + "refusing a quarantined read: no attempt identity to reconcile against" + ); + return refuse(); + }; + match tigris.attempt_landed(owner_slug, repo_name, &attempt).await { + Ok(true) => { + info!( + repo = %repo_name, + attempt = %attempt, + "quarantine lifted: object storage holds this attempt's archive" + ); + clear_quarantine(local_path, repo_name); + Ok(()) + } + Ok(false) => { + // The store does not hold this attempt. That is NOT proof it + // never will — an abandoned PUT can still be in flight — so the + // tree stays put and the read is refused rather than served or + // deleted. + warn!( + repo = %repo_name, + attempt = %attempt, + "refusing a quarantined read: object storage holds a different generation \ + than the unresolved write on local disk" + ); + refuse() + } + Err(e) => { + warn!( + repo = %repo_name, + err = %e, + "refusing a quarantined read: could not reach object storage to reconcile" + ); + refuse() + } + } } - /// Ensure a repo is available on local disk with the **latest** Tigris state. - /// Use this for operations that precede a write (e.g. `info/refs` for - /// `git-receive-pack`) so the client sees the same refs that `acquire_write()` - /// will operate on. - pub async fn acquire_fresh(&self, owner_did: &str, repo_name: &str) -> Result { + /// Non-mutating snapshot of a repo's **latest** Tigris state, for reads that + /// must see fresh data but must NOT write into the live repo path. + /// + /// The fresh-acquire form this replaced (`acquire_fresh`) downloaded and + /// PUBLISHED into the live directory (removing the existing dir and renaming + /// the extract into place); this unpacks into a throwaway temp dir and + /// returns it. The live path is never touched, so an unlocked caller cannot + /// delete or swap the directory under a concurrent guarded write. + /// + /// The returned snapshot owns its temp dir and removes it on drop; when + /// there is no Tigris backend (or no archive), the snapshot borrows the live + /// local path and owns nothing. A HEAD failure refuses rather than guessing, + /// matching the under-lock refresh path: a transient storage blip must be a + /// retryable refusal (`RepoUnavailable`), not a 500 or a silently stale read. + pub async fn read_snapshot(&self, owner_did: &str, repo_name: &str) -> Result { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; if let Some(ref tigris) = self.tigris { - if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { - debug!(repo = %repo_name, "acquire_fresh: downloading latest from tigris"); - if let Err(e) = tigris.download(&owner_slug, repo_name, &local_path).await { - // The Tigris archive is present (HEAD ok) but unreadable — a - // corrupt/partial upload, or a transient GET failure. If we have a - // valid local copy, proceed with it rather than blocking the write; - // the post-write upload re-syncs (self-heals) Tigris. Only hard-fail - // when there is no local copy to fall back to. - if local_path.exists() { - warn!(repo = %repo_name, err = %e, - "acquire_fresh: tigris download failed — falling back to local copy"); - return Ok(local_path); - } - return Err(e).context("downloading repo from tigris (fresh)"); + match tigris.exists(&owner_slug, repo_name).await { + Ok(true) => { + // Snapshot form: unpack into a temp dir, never the live path. + // + // Cancellation cleanup: the extraction runs in a + // `spawn_blocking` that cannot be aborted, so a dropped + // future (client disconnect, a bounded-transfer timeout) + // still leaves the temp dir on disk. The cleanup has to live + // in the ASYNC layer, armed for the whole download await and + // disarmed only when `RepoSnapshot` takes ownership. + let snapshot = tigris + .download_to(&owner_slug, repo_name, &local_path, false, None) + .await + .map_err(|e| { + anyhow::Error::new(RepoUnavailable).context(format!( + "tigris snapshot download failed during read_snapshot for {owner_slug}/{repo_name}: {e:#}" + )) + })?; + return match snapshot { + super::tigris::DownloadExtract::Snapshot(dir) => { + Ok(dir.into_repo_snapshot()) + } + super::tigris::DownloadExtract::Published(_) => { + unreachable!("snapshot downloads never publish into the live path") + } + }; + } + Ok(false) => {} + Err(e) => { + warn!(repo = %repo_name, err = %e, + "read_snapshot: tigris HEAD failed — refusing rather than guessing the archive is absent"); + return Err(anyhow::Error::new(RepoUnavailable).context(format!( + "tigris HEAD failed during read_snapshot for {owner_slug}/{repo_name}" + ))); } - return Ok(local_path); } } - // Tigris disabled or repo not in Tigris — fall back to local - Ok(local_path) + // Tigris disabled or repo not in Tigris — fall back to local. + Ok(RepoSnapshot { + path: local_path.into_path_buf(), + owned: false, + }) } /// Take a write lock (Postgres advisory lock), ensure repo is local, return guard. @@ -262,100 +401,303 @@ impl RepoStore { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); - // Acquire the Postgres advisory lock with retry, using pg_try_advisory_lock so a - // stale lock from a crashed connection can't block us indefinitely. - // - // The connection is checked out INSIDE the loop and RETURNED before each sleep. - // Only the connection that actually took the lock is retained. Two constraints - // pull in opposite directions here, and this is what satisfies both: + // Take the lock on a connection this guard will own for its whole + // lifetime, so the release runs on the same session. `pg_try_advisory_lock` + // with retry rather than a blocking acquire, so a stale lock from a crashed + // connection cannot wedge us indefinitely. // - // * Session ownership. A session-level advisory lock belongs to the CONNECTION - // that took it, so the lock and its `pg_advisory_unlock` must run on the same - // one. Running them through the pool (`fetch_one(&self.pool)`) lets them land - // on different connections: the unlock silently returns false and the lock - // leaks, while a competing acquire that happens to draw the holding - // connection re-enters the lock and two pushes to one repo run concurrently. - // Hence: keep the connection that WON. - // * Occupancy. Holding a connection across the ~60 one-second sleeps would let - // one spinning acquire park a lock-pool connection for a minute. That is not - // just a push-path concern: `api/issues.rs` and `api/pulls.rs` reach - // acquire_write holding no concurrency permit at all, so a caller could park - // the whole pool and starve authenticated pushes on every repo (#173 F1). - // Hence: return the connection when we LOSE, before sleeping. + // Each attempt checks a connection out and, on failure, returns it BEFORE + // sleeping: a writer spinning on a contended repo must not pin a lock-pool + // slot through its backoff, or a handful of spinners would starve the pool + // for everyone else. // - // Returning a losing connection is safe with respect to the cancellation design: - // `after_release` runs `pg_advisory_unlock_all()`, a no-op on a connection that - // took nothing, so it cannot disturb a lock held by any other connection - // (proven by `returning_an_unlocked_connection_does_not_clear_another_connections_lock`). - // - // Cancellation safety is unchanged: the future can only be dropped while a - // connection is checked out, and dropping it runs the same `after_release` hook, - // which clears whatever lock it had just taken (#173 U1). + // Pool exhaustion is a DIFFERENT condition from "someone else holds the + // lock" and is not retried here. Retrying it would burn all 60 attempts + // against a pool that is full for reasons unrelated to this repo, and would + // report a capacity problem as lock contention. It surfaces immediately with + // its own message instead. + // Cap the WALL CLOCK of the WAIT, not just the attempt count. 60 attempts + // each pay a pool acquire (up to db_acquire_timeout_secs) plus a 1s sleep, + // so an attempt-only bound reaches ~360s. This bounds the wait only; the + // under-lock refresh below carries its own separate bound, so do not read + // this as a total for `acquire_write` (see LOCK_ACQUIRE_DEADLINE). + let deadline_budget = self.lock_acquire_deadline; + let deadline = std::time::Instant::now() + deadline_budget; let mut lock_conn = None; for attempt in 0..60 { - let mut conn = self.lock_pool.acquire().await.map_err(|e| { - anyhow::Error::new(LockPoolBusy) - .context(format!("checking out a lock-pool connection: {e}")) - })?; - let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") - .bind(lock_key) - .fetch_one(&mut *conn) - .await - .context("trying advisory lock")?; - if row.0 { - lock_conn = Some(conn); + // The advertised cap is WALL CLOCK, so the remaining budget must bound + // every await in the loop, not just the sleep between attempts. A pool + // checkout or a slow advisory query that starts just before the deadline + // and lands after it would otherwise hold the write task past the budget + // it was promised, which is exactly what the deadline exists to prevent. + let left = match deadline.checked_duration_since(std::time::Instant::now()) { + Some(left) if !left.is_zero() => left, + _ => break, + }; + // Bound the pool checkout by the remaining budget. A checkout that + // would outlive the deadline is not worth starting: it either waits out + // the full DB acquire timeout and fails anyway, or lands a connection + // with no budget left to use it. + let conn = match tokio::time::timeout(left, self.lock_pool.acquire()).await { + Ok(Ok(c)) => c, + Ok(Err(e)) => { + // Saturation is surfaced HERE, in the request path, and + // deliberately not through /ready. Failing readiness on a full + // pool would pull this node out of routing, taking its reads + // with it and pushing its write load onto peers carrying the + // same load — the documented downward spiral. So the signals + // are: a retryable 503 to the caller (via the sqlx downcast on + // this error) and this log line for the operator. + // + // Logged at warn with the pool's own counters so an incident can + // tell "the pool is full" from "the database is gone" without + // reproducing it. Once per failed acquire, and a failed acquire + // already costs a multi-second timeout, so this cannot itself + // become a log flood. + warn!( + repo = %repo_name, + owner = %owner_slug, + pool_size = self.lock_pool.size(), + pool_idle = self.lock_pool.num_idle(), + err = %e, + "advisory-lock pool acquire failed — writes are being shed; \ + raise GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS or investigate long-held write locks" + ); + return Err(anyhow::Error::new(LockPoolBusy)) + .context(format!("advisory-lock pool exhausted or unreachable: {e}")); + } + Err(_) => { + // The pool checkout itself outlived the remaining budget. Same + // refusal as running out of attempts: the wall-clock cap is what + // is advertised, so a checkout that blows past it is contention + // the caller was promised would not happen. + warn!( + repo = %repo_name, + owner = %owner_slug, + waited_secs = deadline_budget.as_secs(), + "advisory-lock pool checkout exceeded the acquire deadline — shedding the write as busy" + ); + return Err(anyhow::Error::new(RepoBusy).context(format!( + "advisory-lock pool checkout exceeded the {}s deadline for {owner_slug}/{repo_name}", + deadline_budget.as_secs() + ))); + } + }; + // Bound the advisory query by the remaining budget too: a query that + // starts with budget left but answers after the deadline must not be + // accepted, or the cap is only as good as the fast path. + let left = match deadline.checked_duration_since(std::time::Instant::now()) { + Some(left) if !left.is_zero() => left, + _ => break, + }; + let mut probe = LockProbe::new(conn); + let acquired = match tokio::time::timeout(left, probe.try_lock(lock_key)).await { + Ok(Ok(acquired)) => acquired, + Ok(Err(e)) => return Err(e).context("trying advisory lock"), + Err(_) => { + // The query outlived the remaining budget. The probe's Drop + // closes its session, which cannot hold the lock it never + // confirmed taking, so this is a plain shed. + warn!( + repo = %repo_name, + owner = %owner_slug, + waited_secs = deadline_budget.as_secs(), + "advisory-lock query exceeded the acquire deadline — shedding the write as busy" + ); + return Err(anyhow::Error::new(RepoBusy).context(format!( + "advisory-lock query exceeded the {}s deadline for {owner_slug}/{repo_name}", + deadline_budget.as_secs() + ))); + } + }; + if acquired { + lock_conn = probe.take_conn(); break; } - // Lost the race: give the connection back so a spinning acquire occupies - // nothing while it waits. - drop(conn); + // Not acquired, and nothing is locked, so hand the connection back + // before the backoff rather than holding a slot while idle. + drop(probe); + // Clamp the backoff to what is left of the budget: sleeping a full + // second past the deadline would turn a short deadline into a longer + // wait than the caller was promised. if attempt < 59 { - tokio::time::sleep(std::time::Duration::from_secs(1)).await; + let remaining = match deadline.checked_duration_since(std::time::Instant::now()) { + Some(remaining) if !remaining.is_zero() => remaining, + _ => break, + }; + tokio::time::sleep(remaining.min(std::time::Duration::from_secs(1))).await; } } let Some(lock_conn) = lock_conn else { - anyhow::bail!("could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}"); + // Contention is transient, so this must NOT land as a 500. The detail + // (which repo, which key, how long) goes to the log; the client gets a + // retryable 503 with a fixed body via the `RepoBusy` downcast. + warn!( + repo = %repo_name, + owner = %owner_slug, + lock_key, + waited_secs = deadline_budget.as_secs(), + "advisory lock not acquired within the deadline — shedding the write as busy" + ); + return Err(anyhow::Error::new(RepoBusy).context(format!( + "could not acquire advisory lock within {}s for {owner_slug}/{repo_name}", + deadline_budget.as_secs() + ))); + }; + // From here the lock is HELD. Any early return must not simply drop the + // connection back into the pool, so it is handed to the guard immediately + // below and every exit after this point goes through the guard. + let refresh_swap_authority = Arc::new(AtomicBool::new(true)); + let mut guard = RepoWriteGuard { + owner_slug: owner_slug.clone(), + repo_name: repo_name.to_string(), + local_path: local_path.clone(), + lock_key, + conn: Some(lock_conn), + tigris: self.tigris.clone(), + lock_held_transfer_timeout: self.lock_held_transfer_timeout, + // Overwritten by the refresh below with the generation actually + // observed under the lock. Only reachable unset when no backend is + // configured, in which case `release` publishes nothing at all. + publish_fence: UploadPrecondition::Unconditional, + refresh_swap_authority: Some(refresh_swap_authority.clone()), + // Seeded from whether a backend exists at all, so a cancelled + // release can tell "there was nothing to publish" from "a publish + // was possible and never started". + publish_stage: Arc::new(PublishStageCell::seeded(match self.tigris { + Some(_) => PublishStage::Idle, + None => PublishStage::NoBackend, + })), + #[cfg(test)] + test_pre_unlock_gate: self.pre_unlock_gate.clone(), + #[cfg(test)] + upload_site_reached: Arc::clone(&self.upload_site_reached), }; - - #[cfg(test)] - if let Some(stall) = self.tigris_stall { - tokio::time::sleep(stall).await; - } // Always download the latest from Tigris before writing. Local disk may be - // stale if another machine pushed since our last access. The lock connection - // is already held, so a cancellation here returns it through `after_release`, - // which clears the lock. + // stale if another machine pushed since our last access. The guard already + // owns the lock + its connection, so a cancellation here drops through Drop. if let Some(ref tigris) = self.tigris { - if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { - debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); - if let Err(e) = tigris.download(&owner_slug, repo_name, &local_path).await { - // Same self-healing fallback as acquire_fresh: a corrupt/unreadable - // Tigris archive must not block a write when a valid local copy - // exists — release(success) will re-upload a good archive. - if local_path.exists() { - warn!(repo = %repo_name, err = %e, - "write acquire: tigris download failed — falling back to local copy"); - } else { - return Err(e).context("downloading repo from tigris for write"); + // ONE budget for the whole refresh, covering the HEAD and the download + // together. Both run with the lock held and a lock-pool slot pinned, so + // bounding only the download would leave a mute endpoint able to hold + // both indefinitely on the HEAD, and bounding them separately would make + // worst-case occupancy two budgets instead of one. + let refreshed = bounded_transfer( + "acquire-refresh", + repo_name, + self.lock_held_transfer_timeout, + async { + // The HEAD and the download fail for epistemically DIFFERENT + // reasons, so they are kept apart rather than collapsed into one + // `Result`. A failed HEAD leaves us not knowing whether an archive + // exists at all, which is the same state a timeout leaves us in; + // a failed download after a successful HEAD tells us an archive is + // there and unreadable. Only the second licenses the local + // fallback. Collapsing them (the `unwrap_or(false)` this replaced + // read a HEAD error as "no archive") skipped the refresh silently + // and then re-uploaded over a possibly-newer archive. + // + // `head_etag` rather than `exists`: the same request answers + // both questions, and the ETag it carries is the generation + // this write is based on. Carrying it to the release-side + // publish is what lets the store refuse a stale PUT, which + // is the only place that fence can hold: dropping an + // in-flight upload's future does not stop the request the + // server is already processing. + match tigris.head_etag(&owner_slug, repo_name).await { + Ok(Some(etag)) => { + debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); + let fence = UploadPrecondition::IfMatch(etag); + match tigris + .download( + &owner_slug, + repo_name, + &local_path, + Some(refresh_swap_authority.clone()), + ) + .await + { + Ok(()) => Ok(fence), + Err(err) => Err(RefreshFailure::Download { err }), + } + } + Ok(None) => Ok(UploadPrecondition::IfAbsent), + Err(e) => Err(RefreshFailure::Unknown(e)), } + }, + ) + .await; + + match refreshed { + Some(Ok(fence)) => { + // The tree at the live path is now the generation this HEAD + // observed (downloaded, or confirmed absent), so any + // quarantine an earlier unresolved write left on this path is + // answered by the refresh itself. + clear_quarantine(&local_path, repo_name); + guard.publish_fence = fence; + } + Some(Err(RefreshFailure::Download { err, .. })) => { + // HEAD established a stored generation but the GET failed, so we + // do not know whether the local tree matches it. Proceeding on a + // cached copy and publishing fenced on the observed ETag can + // overwrite a newer archive with stale-local + this write. + warn!(repo = %repo_name, err = %err, + "write acquire: tigris download failed under the lock — refusing rather than writing against an unverified local tree"); + return Err(anyhow::Error::new(RepoUnavailable).context(format!( + "tigris download failed during acquire_write for {owner_slug}/{repo_name}: {err:#}" + ))); + } + Some(Err(RefreshFailure::Unknown(e))) => { + // The HEAD itself failed, so we do not know whether a newer + // archive exists. Refuse for the same reason the timeout arm + // below refuses: proceeding would write against a possibly-stale + // tree and then re-upload over another node's newer archive. A + // transient object-storage blip costs a retryable refusal here, + // which is the cheaper failure than silent overwrite. + warn!(repo = %repo_name, err = %e, + "write acquire: tigris HEAD failed — refusing the write rather than \ + guessing the archive is absent"); + return Err(anyhow::Error::new(RepoUnavailable).context(format!( + "tigris HEAD failed before a write for {owner_slug}/{repo_name}" + ))); + } + None => { + // TIMED OUT, which is NOT the same as failed, and must not reach + // the fallback above. Two reasons. We do not know whether we have + // the latest tree, so writing against the local copy and then + // re-uploading can silently overwrite another node's newer + // archive. Worse, the abandoned download's extraction runs in an + // uncancellable spawn_blocking that ends in remove_dir_all + + // rename over local_path, so proceeding would run git against a + // directory that a background task is about to delete. + // + // Refuse the acquire. Returning here drops the guard, whose Drop + // frees the lock and its pool slot. + revoke_swap_authority(&refresh_swap_authority); + // + // `error!`, not the sibling `warn!` above, and that is deliberate. + // The handler layer demotes every `RepoUnavailable` to warn because + // the common cause is an ordinary storage blip. A stall that ran out + // the whole bound is not that: it pinned a lock-pool slot for the + // full duration, and this raise-site `error!` is what keeps it + // paging. Do NOT "fix" it to match the arm above. + tracing::error!( + repo = %repo_name, + owner = %owner_slug, + bound_secs = self.lock_held_transfer_timeout.as_secs(), + "under-lock tigris refresh exceeded the transfer bound, refusing the write" + ); + return Err(anyhow::Error::new(RepoUnavailable).context(format!( + "tigris refresh exceeded the {}s under-lock bound for {owner_slug}/{repo_name}", + self.lock_held_transfer_timeout.as_secs() + ))); } } } - Ok(RepoWriteGuard { - owner_slug, - repo_name: repo_name.to_string(), - local_path, - lock_key, - lock_conn: Some(lock_conn), - released: false, - tigris: self.tigris.clone(), - #[cfg(test)] - upload_site_reached: Arc::clone(&self.upload_site_reached), - #[cfg(test)] - test_pre_unlock_gate: self.pre_unlock_gate.clone(), - }) + Ok(guard) } /// Initialize a new bare repo on local disk and upload to Tigris. @@ -371,30 +713,79 @@ impl RepoStore { let repo_name = repo_name.to_string(); let path = local_path.clone(); tokio::spawn(async move { - if let Err(e) = tigris.upload(&owner_slug, &repo_name, &path).await { - warn!(repo = %repo_name, err = %e, "failed to upload new repo to tigris"); + // Create-only, and load-bearing: this uploads a freshly + // initialized EMPTY repo, so a user who pushes immediately + // after creating one would have their archive replaced by this + // background PUT if it were unconditional. A refusal means + // someone else already published this key and dropping our + // bytes is the correct outcome. + match tigris + .upload(&owner_slug, &repo_name, &path, UploadPrecondition::IfAbsent) + .await + { + Ok(_) => {} + // Distinct from the warn arm: the fence refusing is the + // design working, not a storage failure. + Err(UploadError::PreconditionLost { status }) => { + info!(repo = %repo_name, status, "dropped the empty-repo upload: another writer already published this repo"); + } + Err(e) => { + warn!(repo = %repo_name, err = %e, "failed to upload new repo to tigris"); + } } }); } - Ok(local_path) + Ok(local_path.into_path_buf()) } /// Upload a repo to Tigris after a write operation (push, merge, fork, etc.). /// Call this after any operation that modifies the git repo on disk. - pub async fn release_after_write(&self, owner_did: &str, repo_name: &str) { + /// + /// Returns `Err(UploadError::PreconditionLost)` when the create-only upload was + /// refused because the key already exists. That is a DISTINCT outcome from a + /// plain upload failure. The sole caller is fork creation, which uses the + /// former to refuse the fork rather than create a DB record shadowed by an + /// orphan other nodes would fetch. Plain upload failures are propagated so fork + /// creation does not insert a DB row when the archive never landed. + /// + /// `attempt` is the identity the bytes are stamped with, and fork creation + /// passes the `record.id` it is about to insert. That is what makes the + /// object, the disk clone and the database row all name ONE attempt, so a + /// later cleanup can be conditional on still owning the thing it is about to + /// destroy instead of trusting the logical owner/name. + pub async fn release_after_write( + &self, + owner_did: &str, + repo_name: &str, + attempt: &PublishAttemptId, + ) -> Result<(), UploadError> { if let Some(ref tigris) = self.tigris { let (owner_slug, local_path) = match self.local_path(owner_did, repo_name) { Ok(p) => p, Err(e) => { warn!(repo = %repo_name, err = %e, "rejected unsafe path in release_after_write"); - return; + // A path this node refused to build is a path nothing was + // ever sent to. + return Err(UploadError::NotPublished(e)); } }; - if let Err(e) = tigris.upload(&owner_slug, repo_name, &local_path).await { - warn!(repo = %repo_name, err = %e, "failed to upload repo to tigris after write"); - } + // Create-only. The sole caller is fork creation, which rejects a + // name conflict in the database before it clones anything, so the + // key is expected absent here. A refusal therefore means someone + // else already published this key. + tigris + .upload_tracked( + &owner_slug, + repo_name, + &local_path, + UploadPrecondition::IfAbsent, + attempt.clone(), + None, + ) + .await?; } + Ok(()) } /// Compute the local disk path and owner slug for a repo. @@ -407,32 +798,266 @@ impl RepoStore { /// (or the prefix/root from `repos_dir`); any `ParentDir`/`CurDir` /// segment is rejected. This is the CodeQL-recognised barrier /// pattern for `rust/path-injection`. - fn local_path(&self, owner_did: &str, repo_name: &str) -> Result<(String, PathBuf)> { + fn local_path( + &self, + owner_did: &str, + repo_name: &str, + ) -> Result<(String, ValidatedRepoDiskPath)> { let owner_slug = owner_did.replace([':', '/'], "_"); let local_path = validated_repo_disk_path(&self.repos_dir, owner_did, repo_name)?; Ok((owner_slug, local_path)) } + + /// Did `attempt`'s fork archive land? Used to recover a create-only publish + /// whose response was lost, instead of compensating it as a definite failure + /// and fencing the fork name behind its own orphan. + /// + /// `Ok(false)` with no backend configured, where there is nothing to have + /// landed in. + pub async fn fork_attempt_landed( + &self, + owner_did: &str, + repo_name: &str, + attempt: &PublishAttemptId, + ) -> Result { + let Some(ref tigris) = self.tigris else { + return Ok(false); + }; + let (owner_slug, _) = self.local_path(owner_did, repo_name)?; + tigris.attempt_landed(&owner_slug, repo_name, attempt).await + } + + /// Best-effort cleanup when fork creation published an archive but failed to persist + /// the database row. Removes the object-store key and the local mirror clone THIS + /// ATTEMPT OWNS, so a retry is not blocked by its own orphan. + /// + /// Every destructive step is conditional on the resource still belonging to + /// `attempt`. The unconditional version this replaces authorized itself with the + /// logical owner/name, which identifies a namespace and not the attempt that owns a + /// row, an object generation or a filesystem tree: recovery for a failed fork could + /// observe `get_repo == None`, be overtaken by a successor that committed and + /// returned 201, and then delete that successor's archive and directory. A second + /// name lookup immediately before the delete only moves that window; the attempt + /// guard removes it. + pub async fn compensate_fork_archive( + &self, + owner_did: &str, + repo_name: &str, + disk_path: &Path, + attempt: &PublishAttemptId, + ) { + if let Some(ref tigris) = self.tigris { + if let Ok((owner_slug, _)) = self.local_path(owner_did, repo_name) { + match tigris + .delete_if_attempt_matches(&owner_slug, repo_name, attempt) + .await + { + Ok(AttemptDelete::Deleted) | Ok(AttemptDelete::Absent) => {} + Ok(AttemptDelete::NotOurs) => { + info!( + repo = %repo_name, + attempt = %attempt, + "fork compensation left the stored archive alone: it belongs to \ + another attempt" + ); + } + Err(e) => { + warn!( + repo = %repo_name, + err = %e, + "failed to delete fork archive during create_repo compensation — scheduling retry" + ); + let tigris = tigris.clone(); + let slug = owner_slug.clone(); + let name = repo_name.to_string(); + let attempt = attempt.clone(); + tokio::spawn(async move { + retry_fork_archive_delete(&tigris, &slug, &name, &attempt).await; + }); + } + } + } + } + remove_fork_clone_if_ours(disk_path, attempt, "create_repo compensation"); + } +} + +/// Sidecar naming the attempt that owns a fork's on-disk clone, beside the +/// directory for the same reason the quarantine marker is: anything inside the +/// bare repo would be tarred into the archive and shipped to every node. +fn fork_attempt_path(disk_path: &Path) -> Option { + let parent = disk_path.parent()?; + let file_name = disk_path.file_name()?.to_string_lossy().to_string(); + Some(parent.join(format!(".{file_name}.fork-attempt"))) +} + +/// Stamp a freshly cloned fork mirror with the attempt that created it. Written +/// before the archive upload, so every later cleanup can ask "is this still +/// mine?" of the directory as well as of the object. +pub(crate) fn claim_fork_disk_path(disk_path: &Path, attempt: &PublishAttemptId) { + let Some(path) = fork_attempt_path(disk_path) else { + return; + }; + if let Err(e) = std::fs::write(&path, attempt.as_str()) { + warn!( + path = %disk_path.display(), + err = %e, + "failed to stamp the fork clone with its attempt id — cleanup will refuse to \ + remove it rather than risk removing a successor's clone" + ); + } +} + +/// Drop the attempt stamp once the fork's row has committed and no cleanup may +/// ever remove this directory again. +/// +/// Leaving the stamp would be harmless but misleading; removing it also means a +/// LATER attempt's cleanup finds no owner and therefore refuses to delete, which +/// is the safe direction. +pub(crate) fn release_fork_disk_claim(disk_path: &Path) { + if let Some(path) = fork_attempt_path(disk_path) { + let _ = std::fs::remove_file(path); + } +} + +/// Does the clone at `disk_path` still belong to `attempt`? +/// +/// A missing stamp answers NO. That is the fail-safe direction: an unstamped +/// directory is one this attempt cannot prove it owns, and refusing to delete +/// leaves an orphan for an operator, while deleting wrongly destroys a +/// successor's repository after that successor has already returned success. +fn fork_clone_is_ours(disk_path: &Path, attempt: &PublishAttemptId) -> bool { + fork_attempt_path(disk_path) + .and_then(|p| std::fs::read_to_string(p).ok()) + .is_some_and(|owner| owner.trim() == attempt.as_str()) +} + +/// Remove a fork's clone only while it is still this attempt's. +pub(crate) fn remove_fork_clone_if_ours( + disk_path: &Path, + attempt: &PublishAttemptId, + reason: &str, +) { + if !disk_path.exists() { + let _ = fork_attempt_path(disk_path).map(std::fs::remove_file); + return; + } + if !fork_clone_is_ours(disk_path, attempt) { + info!( + path = %disk_path.display(), + attempt = %attempt, + reason, + "left the fork clone alone: it no longer belongs to this attempt" + ); + return; + } + if let Err(e) = std::fs::remove_dir_all(disk_path) { + warn!( + path = %disk_path.display(), + err = %e, + reason, + "failed to remove fork clone" + ); + return; + } + let _ = fork_attempt_path(disk_path).map(std::fs::remove_file); +} + +async fn retry_fork_archive_delete( + tigris: &TigrisClient, + owner_slug: &str, + repo_name: &str, + attempt: &PublishAttemptId, +) { + const MAX_ATTEMPTS: u32 = 6; + for n in 0..MAX_ATTEMPTS { + // Re-evaluated on EVERY retry, not resolved once before the loop. The + // whole point of retrying is that time passes, and the ownership gap the + // unconditional version left open widened with each attempt: a successor + // that commits between retry 2 and retry 3 would still have had its + // archive deleted by retry 3. + match tigris + .delete_if_attempt_matches(owner_slug, repo_name, attempt) + .await + { + Ok(AttemptDelete::Deleted) => { + info!( + repo = %repo_name, + attempt = n, + "fork archive compensation delete succeeded on retry" + ); + return; + } + Ok(AttemptDelete::Absent) => return, + Ok(AttemptDelete::NotOurs) => { + info!( + repo = %repo_name, + "fork archive compensation stopped retrying: the stored archive now \ + belongs to another attempt" + ); + return; + } + Err(e) if n + 1 < MAX_ATTEMPTS => { + warn!( + repo = %repo_name, + attempt = n, + err = %e, + "fork archive compensation delete failed — retrying" + ); + tokio::time::sleep(Duration::from_secs(1u64 << n.min(4))).await; + } + Err(e) => { + warn!( + repo = %repo_name, + err = %e, + "fork archive compensation delete failed after all retries — operator cleanup required" + ); + } + } + } +} + +/// A repository disk path that has passed the three-layer validation barrier. +/// +/// Only [`validated_repo_disk_path`] may construct this type, so sinks such as +/// `remove_dir_all` / `rename` can take it and static analysers can treat it as +/// sanitised input (CodeQL `rust/path-injection`). +#[derive(Debug, Clone)] +pub(crate) struct ValidatedRepoDiskPath(PathBuf); + +impl ValidatedRepoDiskPath { + pub(crate) fn as_path(&self) -> &Path { + &self.0 + } + + pub(crate) fn into_path_buf(self) -> PathBuf { + self.0 + } +} + +impl std::ops::Deref for ValidatedRepoDiskPath { + type Target = Path; + + fn deref(&self) -> &Path { + &self.0 + } +} + +impl AsRef for ValidatedRepoDiskPath { + fn as_ref(&self) -> &Path { + &self.0 + } } /// The three-layer validated form of `store::repo_disk_path`, with NO Tigris fetch and /// no `RepoStore` (#173 round 11, F3). Extracted from `RepoStore::local_path` so a -/// second caller that must not pull a cold repo, the U4 legacy provider-CID sweep, gets -/// the same barrier instead of the raw join. `local_path` is now a thin wrapper over -/// this, so the two cannot drift. -/// -/// Three-layer defence against path traversal: -/// 1. Strict allowlist on `owner_did` and `repo_name` (no `..`, slashes, -/// null bytes, leading dots; length-bounded). -/// 2. The joined path must remain rooted at `repos_dir`. -/// 3. Every component of the joined path must be `Component::Normal` -/// (or the prefix/root from `repos_dir`); any `ParentDir`/`CurDir` -/// segment is rejected. This is the CodeQL-recognised barrier -/// pattern for `rust/path-injection`. +/// second caller that must not pull a cold repo gets the same barrier instead of the +/// raw join. pub(crate) fn validated_repo_disk_path( repos_dir: &Path, owner_did: &str, repo_name: &str, -) -> Result { +) -> Result { validate_path_components(owner_did, repo_name)?; let owner_slug = owner_did.replace([':', '/'], "_"); @@ -445,10 +1070,10 @@ pub(crate) fn validated_repo_disk_path( ); } - // Explicit component walk — sanitisation barrier that static analysers - // (CodeQL `rust/path-injection`) recognise. The path must be composed - // entirely of Normal segments after the root prefix; any ParentDir or - // CurDir component is a traversal attempt. + // Explicit component walk, sanitisation barrier that static analysers + // (CodeQL `rust/path-injection`) recognise. The path must be composed entirely + // of Normal segments after the root prefix; any ParentDir or CurDir component + // is a traversal attempt. for component in local_path.components() { use std::path::Component; match component { @@ -462,7 +1087,168 @@ pub(crate) fn validated_repo_disk_path( } } - Ok(local_path) + Ok(ValidatedRepoDiskPath(local_path)) +} + +/// Revoke an in-flight publish swap. Any worker that has not yet claimed the +/// commit token will refuse before touching the live tree. +pub(crate) fn revoke_swap_authority(authority: &AtomicBool) { + authority.store(false, Ordering::Release); +} + +/// Claim exclusive rights to perform the destructive publish swap. Revocation and +/// commit are mutually exclusive: only one caller can win the `true -> false` +/// transition, and that caller alone may remove/rename the live directory. +pub(crate) fn try_claim_swap_commit(authority: &AtomicBool) -> bool { + authority + .compare_exchange(true, false, Ordering::AcqRel, Ordering::Acquire) + .is_ok() +} + +/// Swap a finished extraction into a validated live repo path. The caller must pass +/// the path returned from [`validated_repo_disk_path`]; this is the CodeQL barrier +/// for `rust/path-injection` on the remove/rename sink. +pub(crate) fn swap_extracted_into_validated_repo( + validated_path: &ValidatedRepoDiskPath, + tmp_dir: &Path, + swap_authority: Option<&Arc>, +) -> Result<()> { + let live = validated_path.as_path(); + let lock = super::tigris::publish_lock(live); + let _publish = lock.lock().expect("publish lock poisoned"); + if let Some(authority) = swap_authority { + if !try_claim_swap_commit(authority) { + let _ = std::fs::remove_dir_all(tmp_dir); + anyhow::bail!("publish swap revoked after lock ownership ended"); + } + } + if live.exists() { + std::fs::remove_dir_all(live).context("removing stale repo dir")?; + } + std::fs::rename(tmp_dir, live).context("swapping extracted repo into place")?; + Ok(()) +} + +/// How long the under-lock reconciliation HEAD gets. Separate from (and much +/// smaller than) the publish bound it follows: it runs after a transfer that has +/// already used its whole budget, with the advisory lock and a lock-pool slot +/// still pinned, so it must be an addendum rather than a second budget. +const RECONCILE_BOUND: Duration = Duration::from_secs(5); + +/// The sidecar that marks a live repo tree as QUARANTINED: present on disk, but +/// with no confirmed object-store generation behind it. +/// +/// A sibling dotfile rather than something inside the repo directory, for two +/// reasons. Anything under the bare repo would be tarred into the next archive +/// and shipped to every node that downloads it, and the marker has to survive +/// exactly as long as the directory it describes — a swap that replaces the +/// directory wholesale must not carry the old marker along inside it. +#[derive(Debug, serde::Serialize, serde::Deserialize)] +struct QuarantineMarker { + /// The attempt whose PUT was left unresolved. `None` when the bound expired + /// with no dispatched attempt to name (a state that should compensate rather + /// than quarantine, kept representable so a marker is never unparseable). + attempt: Option, + at: chrono::DateTime, +} + +/// Sidecar path for a live repo directory: `.{name}.git.quarantine` beside it. +fn quarantine_path(local_path: &Path) -> Option { + let parent = local_path.parent()?; + let file_name = local_path.file_name()?.to_string_lossy().to_string(); + Some(parent.join(format!(".{file_name}.quarantine"))) +} + +/// Mark the live tree as carrying an unresolved generation. Reads must reconcile +/// it before serving; nothing may delete it, because the PUT may have landed and +/// this can be the only local copy. +fn quarantine_local_tree(local_path: &Path, repo_name: &str, attempt: Option<&PublishAttemptId>) { + let Some(path) = quarantine_path(local_path) else { + return; + }; + let marker = QuarantineMarker { + attempt: attempt.map(|a| a.as_str().to_string()), + at: chrono::Utc::now(), + }; + let body = match serde_json::to_vec(&marker) { + Ok(body) => body, + Err(e) => { + warn!(repo = %repo_name, err = %e, "could not serialize the quarantine marker"); + return; + } + }; + match std::fs::write(&path, body) { + Ok(()) => warn!( + repo = %repo_name, + attempt = ?marker.attempt, + "quarantined the local tree: its publish outcome is unresolved, so reads must \ + reconcile it against the store before serving it" + ), + Err(e) => warn!( + repo = %repo_name, + err = %e, + "failed to write the quarantine marker — reads may serve an unconfirmed tree" + ), + } +} + +/// Lift a quarantine. Called wherever the live tree becomes a CONFIRMED +/// generation again: a publish the store acknowledged, an under-lock refresh +/// that overwrote the tree from the stored archive, a reconciliation that found +/// the attempt did land, or an invalidation that removed the tree entirely. +fn clear_quarantine(local_path: &Path, repo_name: &str) { + let Some(path) = quarantine_path(local_path) else { + return; + }; + match std::fs::remove_file(&path) { + Ok(()) => debug!(repo = %repo_name, "cleared the local tree's quarantine"), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => warn!( + repo = %repo_name, + err = %e, + "failed to clear the quarantine marker — reads will keep refusing this repo" + ), + } +} + +/// The quarantine on a live tree, when there is one. +fn read_quarantine(local_path: &Path) -> Option { + let path = quarantine_path(local_path)?; + let body = std::fs::read(&path).ok()?; + // An unparseable marker still means quarantined. Failing open on a corrupt + // sidecar would serve exactly the tree the sidecar exists to withhold. + Some(serde_json::from_slice(&body).unwrap_or(QuarantineMarker { + attempt: None, + at: chrono::Utc::now(), + })) +} + +/// Remove a refused write from the unlocked read cache so `acquire` cannot serve +/// a tree that never landed in object storage. +fn invalidate_local_write_cache(local_path: &Path, repo_name: &str, reason: &str) { + if !local_path.exists() { + clear_quarantine(local_path, repo_name); + return; + } + // Order matters: drop the marker only after the tree it describes is gone, + // so a failed removal leaves the quarantine standing rather than clearing + // the way for the tree it could not delete. + if let Err(e) = std::fs::remove_dir_all(local_path) { + warn!( + repo = %repo_name, + path = %local_path.display(), + err = %e, + reason, + "failed to invalidate local write cache after a refused publish" + ); + } else { + clear_quarantine(local_path, repo_name); + debug!( + repo = %repo_name, + reason, + "invalidated local write cache after a refused publish" + ); + } } /// Strict allowlist validator for `owner_did` and `repo_name`. @@ -644,251 +1430,815 @@ fn validate_repo_name(repo_name: &str) -> Result<()> { Ok(()) } -/// Error marker for "no lock-pool connection was available in time". +/// Owns a lock-pool connection across an in-flight `pg_try_advisory_lock`. /// -/// Carried through the `anyhow` chain (like [`smart_http::GitServiceTimeout`]) so the -/// HTTP handler can `downcast_ref` it and shed a 503 + Retry-After instead of the -/// generic 500 a git error maps to: an exhausted lock pool is a CAPACITY signal, and -/// telling the client to retry shortly is the same shed semantics the surrounding -/// admission code already uses (#173 F1). +/// A cancelled `.await` does not cancel an already-sent SQL statement, so a +/// try-lock whose future is dropped still takes the lock server-side while the +/// caller abandons the result. Protection therefore has to exist *before* the +/// statement goes out, which is what this type is: its `Drop` closes any +/// connection still held, ending the session so Postgres frees the lock. /// -/// [`smart_http::GitServiceTimeout`]: crate::git::smart_http::GitServiceTimeout -#[derive(Debug, thiserror::Error)] -#[error("no lock-pool connection available")] -pub struct LockPoolBusy; - -/// Guard returned by `acquire_write()`. Holds the Postgres advisory lock and -/// uploads to Tigris + releases the lock on `release()`. -pub struct RepoWriteGuard { - owner_slug: String, - repo_name: String, - pub local_path: PathBuf, - lock_key: i64, - /// The lock-pool connection that TOOK the advisory lock. It must be the one - /// that releases it (session locks are owned by their connection), and - /// holding it here is also what makes a guard dropped without `release` - /// safe: the drop returns the connection through the pool's `after_release` - /// hook, which runs `pg_advisory_unlock_all()`. +/// `close_on_drop()` is a one-way setter, so the arming lives here in `Drop` +/// rather than being set up front and cleared on success; "disarming" is +/// `Option::take`, which is what `take_conn` does once an acquire is observed. +/// This is the only place that issues `pg_try_advisory_lock`. +struct LockProbe { + conn: Option>, + /// True only when we have POSITIVELY established that this session does not + /// hold the lock, i.e. `try_lock` came back `false`. /// - /// `Option` because that hook is not a complete answer. When the unlock ERRORS - /// on a live session (a statement timeout, an admin cancel, an aborted - /// transaction), `after_release` issues its `pg_advisory_unlock_all()` on the - /// SAME broken session and it fails too, so the connection goes back to the pool - /// still holding the lock and nothing ever clears it (measured: never freed in - /// 15s, #174 F3b). Those paths `take()` the connection and close it instead; - /// ending the session is what actually frees the lock. `None` only after such a - /// disposal, or after `Drop` has moved it into the detached unlock. - lock_conn: Option>, - /// Set once `release` has run its unlock, making the `Drop` backstop inert. A - /// guard is only ever constructed with the lock already held, so there is no - /// "never locked" state to track alongside it. - released: bool, - tigris: Option, - /// Shared with the store that handed this guard out; see - /// [`RepoStore::upload_site_reached`]. - #[cfg(test)] - upload_site_reached: Arc, - /// Test-only seam: when set, `release` parks on this gate at the exact point it - /// is about to await `pg_advisory_unlock` (connection still owned, not yet - /// returned to the lock pool). Dropping the `release` future while it is parked - /// reproduces a mid-unlock cancellation, so a test can assert the lock is still - /// freed: the drop returns the connection through the pool's `after_release` - /// hook, which runs `pg_advisory_unlock_all()`. Never set outside tests. - #[cfg(test)] - test_pre_unlock_gate: Option>, + /// The predicate has to be "we know nothing was acquired," not "we saw an + /// answer." A `true` answer means the lock IS held, so dropping without handing + /// the connection to a guard leaks it exactly as a cancellation would; an + /// earlier version of this flag meant "settled" and reopened that leak. Default + /// false so both the cancelled-mid-flight and lock-acquired cases close, and + /// only ordinary contention returns the connection. + lock_not_taken: bool, } -/// Deadline for tearing down the connection that saw a failing `pg_advisory_unlock`. -/// Long enough that a healthy socket always finishes well inside it, short enough that -/// a blackholed one does not pin admission resources for a TCP timeout. -const UNLOCK_ERROR_CLOSE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); - -/// Await `close` under a deadline (#174 F3c). -/// -/// `release` awaits this INLINE while the global write permit, the per-source permit -/// and the write lease are all still held, and sqlx puts no deadline on `close()`: -/// it writes Terminate and then tears the socket down. The branch that reaches here is -/// by definition a connection whose last statement errored, and a blackholed TCP path -/// to Postgres (a cloud failover that drops packets without an RST) is a plausible -/// cause, so an unbounded await here parks every later push to the repo behind three -/// pinned admission resources until the steal bound. -/// -/// On elapsed the future is simply dropped, which drops the `PoolConnection` it owns. -/// Dropping it closes the socket, and closing the socket is what actually ends the -/// session and makes Postgres release the lock, so the deadline costs nothing the -/// graceful path was buying. -async fn close_conn_bounded( - repo_name: &str, - close: impl std::future::Future>, -) { - match tokio::time::timeout(UNLOCK_ERROR_CLOSE_TIMEOUT, close).await { - Ok(Ok(())) => {} - Ok(Err(e)) => { - warn!(repo = %repo_name, err = %e, - "closing the write-lock connection failed, the session teardown still frees the lock server-side"); - } - Err(_) => { - warn!(repo = %repo_name, timeout_secs = UNLOCK_ERROR_CLOSE_TIMEOUT.as_secs(), - "closing the write-lock connection timed out, dropping it instead; the socket goes down either way, which is what frees the lock server-side"); +impl LockProbe { + fn new(conn: sqlx::pool::PoolConnection) -> Self { + Self { + conn: Some(conn), + lock_not_taken: false, } } -} -impl RepoWriteGuard { - /// Path to the bare repo on local disk. - pub fn path(&self) -> &Path { - &self.local_path + /// Send the try-lock on the owned connection. + async fn try_lock(&mut self, key: i64) -> Result { + let conn = self + .conn + .as_mut() + .context("LockProbe::try_lock after the connection was taken")?; + // Cleared BEFORE the statement is sent, not after it answers. Once the + // statement is in flight this session may hold the lock, and an error or a + // cancellation gives us no way to find out, so the connection must not be + // returned to the pool on any path but a positive `false`. Assigning only on + // success would leave a previous `true`-derived value standing. + self.lock_not_taken = false; + let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut **conn) + .await + .context("trying advisory lock")?; + // Only a false answer licenses returning the connection: it means the + // statement completed and took nothing. A true answer means this session + // now holds the lock, so Drop must still close unless `take_conn` hands it + // to a guard. + self.lock_not_taken = !row.0; + Ok(row.0) } - /// Upload to Tigris (only when the write succeeded) and release the advisory - /// lock. Pass `success = false` when the write operation failed — uploading a - /// half-applied or otherwise inconsistent repo would propagate corruption to - /// Tigris (and to every node that later downloads it). The lock is always - /// released regardless, to avoid stale locks blocking future writes. - pub async fn release(mut self, success: bool) { - // Upload to Tigris only on success. + /// Hand the lock-owning connection out, leaving `Drop` with nothing to close. + /// Only call this after `try_lock` returned true. + /// + /// Named `take_` rather than `into_` deliberately: clippy expects an `into_*` + /// method to consume `self`, which a type implementing `Drop` cannot do + /// without tripping E0509. + fn take_conn(&mut self) -> Option> { + self.conn.take() + } +} + +impl Drop for LockProbe { + fn drop(&mut self) { + let Some(mut conn) = self.conn.take() else { + // take_conn already handed the connection to the guard. + return; + }; + if self.lock_not_taken { + // The probe ran and reported that someone else holds the key, so nothing + // was acquired here. Return the connection to the pool: closing would + // make a 60-attempt spinner tear down 60 backends for ordinary + // contention. Dropping `conn` unarmed does exactly that. + return; + } + // Either the future was dropped before we saw an answer, or the answer was + // that we DID take the lock and nobody took the connection off us. Both mean + // a session may be holding the lock with no one to release it, so end the + // session — which is what makes Postgres free it. + warn!("advisory-lock probe dropped while its session may hold the lock — closing the session to free it"); + conn.close_on_drop(); + } +} + +/// Non-mutating snapshot of a repo's latest Tigris state. Owns the throwaway +/// temp dir it was unpacked into and removes it on drop; a snapshot that +/// borrowed the live local path owns nothing and drops as a no-op. +pub struct RepoSnapshot { + path: PathBuf, + owned: bool, +} + +impl RepoSnapshot { + /// Path to the snapshot's bare repo directory. + pub fn path(&self) -> &Path { + &self.path + } + + pub(crate) fn from_owned_path(path: PathBuf) -> Self { + Self { path, owned: true } + } +} + +impl Drop for RepoSnapshot { + fn drop(&mut self) { + if self.owned { + let _ = std::fs::remove_dir_all(&self.path); + } + } +} + +/// Guard returned by `acquire_write()`. Holds the Postgres advisory lock and +/// uploads to Tigris + releases the lock on `release()`. +pub struct RepoWriteGuard { + owner_slug: String, + repo_name: String, + pub local_path: ValidatedRepoDiskPath, + lock_key: i64, + /// The connection that TOOK the lock. Postgres advisory locks are + /// session-scoped, so only this session can release it; holding it here is + /// what makes `release` land on the right backend instead of an arbitrary + /// pooled one. + conn: Option>, + tigris: Option, + /// Bound on the release-side upload, which runs with the lock still held. + lock_held_transfer_timeout: Duration, + /// The generation of the stored archive as observed by the HEAD inside + /// `acquire_write`, under the lock. `release` publishes fenced on it, so a + /// PUT abandoned by an earlier writer's timeout cannot land on top of a + /// successor's acknowledged archive. + publish_fence: UploadPrecondition, + /// When set, a timed-out or cancelled under-lock refresh revokes this before the + /// guard drops so a detached `spawn_blocking` extraction cannot swap into the live + /// tree after its advisory-lock ownership ends. + refresh_swap_authority: Option>, + /// How far THIS guard's publish attempt got, readable from outside the + /// `release` future. A cancelled handler never returns a `ReleaseOutcome`, so + /// this is the only thing that can tell "the PUT was never constructed" from + /// "the PUT is on the wire and may commit" after the future is gone. + publish_stage: Arc, + /// Test-only seam: when set, `release` parks on this gate at the exact point + /// it is about to await `pg_advisory_unlock` (connection still owned, not yet + /// released). Dropping the `release` future while it is parked reproduces a + /// mid-unlock cancellation, so a test can assert the `Drop` backstop still + /// frees the session lock. Never set outside tests. + #[cfg(test)] + test_pre_unlock_gate: Option>, + #[cfg(test)] + upload_site_reached: Arc, +} + +impl RepoWriteGuard { + /// Backend pid of the session holding the lock. Test-only observable for the + /// must-not-over-close check: if `release` closed the session instead of + /// returning it, consecutive writes would report different pids. + #[cfg(test)] + async fn backend_pid_for_test(&mut self) -> i32 { + let conn = self + .conn + .as_mut() + .expect("guard still holds its connection"); + let pid: (i32,) = sqlx::query_as("SELECT pg_backend_pid()") + .fetch_one(&mut **conn) + .await + .expect("backend pid"); + pid.0 + } + + /// Path to the bare repo on local disk. + pub fn path(&self) -> &Path { + self.local_path.as_path() + } + + /// Publish the tree this guard wrote, fenced on the generation observed + /// under the lock, with at most ONE supersede-retry after a definite loss. + /// + /// Hard bound of two PUT attempts per release. No loop, no recursion: a + /// third attempt would have no more reason to terminate than the second. + async fn publish(&self, tigris: &TigrisClient) -> std::result::Result<(), PublishRefusal> { + match tigris + .upload_tracked( + &self.owner_slug, + &self.repo_name, + &self.local_path, + self.publish_fence.clone(), + PublishAttemptId::new(), + Some(&self.publish_stage), + ) + .await + { + Ok(receipt) => { + debug!( + repo = %self.repo_name, + attempt = %receipt.attempt, + etag = ?receipt.etag, + "release published the writer's tree" + ); + return Ok(()); + } + Err(UploadError::PreconditionLost { status }) => { + // EPISTEMIC ASYMMETRY, and it is why one retry is sound here + // while the timeout arm in `release` deliberately does nothing. + // A refused precondition is a DEFINITE outcome: the store told + // us the generation we observed under the lock is gone, and that + // our bytes did not land. A timeout tells us nothing at all. + // + // We also still hold the advisory lock, so no successor can have + // acquired and published. Whatever landed underneath was written + // WITHOUT the lock: init's create-only upload of a freshly + // created empty repo, or a PUT abandoned by an earlier writer + // whose own release timed out. This writer's tree is the + // authority over both, which is what makes exactly one + // supersede-retry correct rather than a race. + // + // Honest residual: when the thing underneath was a genuine + // orphan that landed AFTER this writer's refresh, the retry + // supersedes it with a tree that does not contain it. That is + // the same outcome today's unconditional publish produces. The + // fence protects an acknowledged successor from an orphan; it + // does not protect an unlocked orphan from the lock holder. + warn!( + repo = %self.repo_name, + status, + "publish fence lost: the stored archive changed under the lock, \ + republishing once on the current generation" + ); + } + Err(e) => return Err(PublishRefusal::Failed(e)), + } + + let fresh = match tigris.head_etag(&self.owner_slug, &self.repo_name).await { + Ok(Some(etag)) => UploadPrecondition::IfMatch(etag), + // Nothing is stored now, so create-only is the fence that matches + // what was just observed. + Ok(None) => UploadPrecondition::IfAbsent, + // A HEAD is a read. It cannot have published anything, so failing it + // is a definite non-publication for THIS attempt — the first PUT was + // already refused outright above. + Err(e) => return Err(PublishRefusal::Failed(UploadError::NotPublished(e))), + }; + match tigris + .upload_tracked( + &self.owner_slug, + &self.repo_name, + &self.local_path, + fresh, + PublishAttemptId::new(), + Some(&self.publish_stage), + ) + .await + { + Ok(receipt) => { + debug!( + repo = %self.repo_name, + attempt = %receipt.attempt, + etag = ?receipt.etag, + "the supersede-retry published the writer's tree" + ); + Ok(()) + } + Err(UploadError::PreconditionLost { status }) => { + // Two definite losses in a row: something is publishing this key + // without the lock faster than we can fence on it. Refuse rather + // than escalate. The write is on local disk and in this node's + // tree, but it is NOT durable in object storage, so the caller + // must not report success. + warn!( + repo = %self.repo_name, + status, + "publish fence lost again on the refreshed generation, refusing the \ + write rather than attempting a third publish" + ); + Err(PublishRefusal::Fenced) + } + Err(e) => Err(PublishRefusal::Failed(e)), + } + } + + /// The publish stage of this guard's attempt, shared with whoever needs to + /// classify a CANCELLED release. See [`PublishStageCell`]. + pub fn publish_stage(&self) -> Arc { + Arc::clone(&self.publish_stage) + } + + /// Decide what a publish that ran out its bound actually left behind. + /// + /// "The bounded transfer returned `None`" is not one state. `publish` awaits + /// a blocking compression before it constructs the request, so the bound can + /// expire with nothing on the wire at all. Only the stage distinguishes them, + /// and only a dispatched attempt is genuinely unknowable. + /// + /// For a dispatched attempt this spends one short, separately bounded HEAD + /// asking the store whether THIS attempt's bytes are what it holds. A `true` + /// there is a real confirmation — the attempt id travelled with the bytes — + /// and upgrades the outcome to `Released`. A `false` is NOT a refutation: the + /// request may still be in flight, so it stays unknowable and the tree is + /// quarantined rather than deleted. + async fn resolve_unfinished_publish(&self, tigris: &TigrisClient) -> ReleaseOutcome { + let stage = self.publish_stage.get(); + if !stage.may_have_published() { + warn!( + repo = %self.repo_name, + ?stage, + "release upload exceeded its bound before any PUT was dispatched — the write \ + definitively did not publish" + ); + return ReleaseOutcome::UploadFailed; + } + warn!( + repo = %self.repo_name, + "release upload exceeded its bound; the PUT may still land, so the outcome is \ + unknowable and the conditional upload is what keeps a late publish from \ + overwriting a successor's archive" + ); + let Some(attempt) = stage.unresolved_attempt().cloned() else { + // `Published` reaches here only if the bound expired between the + // acknowledged PUT and the outer future resuming. The store already + // answered, so this is durable. + return ReleaseOutcome::Released; + }; + // A separate, deliberately small slice rather than a share of the + // already-exhausted publish budget: this runs with the lock still held, + // and a store that just stalled for the whole bound must not be able to + // hold it for a second one. + match bounded_transfer( + "release-reconcile", + &self.repo_name, + RECONCILE_BOUND, + tigris.attempt_landed(&self.owner_slug, &self.repo_name, &attempt), + ) + .await + { + Some(Ok(true)) => { + info!( + repo = %self.repo_name, + attempt = %attempt, + "reconciled an abandoned publish: the store holds this attempt's archive" + ); + self.publish_stage.set(PublishStage::Published { + attempt, + etag: None, + }); + ReleaseOutcome::Released + } + _ => ReleaseOutcome::UploadUnknowable, + } + } + + /// Upload to Tigris (only when the write succeeded) and release the advisory + /// lock. Pass `success = false` when the write operation failed — uploading a + /// half-applied or otherwise inconsistent repo would propagate corruption to + /// Tigris (and to every node that later downloads it). The lock is always + /// released regardless, to avoid stale locks blocking future writes. + pub async fn release(self, success: bool) -> ReleaseOutcome { + self.release_maybe_compensate(success, None:: anyhow::Result<()>>) + .await + } + + /// Like [`release`], but runs `compensate` on the live tree while the advisory + /// lock is still held when publish ends in a definite refusal (`Fenced` or + /// `UploadFailed`). It is deliberately not run on `UploadUnknowable`, where the + /// PUT may still land and a post-release undo would race a successor writer. + pub async fn release_compensating(self, success: bool, compensate: F) -> ReleaseOutcome + where + F: FnOnce(&Path) -> anyhow::Result<()>, + { + self.release_maybe_compensate(success, Some(compensate)) + .await + } + + async fn release_maybe_compensate( + mut self, + success: bool, + compensate: Option, + ) -> ReleaseOutcome + where + F: FnOnce(&Path) -> anyhow::Result<()>, + { + let mut outcome = ReleaseOutcome::Released; + // Upload to Tigris only on success. if success { - // The upload site, recorded for tests before the client is consulted: with - // no injectable seam on `TigrisClient` a counter inside the arm below could - // never move, and it is reaching this point at all that an interrupted push - // must not do (#173 F2). #[cfg(test)] self.upload_site_reached .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - if let Some(ref tigris) = self.tigris { - if let Err(e) = tigris - .upload(&self.owner_slug, &self.repo_name, &self.local_path) - .await + if let Some(tigris) = self.tigris.clone() { + // ONE budget for the whole publish, covering both attempts and + // the HEAD between them, for the same reason the acquire-side + // refresh uses one for its HEAD and download together: this runs + // with the lock held and a lock-pool slot pinned, so bounding + // each attempt separately would double the worst-case occupancy. + match bounded_transfer( + "release-upload", + &self.repo_name, + self.lock_held_transfer_timeout, + self.publish(&tigris), + ) + .await { - warn!(repo = %self.repo_name, err = %e, "failed to upload repo to tigris after write"); + Some(Ok(())) => {} + // Both attempts were definitively refused. The raise site + // already logged which and why, so this only has to carry + // the refusal out to the caller. + Some(Err(PublishRefusal::Fenced)) => outcome = ReleaseOutcome::Fenced, + Some(Err(PublishRefusal::Failed(e))) => { + // SPLIT BY WHAT THE FAILURE PROVES, not by "it was an + // error". `UploadFailed` runs destructive compensation + // below, and a response-loss error does not license it: + // the store can accept a complete conditional PUT and + // lose the response before the client reads it, so + // invalidating the cache and undoing the write would + // discard state that IS published. + if e.proves_not_published() { + warn!(repo = %self.repo_name, err = %e, "failed to upload repo to tigris after write"); + outcome = ReleaseOutcome::UploadFailed; + } else { + warn!( + repo = %self.repo_name, + err = %e, + "release upload failed WITHOUT proving it did not commit; \ + quarantining the local tree rather than compensating" + ); + outcome = ReleaseOutcome::UploadUnknowable; + } + } + None => { + // Timed out is UNKNOWABLE, not failed: the PUT may well + // still land after this returns, so there is deliberately + // no compensating action here. The lock is released + // normally regardless. Holding it would fence nothing, + // because `release` takes `mut self`: the guard drops the + // moment this function returns and `Drop` closes the + // session, so the lock would free within milliseconds + // either way. What actually protects a successor from a + // late publish is the conditional PUT on the upload, not + // the lifetime of this lock. The caller still must not + // report success: `into_result` maps this to a retryable + // refusal. + // + // But "timed out" is not one state, it is two, and only + // the STAGE can tell them apart. The publish awaits a + // blocking compression before it constructs the request + // at all, so a bound that expires there means no PUT was + // ever dispatched — a definite non-publication, safe to + // compensate. A bound that expires after dispatch is the + // genuinely unknowable case. + outcome = self.resolve_unfinished_publish(&tigris).await; + } } } } else { warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo"); } + if success { + match outcome { + ReleaseOutcome::Fenced | ReleaseOutcome::UploadFailed => { + invalidate_local_write_cache( + &self.local_path, + &self.repo_name, + "definite publish refusal", + ); + if let Some(compensate) = compensate { + if let Err(e) = compensate(self.path()) { + warn!( + repo = %self.repo_name, + err = %e, + "compensation after a definite publish refusal failed" + ); + } + } + } + ReleaseOutcome::Released => { + // Publication confirmed: the live tree IS the stored + // generation, so any quarantine an earlier attempt left on + // this path is answered and must be lifted, or reads would + // stay refused forever after a single unresolved write. + clear_quarantine(&self.local_path, &self.repo_name); + } + ReleaseOutcome::UploadUnknowable => { + // THE CACHE CONTRACT. The local tree carries this writer's + // refs but nothing ties it to a confirmed object-store + // generation, and deleting it is not safe either — the PUT + // may have landed and this could be the only local copy. + // + // So mark it, and make `acquire` reconcile before it serves. + // Without the marker the existing-path fast path hands the + // tree to every subsequent read on filesystem existence + // alone, and a refused write is served indefinitely while + // durable storage still holds the previous generation. + quarantine_local_tree( + &self.local_path, + &self.repo_name, + self.publish_stage.get().unresolved_attempt(), + ); + } + } + } + + // Release the advisory lock on the SAME session that took it. Unlocking + // through the pool would land on an arbitrary backend, where the call is a + // silent no-op. + // + // Read the boolean. `pg_advisory_unlock` reports "you did not hold this + // lock" as a false RETURN VALUE plus a server WARNING, never an error, so a + // discarded result cannot distinguish a real release from a no-op. A false + // here means this session's lock state is not what we believe it is, so the + // connection is left in `self.conn` for `Drop` to close rather than being + // handed back to the pool as clean. Only a confirmed unlock returns it. + let lock_key = self.lock_key; // Test-only: park right before the unlock await so a test can drop this - // future mid-unlock, with the connection still owned. + // future mid-unlock (connection owned, not yet released). #[cfg(test)] if let Some(gate) = self.test_pre_unlock_gate.clone() { gate.notified().await; } - // Release the advisory lock on the connection that took it. Anything else - // (a fresh `&pool` checkout) is a no-op that returns false: Postgres - // scopes a session lock to its owning connection. - // - // Unlock through the connection while it is STILL owned by `self`; do not - // `take()` it first. A cancellation during this await then drops `self` with - // the connection still in place, so it returns to the lock pool and - // `after_release` clears the lock (#174 F4). - let unlock = match self.lock_conn.as_deref_mut() { - Some(conn) => Some( - sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(self.lock_key) - .execute(&mut *conn) - .await, - ), - None => None, + let unlock = if let Some(conn) = self.conn.as_mut() { + bounded_transfer( + "advisory-unlock", + &self.repo_name, + self.lock_held_transfer_timeout, + sqlx::query_as::<_, (bool,)>("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .fetch_one(&mut **conn), + ) + .await + } else { + None }; - // An unlock that ERRORS is a different failure from a cancellation: the await - // resolved, so the session is alive and still holds the lock. Returning that - // connection to the pool does NOT recover it, because `after_release` runs its - // `pg_advisory_unlock_all()` on the same broken session and fails identically - // (#174 F3b). Close it: ending the session is what frees the lock. - if let Some(Err(e)) = unlock { - warn!(repo = %self.repo_name, err = %e, - "advisory unlock failed, closing the connection so the session ends and postgres drops the lock"); - if let Some(conn) = self.lock_conn.take() { - close_conn_bounded(&self.repo_name, conn.close()).await; + match unlock { + Some(Ok((true,))) => { + // Confirmed released: safe to return to the pool. + self.conn.take(); + } + Some(Ok((false,))) => { + warn!( + repo = %self.repo_name, + lock_key, + "advisory unlock reported the session did not hold this lock — closing the session instead of pooling it" + ); + } + Some(Err(e)) => { + warn!( + repo = %self.repo_name, + lock_key, + err = %e, + "advisory unlock failed — closing the session so the lock cannot outlive it" + ); } + None if self.conn.is_some() => { + warn!( + repo = %self.repo_name, + lock_key, + bound_secs = self.lock_held_transfer_timeout.as_secs(), + "advisory unlock exceeded its bound — closing the session so the lock-pool slot is not held longer" + ); + } + None => {} } - // On the clean path, dropping `self` returns the connection to the lock pool, - // where `after_release` sweeps anything the unlock above missed. - self.released = true; + + outcome } } impl Drop for RepoWriteGuard { - /// Backstop for a guard dropped WITHOUT `release` (a cancelled `acquire_write`, a - /// handler future dropped before the release call). The pool's `after_release` - /// hook covers the ordinary case on its own, but not one: if the detached unlock - /// ERRORS on a live session, the hook's `pg_advisory_unlock_all()` fails the same - /// way and the connection returns to the pool still holding the lock (#174 F3b). - /// So the unlock runs here and disposes of the connection when it errors. - /// - /// `Drop` cannot await, so the unlock is spawned; it runs on the same session, - /// which is what makes it effective. With no runtime to spawn onto there is - /// nothing that can unlock, so the connection is detached and dropped instead: - /// closing the socket ends the session, and that frees the lock server-side. fn drop(&mut self) { - if self.released { - return; + if let Some(authority) = self.refresh_swap_authority.take() { + revoke_swap_authority(&authority); } - let Some(mut conn) = self.lock_conn.take() else { + + let Some(mut conn) = self.conn.take() else { + // release() already unlocked and handed the connection back. return; }; - let lock_key = self.lock_key; - let repo_name = self.repo_name.clone(); - match tokio::runtime::Handle::try_current() { - Ok(handle) => { - handle.spawn(async move { - let unlock = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(lock_key) - .execute(&mut *conn) - .await; - // Same failure as `release`'s, one level down: the await RESOLVED - // with an error, so the session is alive and still holds the lock. - // Ending this block would drop `conn` and RETURN it to the pool, - // where `after_release` fails identically. Close it instead. - if let Err(e) = unlock { - warn!(repo = %repo_name, err = %e, "detached advisory-unlock on write-guard drop failed, closing the connection so the session ends and postgres drops the lock"); - close_conn_bounded(&repo_name, conn.close()).await; - } - }); - } - Err(_) => { - // `PoolConnection`'s own drop spawns its return-to-pool task, which - // panics with no runtime. `detach` gives up the pool slot and yields a - // plain `PgConnection`; dropping that closes the socket, which ends the - // session and is what frees the lock. - drop(conn.detach()); - warn!( - repo = %repo_name, - "RepoWriteGuard dropped off a Tokio runtime; no detached unlock is \ - possible, so the pinned connection is disposed of instead: ending \ - the session is what releases the advisory lock" - ); + + // Reached on any exit that skipped release(): an early `?`, a panic, or an + // axum handler future cancelled when the client disconnected. The session + // still holds the advisory lock, so returning it to the pool would block + // every future write to this repo until sqlx recycles the connection. + // + // `PoolConnection::drop` spawns onto the runtime, both to close and to + // return, and panics outright when no runtime handle exists. A panic here + // would run inside a `Drop` and abort the process during unwind, so check + // for a runtime first. With none, the process is already going away: leak + // the handle deliberately rather than panic, and let socket teardown end + // the session, which is what frees the lock at exit anyway. + if tokio::runtime::Handle::try_current().is_ok() { + warn!( + repo = %self.repo_name, + "write guard dropped without release() — closing its session to free the advisory lock" + ); + conn.close_on_drop(); + } else { + warn!( + repo = %self.repo_name, + "write guard dropped with no runtime alive — detaching the connection so Drop cannot panic" + ); + // `PoolConnection::drop` spawns onto the runtime for BOTH closing and + // returning, and panics without a handle; a panic inside Drop aborts the + // process during unwind. `leak()` detaches the raw `PgConnection`, which + // has no Drop impl of its own, so dropping it closes the socket + // synchronously with no runtime involved. That frees the lock + // immediately rather than at process exit, and leaks no fd — strictly + // better than the mem::forget this replaced. + drop(conn.leak()); + } + } +} + +/// Default wall-clock cap on WAITING for the per-repo advisory lock. +/// +/// An attempt-count bound alone is not enough: 60 attempts each paying a pool +/// acquire plus a 1s sleep reach roughly 360s. +/// +/// This bounds the wait only, NOT the whole of `acquire_write`. The under-lock +/// refresh carries its own separate bound (`lock_held_transfer_timeout`, default +/// 300s), so the two compose rather than nest and a caller can legitimately spend +/// this deadline waiting and then that bound refreshing. Do not read 90s as a +/// promise that `acquire_write` returns inside the 120s proxy idle timeout in +/// `infra/fly/fly.toml`; it is not, and reconciling the two is tracked separately. +const LOCK_ACQUIRE_DEADLINE: Duration = Duration::from_secs(90); + +/// Why an under-lock refresh did not complete, split by what it leaves us knowing. +/// +/// `Unknown` (the existence check failed) and `Download` (the archive is there and +/// unreadable) must not share a branch: neither establishes that the local tree +/// matches the generation the HEAD observed. +enum RefreshFailure { + Unknown(anyhow::Error), + /// Carries the GET error. The under-lock path refuses the write rather than + /// falling back to local, because a failed GET does not prove the cached tree + /// is current. + Download { + err: anyhow::Error, + }, +} + +/// What `release` was able to do with the writer's tree. +/// +/// `#[must_use]` because dropping it is the whole defect this type exists to +/// prevent: a publish the store refused would otherwise return 201, fire +/// webhooks, and record a push that no successor can read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[must_use = "a refused publish must reach the caller, or a write that never landed reports success"] +pub enum ReleaseOutcome { + /// The lock was released and nothing definitively refused the publish. + /// Also the answer when there was nothing to publish (a failed write, no + /// storage backend). + Released, + /// The store refused the publish twice. The tree is on local disk but is + /// NOT in object storage, so the caller must not report success. + Fenced, + /// The archive upload exceeded its under-lock bound. The PUT may still + /// land, but durability is unknowable, so the caller must not report + /// success. + UploadUnknowable, + /// The archive upload failed with a definite error (not a fence refusal). + UploadFailed, +} + +impl ReleaseOutcome { + /// Fold into the `Result` a handler propagates with `?`. + /// + /// Call this at every publishing site IMMEDIATELY after `release`, before + /// any post-release effect. A refusal that short-circuits after the DB + /// write, the webhook, or the response body has already happened is not a + /// refusal at all. + pub fn into_result(self) -> anyhow::Result<()> { + match self { + ReleaseOutcome::Released => Ok(()), + ReleaseOutcome::UploadUnknowable | ReleaseOutcome::UploadFailed => { + Err(anyhow::Error::new(RepoUnavailable).context( + "archive upload did not complete durably before the lock was released", + )) } + // The raise site inside `publish` already logged the repo and the + // status, so this carries no detail: the handler layer turns it + // into a fixed 503 body. + ReleaseOutcome::Fenced => Err(anyhow::Error::new(RepoWriteFenced) + .context("release-side publish refused by the store on both attempts")), } } } -/// Build the dedicated advisory-lock pool a `RepoStore` runs its write locks on. -/// Connect options are cloned off an existing pool so callers need not re-parse -/// the database URL; the pool is lazy, so no connection is opened here. +/// Why the release-side publish did not land, split by what it leaves the +/// caller able to claim. /// -/// Two properties, both load-bearing: +/// `Fenced` is a DEFINITE refusal by the store after both attempts, so the +/// write is not durable and the caller must not report success. `Failed` is +/// every other upload failure, which keeps today's behavior (log it, release +/// the lock, let the caller answer normally). +enum PublishRefusal { + Fenced, + Failed(UploadError), +} + +/// The per-repo advisory lock was not obtained within the acquire deadline. +/// +/// A distinct type rather than a bare `anyhow` string so the handler layer can map +/// it to a retryable 503 with a FIXED body. Contention is transient and ordinary, +/// and the internal message names the owner slug and repo, which must stay in the +/// log rather than reaching the client. +#[derive(Debug)] +pub struct RepoBusy; + +impl std::fmt::Display for RepoBusy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("repository is busy") + } +} + +impl std::error::Error for RepoBusy {} + +/// The under-lock refresh could not establish what is in object storage, so the +/// write was refused rather than run against a possibly-stale tree. +/// +/// A distinct type rather than a bare `anyhow` string so the handler layer can map +/// it to a retryable 503 with a FIXED body. The internal message names the owner +/// slug and repo, which must stay in the log at the raise site rather than reaching +/// the client. +#[derive(Debug)] +pub struct RepoUnavailable; + +impl std::fmt::Display for RepoUnavailable { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("repository is temporarily unavailable") + } +} + +impl std::error::Error for RepoUnavailable {} + +/// The release-side publish was refused by the store on both attempts, so the +/// write is not durable in object storage. +/// +/// A distinct type rather than a bare `anyhow` string, for the same reason as +/// its two siblings above: the handler layer maps it to a retryable 503 with a +/// FIXED body, and the detail (which repo, which status) stays in the log at +/// the raise site. Distinct FROM those siblings because the condition is +/// different: not contention and not an unreadable store, but another writer +/// holding the key. The client's retry re-runs the whole write against the tree +/// that actually won. +#[derive(Debug)] +pub struct RepoWriteFenced; + +impl std::fmt::Display for RepoWriteFenced { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("repository write was fenced by a concurrent publish") + } +} + +impl std::error::Error for RepoWriteFenced {} + +/// Run a future under a wall-clock bound, returning `None` if it did not finish. +/// +/// For the object-storage transfers that run while the per-repo advisory lock is +/// held. Those were free before the lock's connection was pinned to the guard; +/// now an unbounded transfer holds a lock-pool slot for as long as it stalls, and +/// enough of them deny every write on the node. +/// +/// A timed-out transfer is **unknowable**, not failed: it may well have landed. +/// Callers must not compensate as though it definitely failed. +async fn bounded_transfer(label: &str, repo: &str, limit: Duration, fut: F) -> Option +where + F: std::future::Future, +{ + match tokio::time::timeout(limit, fut).await { + Ok(v) => Some(v), + Err(_) => { + warn!( + repo = %repo, + transfer = label, + limit_secs = limit.as_secs(), + "object-storage transfer exceeded its under-lock bound — giving up so the advisory lock and its pool slot are not held longer" + ); + None + } + } +} + +/// Test-only re-export of the advisory-lock key derivation, so handler tests can +/// hold a repo's lock from an independent session. +#[cfg(test)] +pub fn advisory_lock_key_for_test(owner_slug: &str, repo_name: &str) -> i64 { + advisory_lock_key(owner_slug, repo_name) +} + +/// Error marker for "no lock-pool connection was available in time". /// -/// * The `after_release` hook runs `pg_advisory_unlock_all()` before a -/// connection goes back into the pool. sqlx's `PoolConnection::drop` spawns -/// `return_to_pool()`, which invokes this hook, so a connection dropped by -/// CANCELLATION still clears its locks. That is what keeps an `acquire_write` -/// killed mid-Tigris by the caller's `tokio::time::timeout` from leaking a -/// lock and wedging every later push to that repo (#173). Note the hook runs -/// from that spawned task, so the unlock is asynchronous with respect to the -/// drop: the lock clears shortly after the connection goes away, not -/// synchronously with it. -/// * It is a SEPARATE pool from the main query pool, not a slice of it. A push -/// holds its lock connection for the whole receive-pack, so drawing these -/// from the main pool would let a burst of `max_concurrent_git_pushes` -/// pushes park that many query connections for the length of their -/// receive-packs and starve every other query. That is true at any pool -/// size, so the separation does not rest on how the two knobs are set; -/// `Config::validate` separately requires `db_max_connections` to clear -/// `max_concurrent_git_pushes` by `DB_POOL_APP_HEADROOM`. +/// Carried through the `anyhow` chain so the HTTP handler can `downcast_ref` it +/// and shed a 503 + Retry-After instead of the generic 500 a git error maps to +/// (#173 F1). +#[derive(Debug, thiserror::Error)] +#[error("no lock-pool connection available")] +pub struct LockPoolBusy; + +/// Build a dedicated advisory-lock pool for tests and for callers that derive one +/// from an existing pool. Connect options are cloned off `source`; the pool is lazy. /// -/// `acquire_timeout` bounds the wait when every lock-pool connection is busy, so -/// exhaustion surfaces as a clean error rather than an unbounded hang. +/// The `after_release` hook runs `pg_advisory_unlock_all()` before a connection is +/// reused, which is what makes cancellation of an in-flight `acquire_write` safe +/// when combined with the session-pinning design in `RepoWriteGuard`. +#[allow(dead_code)] // production uses Db::lock_pool; tests build pools through this helper pub fn build_lock_pool(source: &PgPool, max_connections: u32, acquire_timeout: Duration) -> PgPool { PgPoolOptions::new() .max_connections(max_connections) @@ -931,511 +2281,94 @@ pub(crate) fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { #[cfg(test)] mod tests { use super::*; - use std::time::Duration; - - // ── advisory-lock test helpers (#173 U1) ─────────────────────────────── + use crate::git::tigris::ATTEMPT_METADATA_KEY; + use std::collections::HashMap; - /// Postgres advisory locks live in a CLUSTER-wide space, not a per-database - /// one, so two `#[sqlx::test]` cases running against their own temporary - /// databases still share the key space. Every lock test therefore mints its - /// own key instead of reusing a fixed constant. - fn unique_lock_key() -> i64 { - use std::sync::atomic::{AtomicI64, Ordering}; - static NEXT: AtomicI64 = AtomicI64::new(0); - let n = NEXT.fetch_add(1, Ordering::Relaxed); - ((std::process::id() as i64) << 24) | (n & 0xff_ffff) - } - - /// A plain pool (no `after_release` hook, no idle timeout) at the same - /// database as the `#[sqlx::test]` pool. Two separate reasons these tests - /// cannot just use the pool the harness hands them: - /// - /// 1. Observing lock state has to happen from a session that is definitely - /// not the one under test. Session advisory locks are re-entrant, so - /// `pg_try_advisory_lock` on the very connection that already holds the key - /// returns true, and a same-pool probe silently reports a leaked lock free. - /// 2. The harness pool sets `idle_timeout(1s)`, so a connection returned to it - /// is closed about a second later and Postgres drops every lock that - /// session held. That would mask exactly the leak these tests exist to - /// catch, so the store under test runs on one of these too. - fn sibling_pool(pool: &PgPool, max_connections: u32) -> PgPool { - sqlx::postgres::PgPoolOptions::new() - .max_connections(max_connections) - .connect_lazy_with((*pool.connect_options()).clone()) - } - - /// Probe the lock from a connection that is NOT the one under test. Session - /// advisory locks are re-entrant within their own session, so a check from the - /// holding connection would pass vacuously and prove nothing. - async fn lock_is_free_elsewhere(pool: &PgPool, key: i64) -> bool { - let mut probe = pool.acquire().await.expect("probe connection"); - let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") - .bind(key) - .fetch_one(&mut *probe) - .await - .expect("probe try-lock"); - if taken.0 { - sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(key) - .execute(&mut *probe) - .await - .expect("probe unlock"); - } - taken.0 + #[test] + fn non_durable_release_outcomes_do_not_report_success() { + assert!(ReleaseOutcome::Released.into_result().is_ok()); + assert!(ReleaseOutcome::UploadUnknowable.into_result().is_err()); + assert!(ReleaseOutcome::UploadFailed.into_result().is_err()); + assert!(ReleaseOutcome::Fenced.into_result().is_err()); } - /// `after_release` runs from the task sqlx spawns in `PoolConnection::drop`, - /// so the unlock is ASYNCHRONOUS with respect to the drop. Callers must poll - /// rather than assume the lock is gone the instant the connection goes away. - async fn wait_until_free(pool: &PgPool, key: i64, within: Duration) -> bool { - let deadline = std::time::Instant::now() + within; - loop { - if lock_is_free_elsewhere(pool, key).await { - return true; - } - if std::time::Instant::now() >= deadline { - return false; - } - tokio::time::sleep(Duration::from_millis(25)).await; - } + #[test] + fn swap_commit_token_is_exclusive() { + let authority = Arc::new(AtomicBool::new(true)); + assert!(try_claim_swap_commit(&authority)); + assert!(!try_claim_swap_commit(&authority)); + revoke_swap_authority(&authority); + assert!(!try_claim_swap_commit(&authority)); } - // ── DESIGN GATE ──────────────────────────────────────────────────────── - // The whole cancellation-safety design rests on one sqlx behaviour: - // `PoolConnection::drop` spawns `return_to_pool()`, which invokes the pool's - // `after_release` hook before the connection is reused. If that holds, a - // connection dropped by cancellation still runs `pg_advisory_unlock_all()` - // and the lock cannot leak. This test proves it by execution, through the - // production `build_lock_pool` so that stripping the hook there turns it red. - - #[sqlx::test] - async fn dropped_pool_connection_runs_after_release_and_clears_locks(pool: PgPool) { - let key = unique_lock_key(); - let lock_pool = build_lock_pool(&pool, 4, Duration::from_secs(5)); - - { - let mut conn = lock_pool.acquire().await.expect("lock-pool connection"); - let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") - .bind(key) - .fetch_one(&mut *conn) - .await - .expect("try-lock"); - assert!(taken.0, "first try-lock must succeed"); - assert!( - !lock_is_free_elsewhere(&pool, key).await, - "lock must be observably HELD from another session while the connection lives" - ); - // Drop WITHOUT calling pg_advisory_unlock: this models cancellation. - } - + #[test] + fn revoked_publish_swap_cannot_replace_the_live_tree() { + let root = tempfile::TempDir::new().unwrap(); + let repos_dir = root.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let owner = "did:key:z6MkRevokedSwapAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "repo"; + let live = validated_repo_disk_path(&repos_dir, owner, name).unwrap(); + std::fs::create_dir_all(&live).unwrap(); + std::fs::write(live.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + + let tmp = live + .parent() + .unwrap() + .join(format!(".{}.tmp-extract.test", name)); + std::fs::create_dir_all(&tmp).unwrap(); + std::fs::write(tmp.join("HEAD"), "ref: refs/heads/evil\n").unwrap(); + + let authority = Arc::new(AtomicBool::new(false)); + let err = swap_extracted_into_validated_repo(&live, &tmp, Some(&authority)) + .expect_err("a revoked authority must refuse the swap"); assert!( - wait_until_free(&pool, key, Duration::from_secs(5)).await, - "after_release must clear the advisory lock of a dropped connection" + err.to_string().contains("publish swap revoked"), + "unexpected error: {err:#}" ); - } - - // ── acquire_write cancellation safety (#173 U1) ──────────────────────── - - /// The reviewer's named regression. `api/repos.rs` wraps `acquire_write` in a - /// `tokio::time::timeout`; when that fires during the Tigris phase the future - /// is dropped after the advisory lock was taken and before `RepoWriteGuard` - /// (the only thing that unlocks) exists. The lock then leaks and every later - /// push to the same repo spins the 60-attempt / 60s ceiling and fails. - #[sqlx::test] - async fn cancelled_acquire_write_mid_tigris_does_not_leak_the_lock(pool: PgPool) { - let repos_dir = PathBuf::from("/tmp/gitlawb-test-repos"); - let owner = "did:key:z6MkCancelMidTigris"; - let repo = "cancel-mid-tigris"; - - let store_pool = sibling_pool(&pool, 8); - let stalling = RepoStore::for_testing(repos_dir.clone(), store_pool.clone()) - .with_tigris_stall(Duration::from_secs(30)); - let cancelled = tokio::time::timeout( - Duration::from_millis(500), - stalling.acquire_write(owner, repo), - ) - .await; assert!( - cancelled.is_err(), - "the acquire must still be inside the Tigris phase when the timeout fires" + live.join("HEAD").exists(), + "the live tree must survive a revoked late extraction" ); - - // Observed from an independent session, so the check cannot be satisfied - // by re-entrancy on whichever pooled connection happens to be handed back. - let probe = sibling_pool(&pool, 2); - let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); assert!( - wait_until_free(&probe, key, Duration::from_secs(5)).await, - "a cancelled acquire_write must leave no advisory lock held" + !tmp.exists(), + "the temp extraction must be cleaned up on refusal" ); - - // A subsequent acquire for the SAME repo must succeed promptly. Before the - // fix it blocks on the leaked lock until the 60-attempt ceiling. - let store = RepoStore::for_testing(repos_dir, store_pool); - let guard = tokio::time::timeout(Duration::from_secs(5), store.acquire_write(owner, repo)) - .await - .expect("second acquire_write must not block on a leaked lock") - .expect("second acquire_write must succeed"); - guard.release(false).await; + let head = std::fs::read_to_string(live.join("HEAD")).unwrap(); + assert_eq!(head, "ref: refs/heads/main\n"); } - /// Cancellation BEFORE the lock is taken must leave nothing behind: no lock, - /// and no lock-pool connection stranded. The lock pool here holds exactly one - /// connection, so a stranded one would make the follow-up acquire time out - /// waiting for a checkout. - #[sqlx::test] - async fn cancelled_acquire_write_before_the_lock_leaves_nothing_held(pool: PgPool) { - let probe = sibling_pool(&pool, 2); - let owner = "did:key:z6MkCancelEarly"; - let repo = "cancel-early"; - let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); - - let store = RepoStore::new( - PathBuf::from("/tmp/gitlawb-test-repos"), - None, - build_lock_pool(&pool, 1, Duration::from_secs(3)), - ); - - // A zero deadline polls the future once, which gets it no further than the - // first await (the pool checkout / the first try-lock round trip), so it is - // cancelled before any lock can be taken. - let cancelled = - tokio::time::timeout(Duration::ZERO, store.acquire_write(owner, repo)).await; - assert!(cancelled.is_err(), "the acquire must be cancelled"); + #[test] + fn non_revoked_publish_swap_replaces_the_live_tree() { + let root = tempfile::TempDir::new().unwrap(); + let repos_dir = root.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let owner = "did:key:z6MkHappySwapAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "repo"; + let live = validated_repo_disk_path(&repos_dir, owner, name).unwrap(); + std::fs::create_dir_all(&live).unwrap(); + std::fs::write(live.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + + let tmp = live + .parent() + .unwrap() + .join(format!(".{}.tmp-extract.test", name)); + std::fs::create_dir_all(&tmp).unwrap(); + std::fs::write(tmp.join("HEAD"), "ref: refs/heads/new\n").unwrap(); + + let authority = Arc::new(AtomicBool::new(true)); + swap_extracted_into_validated_repo(&live, &tmp, Some(&authority)).expect("swap succeeds"); + let head = std::fs::read_to_string(live.join("HEAD")).unwrap(); + assert_eq!(head, "ref: refs/heads/new\n"); + } - assert!( - lock_is_free_elsewhere(&probe, key).await, - "no lock may be held when the acquire never got that far" - ); + // ── sync slug validation (#272) ──────────────────────────────────────── - // The single lock-pool connection must be back: if cancellation stranded - // it, this checkout blocks until the 3s acquire timeout and fails. - let guard = tokio::time::timeout(Duration::from_secs(2), store.acquire_write(owner, repo)) - .await - .expect("the lock-pool connection must have been returned") - .expect("acquire after cancellation"); - guard.release(false).await; - } - - /// Lock-pool exhaustion is a bounded wait and a clean error, never a panic and - /// never an unbounded hang. - #[sqlx::test] - async fn lock_pool_exhaustion_is_a_bounded_error(pool: PgPool) { - let owner = "did:key:z6MkExhaustion"; - let store = RepoStore::new( - PathBuf::from("/tmp/gitlawb-test-repos"), - None, - build_lock_pool(&pool, 1, Duration::from_secs(2)), - ); - - let held = store - .acquire_write(owner, "exhaust-a") - .await - .expect("first acquire"); - - // Different repo, so this is not the advisory lock queueing: the only - // connection in the lock pool is checked out by `held`. - let started = std::time::Instant::now(); - let err = tokio::time::timeout( - Duration::from_secs(10), - store.acquire_write(owner, "exhaust-b"), - ) - .await - .expect("the wait must be bounded by the pool acquire timeout"); - let err = match err { - Ok(_) => panic!("an exhausted lock pool must surface an error, not a guard"), - Err(e) => e, - }; - assert!( - started.elapsed() < Duration::from_secs(6), - "the error must arrive on the acquire timeout, not after a long hang" - ); - assert!( - err.to_string().contains("lock-pool connection"), - "the error must name the lock-pool checkout, got: {err}" - ); - - held.release(false).await; - } - - /// #173 F1 (RED-before/GREEN-after). A contended `acquire_write` spins for up to - /// 60 one-second attempts. It must not OCCUPY a lock-pool connection for that whole - /// spin: `acquire_write` has non-push callers (`api/issues.rs`, `api/pulls.rs`) that - /// hold no concurrency permit, so any self-minted did:key could otherwise park a - /// connection per call and starve authenticated pushes on EVERY repo. - /// - /// Lock pool of exactly 2, two spinners. Pre-fix (checkout hoisted above the retry - /// loop) they pin both connections for the full spin and an UNCONTENDED acquire on a - /// third repo dies on the pool acquire timeout. Post-fix each spinner returns its - /// connection before sleeping, so it occupies ~0 and the uncontended acquire sails - /// through. - #[sqlx::test] - async fn a_spinning_acquire_write_does_not_occupy_a_lock_pool_connection(pool: PgPool) { - let owner = "did:key:z6MkSpinOccupancy"; - let owner_slug = owner.replace([':', '/'], "_"); - let store = RepoStore::new( - PathBuf::from("/tmp/gitlawb-test-repos"), - None, - build_lock_pool(&pool, 2, Duration::from_secs(2)), - ); - - // An independent session holds both contended keys, so the spinners' try-locks - // return false on every iteration and they stay in the retry loop. - let holder = sibling_pool(&pool, 2); - let mut held_conn = holder.acquire().await.expect("holder connection"); - for repo in ["spin-a", "spin-b"] { - let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") - .bind(advisory_lock_key(&owner_slug, repo)) - .fetch_one(&mut *held_conn) - .await - .expect("holder try-lock"); - assert!(taken.0, "the holder must own {repo}'s key"); - } - - let mut spinners = Vec::new(); - for repo in ["spin-a", "spin-b"] { - let store = store.clone(); - spinners.push(tokio::spawn(async move { - store.acquire_write(owner, repo).await - })); - } - // Let both reach the spin (each has done at least one failed try-lock by now). - tokio::time::sleep(Duration::from_millis(500)).await; - - let started = std::time::Instant::now(); - let uncontended = tokio::time::timeout( - Duration::from_secs(10), - store.acquire_write(owner, "spin-free"), - ) - .await - .expect("the uncontended acquire must return, not hang"); - let elapsed = started.elapsed(); - let free_guard = uncontended.unwrap_or_else(|e| { - panic!( - "an UNCONTENDED acquire_write on a DIFFERENT repo must not be starved by \ - spinners holding the lock pool; got: {e}" - ) - }); - assert!( - elapsed < Duration::from_secs(2), - "the uncontended acquire must not queue behind the spinners for the pool \ - acquire timeout; took {elapsed:?}" - ); - free_guard.release(false).await; - - // The drop-and-retake cycle must still END in a real, exclusive lock: free - // spin-a's key and the spinner that was cycling connections must take it. - sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(advisory_lock_key(&owner_slug, "spin-a")) - .execute(&mut *held_conn) - .await - .expect("release spin-a"); - let winner = tokio::time::timeout(Duration::from_secs(15), spinners.remove(0)) - .await - .expect("the spinner must finish once its key frees") - .expect("spinner task") - .expect("the spinner must acquire once the key frees"); - let probe = sibling_pool(&pool, 2); - assert!( - !lock_is_free_elsewhere(&probe, advisory_lock_key(&owner_slug, "spin-a")).await, - "the lock a spinner finally took must be observably held from another session" - ); - winner.release(false).await; - - for s in spinners { - s.abort(); - } - sqlx::query("SELECT pg_advisory_unlock_all()") - .execute(&mut *held_conn) - .await - .expect("release the remaining holder lock"); - } - - /// #173 F1, the property the fix rests on: returning a lock-pool connection that - /// holds NOTHING runs `after_release`'s `pg_advisory_unlock_all()`, which is a no-op - /// and must not disturb a lock held on a DIFFERENT connection of the same pool. - /// Session advisory locks are per connection, so this is by construction, but the - /// spin fix depends on it, so it is proven by execution rather than assumed. - #[sqlx::test] - async fn returning_an_unlocked_connection_does_not_clear_another_connections_lock( - pool: PgPool, - ) { - let owner = "did:key:z6MkNoOpUnlockAll"; - let repo = "noop-unlock"; - let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); - let probe = sibling_pool(&pool, 2); - let lock_pool = build_lock_pool(&pool, 4, Duration::from_secs(5)); - let store = RepoStore::new( - PathBuf::from("/tmp/gitlawb-test-repos"), - None, - lock_pool.clone(), - ); - - let guard = store.acquire_write(owner, repo).await.expect("acquire"); - - // Churn the pool: check out and drop connections that hold no lock, exactly what - // a spinning acquire now does between attempts. Each return fires - // pg_advisory_unlock_all() on that connection. - for _ in 0..10 { - let mut conn = lock_pool.acquire().await.expect("churn checkout"); - let _: (i32,) = sqlx::query_as("SELECT 1") - .fetch_one(&mut *conn) - .await - .expect("churn query"); - drop(conn); - tokio::time::sleep(Duration::from_millis(10)).await; - } - - assert!( - !lock_is_free_elsewhere(&probe, key).await, - "a held write lock must survive other lock-pool connections being returned" - ); - guard.release(true).await; - assert!( - lock_is_free_elsewhere(&probe, key).await, - "release must still free the lock after the churn" - ); - } - - /// #173 F1: lock-pool exhaustion is a DISTINCT error the handler can shed as a 503, - /// not a generic git 500. Both directions: an exhausted pool downcasts to - /// [`LockPoolBusy`], and an unrelated failure (a rejected repo name) does not. - #[sqlx::test] - async fn lock_pool_exhaustion_is_a_distinct_downcastable_error(pool: PgPool) { - let owner = "did:key:z6MkBusyDowncast"; - let store = RepoStore::new( - PathBuf::from("/tmp/gitlawb-test-repos"), - None, - build_lock_pool(&pool, 1, Duration::from_secs(1)), - ); - let held = store - .acquire_write(owner, "busy-a") - .await - .expect("first acquire"); - - let err = match store.acquire_write(owner, "busy-b").await { - Ok(_) => panic!("an exhausted lock pool must error, not hand back a guard"), - Err(e) => e, - }; - assert!( - err.downcast_ref::().is_some(), - "lock-pool exhaustion must be downcastable so the handler sheds 503, got: {err}" - ); - - // MUST-NOT: an ordinary rejection is not a capacity signal. - let other = match store.acquire_write(owner, "../escape").await { - Ok(_) => panic!("a traversal repo name must be rejected"), - Err(e) => e, - }; - assert!( - other.downcast_ref::().is_none(), - "a validation failure must not masquerade as lock-pool capacity, got: {other}" - ); - - held.release(false).await; - } - - /// Round trip: the lock is observably HELD between acquire and release, and - /// observably FREE after. Both checks run from an independent session; from - /// the holding session they would pass vacuously (session locks are - /// re-entrant) and would not notice an unlock that landed on the wrong - /// connection. - #[sqlx::test] - async fn acquire_write_holds_the_lock_until_release(pool: PgPool) { - let probe = sibling_pool(&pool, 2); - let owner = "did:key:z6MkRoundTrip"; - let repo = "round-trip"; - let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); - - let store = RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-test-repos"), pool.clone()); - let guard = store.acquire_write(owner, repo).await.expect("acquire"); - assert!( - !lock_is_free_elsewhere(&probe, key).await, - "the lock must be held while the guard is alive" - ); - - // No polling here, deliberately. `release` must free the lock SYNCHRONOUSLY, - // which it can only do by unlocking on the connection that took it; a - // `pg_advisory_unlock` sent through the pool would land on some other - // session and return false. The `after_release` hook is a net for the - // cancellation path and fires from a spawned task well after this point, so - // it must not be what makes this assertion pass. - guard.release(true).await; - assert!( - lock_is_free_elsewhere(&probe, key).await, - "release must free the lock as seen from another session" - ); - } - - /// `release(false)` skips the Tigris upload but must still free the lock; a - /// failed write that kept the lock would wedge the repo. - #[sqlx::test] - async fn release_after_failed_write_still_frees_the_lock(pool: PgPool) { - let probe = sibling_pool(&pool, 2); - let owner = "did:key:z6MkFailedWrite"; - let repo = "failed-write"; - let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); - - let store = RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-test-repos"), pool.clone()); - let guard = store.acquire_write(owner, repo).await.expect("acquire"); - guard.release(false).await; - - assert!( - wait_until_free(&probe, key, Duration::from_secs(5)).await, - "release(success = false) must still free the lock" - ); - } - - /// The lock is per repo: a second acquire for the SAME repo waits for the - /// first to release, while a different repo proceeds straight through. - #[sqlx::test] - async fn same_repo_acquires_serialize_and_different_repos_do_not(pool: PgPool) { - let repos_dir = PathBuf::from("/tmp/gitlawb-test-repos"); - let owner = "did:key:z6MkSerialize"; - let store = RepoStore::for_testing(repos_dir, pool.clone()); - - let first = store - .acquire_write(owner, "serialize-a") - .await - .expect("first acquire"); - - // Different repo: unaffected by the held lock. - let other = tokio::time::timeout( - Duration::from_secs(2), - store.acquire_write(owner, "serialize-b"), - ) - .await - .expect("a different repo must not wait on this lock") - .expect("acquire other repo"); - other.release(false).await; - - // Same repo: must not acquire while `first` is alive. - let contender = tokio::spawn({ - let store = store.clone(); - async move { store.acquire_write(owner, "serialize-a").await } - }); - tokio::time::sleep(Duration::from_millis(1500)).await; - assert!( - !contender.is_finished(), - "a second acquire for the same repo must block while the first guard lives" - ); - - first.release(false).await; - let second = tokio::time::timeout(Duration::from_secs(10), contender) - .await - .expect("contender must finish once the lock is free") - .expect("contender task") - .expect("contender acquire"); - second.release(false).await; - } - - // ── sync slug validation (#272) ──────────────────────────────────────── - - #[test] - fn slug_accepts_owner_and_name() { - let (owner, name) = validate_repo_slug("z6Mkfoo/hello").expect("valid slug"); - assert_eq!(owner, "z6Mkfoo"); - assert_eq!(name, "hello"); + #[test] + fn slug_accepts_owner_and_name() { + let (owner, name) = validate_repo_slug("z6Mkfoo/hello").expect("valid slug"); + assert_eq!(owner, "z6Mkfoo"); + assert_eq!(name, "hello"); } #[test] @@ -1790,7 +2723,12 @@ mod tests { // the pool or the network. Fabricate a pool reference via PgPool::connect_lazy // so we don't need a live DB. let pool = sqlx::PgPool::connect_lazy("postgres://invalid").unwrap(); - RepoStore::new(PathBuf::from("/var/lib/gitlawb/repos"), None, pool) + RepoStore::new( + PathBuf::from("/var/lib/gitlawb/repos"), + None, + pool, + Duration::from_secs(300), + ) } #[tokio::test] @@ -1934,7 +2872,7 @@ mod tests { let mut checker = pool.acquire().await.expect("checker connection"); let guard = store.acquire_write(owner, name).await.expect("acquire"); - guard.release(false).await; + let _ = guard.release(false).await; let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") .bind(key) @@ -1954,17 +2892,16 @@ mod tests { // ── cancellation-safe unlock (#174 F4, RED-before/GREEN-after) ────────── /// F4 (P1): a cancellation DURING the unlock await must still free the session - /// advisory lock. The guard owns the lock-pool connection that took the lock, so - /// dropping the parked `release` future returns that connection to the pool, - /// where the `after_release` hook runs `pg_advisory_unlock_all()` and clears - /// whatever the interrupted unlock did not. A test-only gate parks `release` at - /// the exact pre-unlock point; dropping the future there reproduces the - /// cancellation. + /// advisory lock. `release` unlocks through the connection while `self` still + /// owns it, so if the future is dropped mid-unlock, `Drop` sees `conn == Some` + /// + `locked && !released` and runs its detached-unlock backstop. A test-only + /// gate parks `release` at the exact pre-unlock point; dropping the future + /// there reproduces the cancellation. /// - /// Load-bearing: build the store's lock pool WITHOUT the `after_release` hook - /// and this goes RED, since the connection then returns to the pool still - /// holding the session lock and the checker's `pg_try_advisory_lock` returns - /// false. + /// Load-bearing: RED on the original ordering (`self.conn.take()` before the + /// await → at cancellation `self.conn == None` → `Drop` skips → the local + /// connection returns to the pool with the session lock still held → the + /// checker's `pg_try_advisory_lock` returns false). GREEN after the reorder. #[sqlx::test] async fn write_guard_release_cancelled_mid_unlock_frees_the_lock(pool: sqlx::PgPool) { let dir = tempfile::TempDir::new().unwrap(); @@ -2027,7 +2964,7 @@ mod tests { .acquire_write(owner, name) .await .expect("first acquire"); - guard.release(true).await; + let _ = guard.release(true).await; let again = tokio::time::timeout( std::time::Duration::from_secs(2), @@ -2036,7 +2973,7 @@ mod tests { .await .expect("second acquire_write must not hit the ~60s stale-lock retry loop") .expect("second acquire"); - again.release(true).await; + let _ = again.release(true).await; } // ── unlock error disposes the connection (#174 F3b, RED-before/GREEN-after) ─ @@ -2050,7 +2987,7 @@ mod tests { /// in this module and can reach `conn` directly. async fn poison_guard_connection(guard: &mut RepoWriteGuard) { let conn = guard - .lock_conn + .conn .as_deref_mut() .expect("guard holds its connection before release"); sqlx::query("BEGIN") @@ -2106,101 +3043,6 @@ mod tests { } } - /// F3c (P2): the connection teardown on the failing-unlock path must be BOUNDED. - /// `release` awaits it inline while the global write permit, the per-source permit - /// and the write lease are all still held, and sqlx's `close()` carries no deadline - /// of its own, so a blackholed socket would park every later push to that repo - /// behind three pinned admission resources. - /// - /// What this covers: the deadline itself. A close that never resolves still lets - /// `close_conn_bounded` return, which is the property `release` depends on. What it - /// does NOT cover, and is reasoned rather than run: that sqlx's own `close()` is - /// what stalls in production. Making a real `PgConnection::close` hang needs a - /// blackholed TCP path to Postgres, and the flip has to land after the unlock - /// statement round-trips but before the Terminate write, which is not a seam this - /// module exposes. A never-resolving future is the faithful stand-in for that - /// close, and the F3b tests above already cover that `release` really routes its - /// close through here. - /// - /// Time is paused, so nothing here depends on wall clock: the runtime auto-advances - /// to the next timer, and the assertion is on which timer fired, not on elapsed - /// time. The outer bound is what turns a removed deadline into a failure rather - /// than a hung suite. - /// - /// Load-bearing: drop the `tokio::time::timeout` in `close_conn_bounded` and the - /// inner future never resolves, so the outer bound fires and this fails. - #[tokio::test(start_paused = true)] - async fn unlock_error_connection_close_is_bounded() { - let hanging = std::future::pending::>(); - let outcome = tokio::time::timeout( - UNLOCK_ERROR_CLOSE_TIMEOUT * 4, - close_conn_bounded("boundedclosetest", hanging), - ) - .await; - assert!( - outcome.is_ok(), - "a connection close that never completes must not hold the write lease and \ - both admission permits open-endedly: close_conn_bounded must give up and \ - drop the connection" - ); - } - - /// U8, the off-runtime arm: with no Tokio runtime there is nothing to spawn the - /// unlock onto, and the connection has already been taken out of the guard, so - /// dropping it with no unlock attempted returns it to the pool with the session - /// lock still held. Dropping a `PoolConnection` off a runtime is worse than that: - /// sqlx's return-to-pool path spawns, and its no-runtime fallback panics, so that - /// arm also panics in a destructor. - /// - /// Reached by dropping the guard on a plain `std::thread`, where - /// `Handle::try_current()` fails. - /// - /// Load-bearing: replace the `detach` arm with a plain `drop(conn)` and the join - /// sees sqlx's "requires a Tokio context" panic; `detach` gives up the pool slot, - /// so nothing is spawned and dropping the detached connection closes the socket, - /// which ends the session and frees the lock. - #[sqlx::test] - async fn write_guard_dropped_off_runtime_disposes_the_connection(pool: sqlx::PgPool) { - let dir = tempfile::TempDir::new().unwrap(); - let store_pool = pool_without_idle_reaper(&pool).await; - let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); - let owner = "did:key:z6MkDropOffRuntimeProofKKKKKKKKKKKKKKKKKK"; - let name = "dropoffruntimetest"; - let slug = owner.replace([':', '/'], "_"); - let key = advisory_lock_key(&slug, name); - - let mut checker = pool.acquire().await.expect("checker connection"); - let guard = store.acquire_write(owner, name).await.expect("acquire"); - // The guard's connection lives in the store's DERIVED lock pool, not the pool - // handed to `for_testing`; see `RepoStore::lock_pool`. - let lock_pool = store.lock_pool().clone(); - let size_before = lock_pool.size(); - assert!(size_before > 0, "the lock pool owns the guard's connection"); - - let dropped = std::thread::spawn(move || drop(guard)).join(); - assert!( - dropped.is_ok(), - "dropping a write guard off a Tokio runtime must not panic" - ); - - wait_until( - || lock_pool.size() == size_before - 1, - "the connection of a guard dropped off a runtime to be disposed of rather \ - than returned to the pool with no unlock attempted", - ) - .await; - wait_until_lock_free( - &mut checker, - key, - "a guard dropped off a runtime to end its session so postgres drops the lock", - ) - .await; - let _ = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(key) - .execute(&mut *checker) - .await; - } - /// A second pool over the same test database with the idle reaper DISABLED. /// /// `#[sqlx::test]`'s own pool sets `idle_timeout(1s)`, so a connection returned to @@ -2264,7 +3106,7 @@ mod tests { "the poisoned session must still hold the lock before release" ); - guard.release(false).await; + let _ = guard.release(false).await; // Postgres drops the lock when the disposed session's backend exits, which is // asynchronous to our socket close: poll for it rather than sleeping a @@ -2299,16 +3141,15 @@ mod tests { let mut guard = store.acquire_write(owner, name).await.expect("acquire"); poison_guard_connection(&mut guard).await; - let lock_pool = store.lock_pool().clone(); - let size_before = lock_pool.size(); - assert!(size_before > 0, "the lock pool owns the guard's connection"); + let size_before = store_pool.size(); + assert!(size_before > 0, "the pool owns the guard's connection"); - guard.release(false).await; + let _ = guard.release(false).await; // The pool's size drops when the closed connection's slot is given up, which // is not synchronous with `release` returning: poll rather than sleep. wait_until( - || lock_pool.size() == size_before - 1, + || store_pool.size() == size_before - 1, "the connection that saw the unlock error to be closed rather than returned \ to the pool still holding the session lock", ) @@ -2333,7 +3174,7 @@ mod tests { let guard = store.acquire_write(owner, name).await.expect("acquire"); let size_before = pool.size(); - guard.release(false).await; + let _ = guard.release(false).await; tokio::time::sleep(std::time::Duration::from_millis(400)).await; assert_eq!( @@ -2386,15 +3227,14 @@ mod tests { let mut guard = store.acquire_write(owner, name).await.expect("acquire"); poison_guard_connection(&mut guard).await; - let lock_pool = store.lock_pool().clone(); - let size_before = lock_pool.size(); - assert!(size_before > 0, "the lock pool owns the guard's connection"); + let size_before = store_pool.size(); + assert!(size_before > 0, "the pool owns the guard's connection"); // The backstop shape: dropped without release(), with an unlock that errors. drop(guard); wait_until( - || lock_pool.size() == size_before - 1, + || store_pool.size() == size_before - 1, "the connection whose detached unlock errored to be closed rather than \ returned to the pool still holding the session lock", ) @@ -2412,42 +3252,60 @@ mod tests { .await; } - /// U8 regression guard on the success path: a detached unlock that SUCCEEDS must - /// still return the connection to the pool. Without this, "close the connection on - /// Drop" could be widened to "always close" and the test above would not notice. + /// U8, the off-runtime arm: with no Tokio runtime there is nothing to spawn the + /// unlock onto, and the connection has already been taken out of the guard, so the + /// old code dropped it with no unlock attempted at all, back to the pool, session + /// lock still held. Dropping a `PoolConnection` off a runtime is worse than that: + /// sqlx's return-to-pool path spawns, and its no-runtime fallback panics, so the + /// old arm also panicked in a destructor. + /// + /// Reached by dropping the guard on a plain `std::thread`, where + /// `Handle::try_current()` fails. + /// + /// Load-bearing: RED before the fix (the join sees the "requires a Tokio context" + /// panic from sqlx's return-to-pool spawn), GREEN after (`leak` gives up the + /// pool slot, so nothing is spawned and dropping the leaked connection closes + /// the socket, which ends the session and frees the lock). + /// + /// `leak`, not `detach`: the branch's off-runtime arm deliberately leaks the + /// slot (permanently checked out, `size()` unchanged) rather than detaching + /// (which lets the pool open a replacement), because at process-teardown time + /// there is no runtime to service the replacement's connect. The observable + /// is `num_idle()`: the leaked slot never returns to idle. #[sqlx::test] - async fn write_guard_drop_with_successful_unlock_keeps_the_connection(pool: sqlx::PgPool) { + async fn write_guard_dropped_off_runtime_disposes_the_connection(pool: sqlx::PgPool) { let dir = tempfile::TempDir::new().unwrap(); let store_pool = pool_without_idle_reaper(&pool).await; let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); - let owner = "did:key:z6MkDropUnlockOkProofJJJJJJJJJJJJJJJJJJJJ"; - let name = "dropunlockoktest"; + let owner = "did:key:z6MkDropOffRuntimeProofKKKKKKKKKKKKKKKKKK"; + let name = "dropoffruntimetest"; let slug = owner.replace([':', '/'], "_"); let key = advisory_lock_key(&slug, name); let mut checker = pool.acquire().await.expect("checker connection"); let guard = store.acquire_write(owner, name).await.expect("acquire"); - let lock_pool = store.lock_pool().clone(); - let size_before = lock_pool.size(); - assert!(size_before > 0, "the lock pool owns the guard's connection"); + let size_before = store_pool.size(); + assert!(size_before > 0, "the pool owns the guard's connection"); - drop(guard); + let dropped = std::thread::spawn(move || drop(guard)).join(); + assert!( + dropped.is_ok(), + "dropping a write guard off a Tokio runtime must not panic" + ); - // The connection goes back only once the detached unlock task has finished. + // The disposed connection must NOT come back to the pool as idle: that is + // the leak-vs-return distinction that made the old code return a session + // still holding the lock. `leak` keeps the slot permanently checked out, so + // `num_idle` cannot rise here. wait_until( - || lock_pool.num_idle() > 0, - "the detached unlock to finish and hand the connection back", + || store_pool.num_idle() == 0, + "the leaked connection to never return to the pool's idle set", ) .await; - assert_eq!( - lock_pool.size(), - size_before, - "a successful detached unlock must leave the connection in the pool" - ); wait_until_lock_free( &mut checker, key, - "the Drop backstop's successful unlock to free the lock", + "a guard dropped off a runtime to end its session so postgres drops the lock", ) .await; let _ = sqlx::query("SELECT pg_advisory_unlock($1)") @@ -2455,4 +3313,4045 @@ mod tests { .execute(&mut *checker) .await; } + + /// F4: releasing a guard that never took the lock (`locked == false`, the state + /// acquire_write leaves after a failed acquisition) must not unlock or panic. + #[sqlx::test] + async fn write_guard_release_when_not_locked_does_not_unlock_or_panic(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let owner = "did:key:z6MkNotLockedProofEEEEEEEEEEEEEEEEEEEEEE"; + let name = "notlockedtest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let guard = RepoWriteGuard { + owner_slug: slug, + repo_name: name.to_string(), + local_path: validated_repo_disk_path(dir.path(), owner, name).expect("test path"), + lock_key: key, + conn: Some(pool.acquire().await.expect("conn")), + tigris: None, + lock_held_transfer_timeout: Duration::from_secs(300), + publish_fence: UploadPrecondition::Unconditional, + refresh_swap_authority: None, + publish_stage: Arc::new(PublishStageCell::new()), + #[cfg(test)] + test_pre_unlock_gate: None, + #[cfg(test)] + upload_site_reached: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + }; + // Must complete without panic and issue no unlock. + let _ = guard.release(false).await; + + let mut checker = pool.acquire().await.expect("checker"); + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!(free, "release on an unlocked guard must not touch the key"); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + // ── U1: cancellation-safe lock probe ─────────────────────────────────── + + /// A pool with every reaping path disabled, so a leaked lock persists through + /// the observation window instead of being freed by ambient recycling. + async fn no_reap_pool(opts: &sqlx::postgres::PgConnectOptions, max: u32) -> PgPool { + sqlx::postgres::PgPoolOptions::new() + .max_connections(max) + .acquire_timeout(std::time::Duration::from_secs(5)) + .min_connections(0) + .idle_timeout(None) + .max_lifetime(None) + .test_before_acquire(false) + .connect_with(opts.clone()) + .await + .expect("no-reap pool") + } + + /// Poll a STANDALONE connection until the key is free, or the deadline passes. + /// + /// Standalone, never from the pool under test: pool reuse would hand the + /// observer the lock-holding session itself, where `pg_try_advisory_lock` + /// succeeds reentrantly and hides the very leak being measured. Polling rather + /// than asserting once because `PoolConnection::drop` spawns the close. + async fn poll_until_free( + opts: &sqlx::postgres::PgConnectOptions, + key: i64, + deadline: std::time::Duration, + ) -> bool { + use sqlx::Connection; + let start = std::time::Instant::now(); + let mut observer = sqlx::PgConnection::connect_with(opts) + .await + .expect("standalone observer connection"); + while start.elapsed() < deadline { + let got: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut observer) + .await + .expect("observer try-lock"); + if got.0 { + let _: (bool,) = sqlx::query_as("SELECT pg_advisory_unlock($1)") + .bind(key) + .fetch_one(&mut observer) + .await + .expect("observer unlock"); + return true; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + false + } + + /// THE COMMITTED GATE for the cancellation window (U1). + /// + /// Dropping the probe without taking its connection is exactly the state a + /// cancellation between the try-lock's send and the guard's construction + /// leaves behind. Deterministic on purpose: the timing sweep that first found + /// this window leaks about 1 in 600, which is not a signal a CI gate can rest + /// on. That sweep stays a local repro. + #[sqlx::test] + async fn lock_probe_dropped_without_taking_frees_the_lock(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 4).await; + let key: i64 = 990_001; + + { + let mut probe = LockProbe::new(lock_pool.acquire().await.unwrap()); + assert!( + probe.try_lock(key).await.unwrap(), + "probe should take a free key" + ); + // dropped here WITHOUT take_conn(): the cancellation shape + } + + assert!( + poll_until_free(&opts, key, std::time::Duration::from_secs(10)).await, + "lock must be freed after a probe is dropped without taking its connection" + ); + } + + /// Must-not: a successful acquire hands the connection out intact, so the + /// normal path does not pay a reconnect per write. + #[sqlx::test] + async fn lock_probe_take_conn_yields_a_usable_connection(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 4).await; + let key: i64 = 990_002; + + let mut probe = LockProbe::new(lock_pool.acquire().await.unwrap()); + assert!(probe.try_lock(key).await.unwrap()); + let mut conn = probe + .take_conn() + .expect("connection after a successful acquire"); + drop(probe); + + let one: (i32,) = sqlx::query_as("SELECT 1") + .fetch_one(&mut *conn) + .await + .expect("handed-out connection must still be usable"); + assert_eq!(one.0, 1); + + let released: (bool,) = sqlx::query_as("SELECT pg_advisory_unlock($1)") + .bind(key) + .fetch_one(&mut *conn) + .await + .unwrap(); + assert!(released.0, "the handed-out connection still owns the lock"); + } + + /// Must-not: a failed probe returns its connection without closing it. Nothing + /// was locked, so closing would be pure churn, and closing on every failed + /// probe would make a 60-attempt spinner tear down 60 backends. + #[sqlx::test] + async fn lock_probe_failed_acquire_does_not_hold_anything(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 4).await; + let key: i64 = 990_003; + + // a standalone holder takes the key first + use sqlx::Connection; + let mut holder = sqlx::PgConnection::connect_with(&opts).await.unwrap(); + let held: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut holder) + .await + .unwrap(); + assert!(held.0); + + { + let mut probe = LockProbe::new(lock_pool.acquire().await.unwrap()); + assert!( + !probe.try_lock(key).await.unwrap(), + "probe must observe false for a key held elsewhere" + ); + } + + // the holder still owns it: the failed probe neither took nor released it + let still: (i64,) = sqlx::query_as( + "SELECT count(*) FROM pg_locks WHERE locktype='advisory' \ + AND ((classid::bigint<<32)|objid::bigint) = $1", + ) + .bind(key) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(still.0, 1, "the original holder must still own the key"); + } + + // ── U3: the #279 acceptance tests ────────────────────────────────────── + + /// The store under test. Pre-U3 this ignores `opts` and shares the app pool, + /// which is exactly the broken shape; the wiring change swaps in a dedicated + /// no-reap lock pool without touching a single test body below. + async fn write_store(pool: &PgPool, opts: &sqlx::postgres::PgConnectOptions) -> RepoStore { + let _ = pool; + RepoStore::for_testing( + PathBuf::from("/tmp/gitlawb-u3"), + no_reap_pool(opts, 8).await, + ) + } + + fn advisory_locks_held(key: i64) -> String { + format!( + "SELECT count(*) FROM pg_locks WHERE locktype='advisory' \ + AND ((classid::bigint<<32)|objid::bigint) = {key}" + ) + } + + /// ACCEPTANCE 1 (#279): two writers on one node and the same repo must not + /// both hold the lock. On the pre-fix shape the second acquire succeeds + /// because the pool hands it the very session holding the lock, where + /// pg_try_advisory_lock is reentrant. + #[sqlx::test] + async fn two_writers_on_the_same_repo_are_not_both_admitted(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let store = write_store(&pool, &opts) + .await + .with_lock_acquire_deadline(std::time::Duration::from_millis(300)); + + let _first = store + .acquire_write("did:key:z6MkU3Excl", "same-repo") + .await + .expect("first writer acquires"); + + let err = match store.acquire_write("did:key:z6MkU3Excl", "same-repo").await { + Err(e) => e, + Ok(second) => { + let _ = second.release(false).await; + panic!("a second writer must NOT be admitted while the first holds the guard"); + } + }; + assert!( + err.downcast_ref::().is_some(), + "the second writer must be shed as RepoBusy, got {err:#}" + ); + } + + /// ACCEPTANCE 2 (#279): a completed write leaves no advisory lock behind. + /// On the pre-fix shape the unlock runs on a different pooled session and + /// returns false, so the lock leaks on essentially every write. + #[sqlx::test] + async fn completed_write_releases_its_advisory_lock(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let store = write_store(&pool, &opts).await; + + let guard = store + .acquire_write("did:key:z6MkU3Rel", "leak-check") + .await + .expect("acquire"); + let _ = guard.release(true).await; + + let key = advisory_lock_key("did_key_z6MkU3Rel", "leak-check"); + let held: (i64,) = sqlx::query_as(&advisory_locks_held(key)) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + held.0, 0, + "a completed write must leave zero advisory locks for its key" + ); + } + + // ── U4: a guard that dies without releasing must free the lock ────────── + + /// A guard dropped without `release()` (an early `?`, a panic, or a handler + /// future cancelled on client disconnect) must not return a lock-bearing + /// connection to the pool, where it would block every future write to that + /// repo until sqlx recycles the session. + #[sqlx::test] + async fn guard_dropped_without_release_frees_the_lock(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let store = write_store(&pool, &opts).await; + let key = advisory_lock_key("did_key_z6MkU4Drop", "dropped"); + + { + let _guard = store + .acquire_write("did:key:z6MkU4Drop", "dropped") + .await + .expect("acquire"); + // dropped here without release() + } + + assert!( + poll_until_free(&opts, key, std::time::Duration::from_secs(10)).await, + "lock must be freed when a guard is dropped without release()" + ); + } + + /// Must-not over-close: the normal path returns its connection to the pool, so + /// a healthy write does not pay a reconnect. Sized to one connection so the + /// backend pid is a direct observable: if `release` were closing the session, + /// each cycle would land on a fresh backend. + #[sqlx::test] + async fn normal_release_reuses_the_same_backend(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let store = RepoStore::for_testing( + PathBuf::from("/tmp/gitlawb-u4"), + no_reap_pool(&opts, 1).await, + ); + + let mut pids = Vec::new(); + for i in 0..4 { + let repo = format!("reuse-{i}"); + let mut guard = store + .acquire_write("did:key:z6MkU4Reuse", &repo) + .await + .expect("acquire"); + pids.push(guard.backend_pid_for_test().await); + let _ = guard.release(true).await; + } + assert!( + pids.windows(2).all(|w| w[0] == w[1]), + "a released guard must return its connection to the pool, so all four \ + writes share one backend; saw {pids:?}" + ); + } + + /// A guard abandoned while the runtime is tearing down must not panic. + /// `PoolConnection::drop` calls `crate::rt::spawn`, which panics without a + /// runtime handle, and a panic inside `Drop` during unwind aborts the process. + /// At real process exit the lock is freed by socket teardown, not by this Drop + /// body, so this asserts no-panic rather than lock release. + #[test] + fn guard_dropped_at_runtime_teardown_does_not_panic() { + // No silent skip: a test that returns green when its precondition is + // absent is worse than one that fails, because it reports coverage it does + // not have. CI provisions Postgres, so an absent DATABASE_URL is a broken + // environment rather than an expected one. + let url = std::env::var("DATABASE_URL") + .expect("DATABASE_URL must be set; this test cannot pass vacuously"); + let rt = tokio::runtime::Runtime::new().unwrap(); + let guard = rt.block_on(async { + let lock_pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(2) + .connect(&url) + .await + .expect("lock pool"); + let store = RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-u4b"), lock_pool); + store + .acquire_write("did:key:z6MkU4Teardown", "teardown") + .await + .expect("acquire") + }); + // Shut the runtime down first, then drop the guard with no runtime alive. + drop(rt); + drop(guard); + } + + // ── U5: the unlock's boolean result must be observed ──────────────────── + + /// `pg_advisory_unlock` reports "you did not hold this lock" as a `false` + /// RETURN VALUE plus a server WARNING, never an error, so a discarded result + /// cannot tell a real release from a no-op. A session that did not hold the + /// key must not be returned to the pool as if it were clean. + /// + /// The observable is the backend pid: on a one-connection pool, a session that + /// was closed forces the next acquire onto a fresh backend, while one returned + /// normally is handed straight back. + #[sqlx::test] + async fn release_that_did_not_hold_the_lock_closes_the_session(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 1).await; + + let pid_before = { + let mut c = lock_pool.acquire().await.unwrap(); + let pid: (i32,) = sqlx::query_as("SELECT pg_backend_pid()") + .fetch_one(&mut *c) + .await + .unwrap(); + pid.0 + }; + + let dir = tempfile::TempDir::new().unwrap(); + // A guard whose key was never locked: release()'s unlock returns false. + let guard = RepoWriteGuard { + owner_slug: "did_key_z6MkU5".to_string(), + repo_name: "never-locked".to_string(), + local_path: validated_repo_disk_path(dir.path(), "did:key:z6MkU5", "never-locked") + .expect("test path"), + lock_key: 995_001, + conn: Some(lock_pool.acquire().await.unwrap()), + tigris: None, + lock_held_transfer_timeout: Duration::from_secs(300), + // No backend, so nothing is ever published and the fence is unread. + publish_fence: UploadPrecondition::Unconditional, + refresh_swap_authority: None, + publish_stage: Arc::new(PublishStageCell::new()), + #[cfg(test)] + test_pre_unlock_gate: None, + #[cfg(test)] + upload_site_reached: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + }; + let _ = guard.release(true).await; + + // Wait for the backend to actually go away rather than sleeping a fixed + // span, which is flaky on slow CI. The observer is a STANDALONE + // connection for the same reason `poll_until_free` uses one: taking it + // from the pool under test would hand us the very session being measured. + // Nothing but the close under test can retire that backend, because + // `no_reap_pool` disables idle timeout and max lifetime, so a zero count + // here is attributable to `release()` and to nothing else. + { + use sqlx::Connection; + let deadline = std::time::Duration::from_secs(5); + let start = std::time::Instant::now(); + let mut observer = sqlx::PgConnection::connect_with(&opts) + .await + .expect("standalone observer connection"); + while start.elapsed() < deadline { + let alive: (i64,) = + sqlx::query_as("SELECT count(*) FROM pg_stat_activity WHERE pid = $1") + .bind(pid_before) + .fetch_one(&mut observer) + .await + .expect("observer pg_stat_activity probe"); + if alive.0 == 0 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + } + + let pid_after = { + let mut c = lock_pool.acquire().await.unwrap(); + let pid: (i32,) = sqlx::query_as("SELECT pg_backend_pid()") + .fetch_one(&mut *c) + .await + .unwrap(); + pid.0 + }; + + assert_ne!( + pid_before, pid_after, + "an unlock that returned false means the session's lock state is not \ + what we think it is; that connection must be closed, not pooled" + ); + } + + // ── U6: under-lock transfers are bounded ──────────────────────────────── + + /// The bound itself. Driving a real stalled transfer through `acquire_write` + /// would need either the object-store abstraction (out of scope here) or a + /// process-global `AWS_ENDPOINT_URL_S3` mutation, which would make the suite + /// order-dependent under the concurrent test runner. So this covers the + /// mechanism deterministically and the wiring is verified by reading, which is + /// recorded as a coverage gap rather than papered over. + #[tokio::test] + async fn bounded_transfer_gives_up_past_the_limit() { + let slow = async { + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + Ok::<(), anyhow::Error>(()) + }; + let out = + bounded_transfer("test", "repo", std::time::Duration::from_millis(50), slow).await; + assert!( + out.is_none(), + "a transfer past its limit must report None so the caller stops holding the lock" + ); + } + + /// Must-not: a transfer that finishes inside the limit is returned intact and + /// is not truncated by the bound. + #[tokio::test] + async fn bounded_transfer_passes_through_a_prompt_result() { + let quick = async { Ok::(7) }; + let out = bounded_transfer("test", "repo", std::time::Duration::from_secs(30), quick).await; + assert!( + matches!(out, Some(Ok(7))), + "a prompt transfer must pass through untouched" + ); + } + + /// F2 regression: an ordinary failed probe must RETURN its connection, not + /// close it. The old test asserted the holder's pg_locks count, which cannot + /// see what happened to the probe's own connection — so it passed while a + /// 60-attempt spinner tore down 60 backends. The observable that discriminates + /// is the backend pid on a one-connection pool. + #[sqlx::test] + async fn failed_probe_returns_its_connection_to_the_pool(pool: PgPool) { + use sqlx::Connection; + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 1).await; + let key: i64 = 991_100; + + // someone else holds the key, from an independent session + let mut holder = sqlx::PgConnection::connect_with(&opts).await.unwrap(); + let held: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut holder) + .await + .unwrap(); + assert!(held.0); + + let mut pids = Vec::new(); + for _ in 0..3 { + let mut probe = LockProbe::new(lock_pool.acquire().await.unwrap()); + let pid: (i32,) = sqlx::query_as("SELECT pg_backend_pid()") + .fetch_one(&mut **probe.conn.as_mut().unwrap()) + .await + .unwrap(); + pids.push(pid.0); + assert!(!probe.try_lock(key).await.unwrap(), "key is held elsewhere"); + drop(probe); + tokio::time::sleep(std::time::Duration::from_millis(150)).await; + } + assert!( + pids.windows(2).all(|w| w[0] == w[1]), + "a failed probe must return its connection so a spinner does not churn \ + backends; saw {pids:?}" + ); + } + + // ── F5: the tests the plan required and the first pass never wrote ──────── + + /// R4, the test the plan named as proving the pool split and the one that would + /// have caught PR #215's node-wide two-write ceiling. Holding N guards on + /// DISTINCT repos must pin N lock-pool connections while leaving the app pool + /// free to serve ordinary queries. + #[sqlx::test] + async fn lock_pool_exhaustion_does_not_starve_the_app_pool(pool: PgPool) { + const N: u32 = 3; + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, N).await; + let store = RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-f5"), lock_pool.clone()); + + let mut guards = Vec::new(); + for i in 0..N { + guards.push( + store + .acquire_write(&format!("did:key:z6MkF5Iso{i}"), "iso") + .await + .expect("distinct repos each acquire"), + ); + } + + // Every slot is accounted for by a guard, so the pool really is exhausted + // rather than merely slow. Asserted directly, because the starvation check + // below cannot tell the two apart on its own. + assert_eq!( + lock_pool.size() as usize - lock_pool.num_idle(), + N as usize, + "all N slots must be checked out by the guards" + ); + + // An N+1th checkout must be refused BY THE POOL. The specific error matters: + // `Ok(Err(_)) | Err(_)` would also be satisfied by the outer tokio timeout + // firing for an unrelated reason, which would let this pass without the pool + // ever having refused anything. + let starved = + tokio::time::timeout(std::time::Duration::from_secs(8), lock_pool.acquire()).await; + match starved { + Ok(Err(sqlx::Error::PoolTimedOut)) => {} + Ok(Err(e)) => panic!("expected the pool's own timeout, got {e:?}"), + Ok(Ok(_)) => panic!("with N guards held, an N+1th lock-pool checkout must not succeed"), + Err(_) => panic!( + "the pool must refuse the checkout itself within its acquire_timeout; \ + the outer timeout firing means it never did" + ), + } + + // ...while the APP pool still serves queries. This is the whole point of + // the split: write pressure must not deny ordinary reads. Weak on its own (it + // is a different pool object, so it would serve regardless), so it is the + // exhaustion assertions above that carry the isolation claim; this only + // confirms the reads are actually reachable in that state. + let alive: (i32,) = sqlx::query_as("SELECT 1") + .fetch_one(&pool) + .await + .expect("app pool must remain usable while the lock pool is exhausted"); + assert_eq!(alive.0, 1); + + for g in guards.drain(..) { + let _ = g.release(true).await; + } + } + + /// A waiter spinning on a contended repo must hand its pool slot back for the + /// duration of each backoff, and must not block a write to an unrelated repo + /// (R5, both halves). + /// + /// The pool-counter sampling is the load-bearing half. A second `acquire_write` + /// succeeding proves only that two different lock keys do not collide, which is + /// true whether or not the spinner released anything: with the slot held through + /// the sleep, a pool of 3 still has room for it. So this samples what the + /// spinner actually occupies across more than two backoff cycles. Moving + /// `drop(probe)` after the backoff sleep turns it red. + #[sqlx::test] + async fn waiter_on_one_repo_does_not_block_another(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 3).await; + let store = std::sync::Arc::new(RepoStore::for_testing( + PathBuf::from("/tmp/gitlawb-f5b"), + lock_pool.clone(), + )); + + let held = store + .acquire_write("did:key:z6MkF5Cont", "contended") + .await + .unwrap(); + + let spinner = { + let s = store.clone(); + tokio::spawn(async move { s.acquire_write("did:key:z6MkF5Cont", "contended").await }) + }; + + // Sample across >2 backoff cycles. `held` accounts for exactly one + // checked-out connection throughout, so every sample above that is the + // spinner sitting on a slot it is not using. + let mut spinner_idle = 0; + let mut samples = 0; + for _ in 0..50 { + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + let checked_out = lock_pool.size() as usize - lock_pool.num_idle(); + if checked_out == 1 { + spinner_idle += 1; + } + samples += 1; + } + assert!( + spinner_idle * 10 >= samples * 7, + "a spinner must hold no lock-pool slot through its backoff: only {spinner_idle}/{samples} \ + samples showed just the held guard checked out" + ); + + let unrelated = tokio::time::timeout( + std::time::Duration::from_secs(8), + store.acquire_write("did:key:z6MkF5Other", "innocent"), + ) + .await + .expect("an unrelated repo must not wait on someone else's contention") + .expect("and must acquire"); + let _ = unrelated.release(true).await; + + spinner.abort(); + let _ = held.release(true).await; + } + + /// Lock contention that runs out the acquire deadline must surface as a + /// retryable 503 with a fixed body, not a 500 carrying the owner slug and repo + /// name. The deadline is a field so this does not wait out the 90s default. + #[sqlx::test] + async fn contended_acquire_sheds_as_repo_busy_not_internal_error(pool: PgPool) { + use axum::response::IntoResponse; + + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 4).await; + let store = RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-busy"), lock_pool) + .with_lock_acquire_deadline(std::time::Duration::from_millis(300)); + + let held = store + .acquire_write("did:key:z6MkBusyOwner", "busyrepo") + .await + .expect("first writer acquires"); + + // Not `expect_err`: the guard is not Debug, and a guard obtained here must be + // released rather than dropped on a panic path. + let err = match store + .acquire_write("did:key:z6MkBusyOwner", "busyrepo") + .await + { + Err(e) => e, + Ok(second) => { + let _ = second.release(false).await; + panic!("a second writer must be shed once the deadline expires"); + } + }; + + // The internal chain keeps the operator detail... + let chain = format!("{err:#}"); + assert!( + chain.contains("busyrepo"), + "the log-side error must name the repo, got {chain}" + ); + + // ...and the client-visible mapping must carry neither it nor a 500. + let resp = crate::error::AppError::from(err).into_response(); + assert_eq!( + resp.status(), + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "contention is transient and must be retryable" + ); + let body = axum::body::to_bytes(resp.into_body(), 64 * 1024) + .await + .expect("body"); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("repo_busy") && !body.contains("busyrepo"), + "the 503 body must be fixed and must not name the repo, got {body}" + ); + + let _ = held.release(true).await; + } + + /// An under-lock refresh refusal must surface as a retryable 503 with a fixed + /// body, not a 500 carrying the owner slug and repo name. Built directly from + /// the typed error so it needs no database; the `.context()` layer is kept + /// deliberately, because the real raise path wraps one and this proves anyhow + /// preserves downcastability through it. + #[tokio::test] + async fn repo_unavailable_maps_to_retryable_503_with_fixed_body() { + use axum::response::IntoResponse; + + let err = anyhow::Error::new(RepoUnavailable) + .context("tigris HEAD failed before a write for did_key_z6MkTest/secret-repo"); + + let resp = crate::error::AppError::from(err).into_response(); + assert_eq!( + resp.status(), + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "a storage blip is transient and must be retryable, not a 500" + ); + let body = axum::body::to_bytes(resp.into_body(), 64 * 1024) + .await + .expect("body"); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("repo_unavailable"), + "the 503 must carry the repo_unavailable code, got {body}" + ); + assert!( + !body.contains("secret-repo"), + "the 503 body must be fixed and must not name the repo, got {body}" + ); + assert!( + !body.contains("did_key_z6MkTest"), + "the 503 body must be fixed and must not name the owner, got {body}" + ); + } + + /// The new downcast rung must be additive: an unrelated anyhow error still + /// falls through to the internal 500. + #[tokio::test] + async fn repo_unavailable_rung_does_not_swallow_unrelated_errors() { + use axum::response::IntoResponse; + + let resp = + crate::error::AppError::from(anyhow::anyhow!("some other failure")).into_response(); + assert_eq!( + resp.status(), + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + "an unrelated failure must not be reclassified as retryable" + ); + let body = axum::body::to_bytes(resp.into_body(), 64 * 1024) + .await + .expect("body"); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("internal_error"), + "an unrelated failure must keep the internal_error code, got {body}" + ); + } + + /// A Tigris client aimed at a closed port, so every call fails at the + /// transport layer promptly and `exists()` returns `Err` rather than + /// `Ok(false)`. + #[cfg(test)] + fn unreachable_tigris() -> TigrisClient { + TigrisClient::for_testing_with_endpoint("test-bucket", "http://127.0.0.1:1") + } + + /// The under-lock sibling of the above. `acquire_write` already refuses on + /// this condition; this proves the `RefreshFailure::Unknown` arm end to end + /// against a real failing HEAD rather than by reading the code. + #[sqlx::test] + async fn acquire_write_refuses_when_the_head_check_fails(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 2).await; + let store = RepoStore::for_testing_with_tigris( + PathBuf::from("/tmp/gitlawb-headfail-write"), + lock_pool, + unreachable_tigris(), + ); + + // Not `expect_err`: the guard is not Debug, and a guard obtained here + // must be released rather than dropped on a panic path. + let err = match store + .acquire_write("did:key:z6MkHeadFail", "writerepo") + .await + { + Err(e) => e, + Ok(guard) => { + let _ = guard.release(false).await; + panic!("a failed HEAD must refuse the write rather than proceed on a stale tree"); + } + }; + assert!( + err.downcast_ref::().is_some(), + "the refusal must be typed so the handler layer maps it to a retryable 503, got {err:#}" + ); + } + + /// P2 (cold-cache): the under-lock refresh's HEAD succeeds but the GET fails + /// on a node with no local copy. The download arm must refuse as + /// `RepoUnavailable` (retryable 503), matching the HEAD arm and the + /// `acquire_fresh` sibling, not a bare anyhow error that the handler layer + /// maps to a permanent 500. + #[sqlx::test] + async fn acquire_write_refuses_when_the_download_fails_and_no_local_copy_exists(pool: PgPool) { + use axum::response::IntoResponse; + + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any(|method: axum::http::Method| async move { + if method == axum::http::Method::HEAD { + // A real Tigris HEAD 200 carries the generation ETag. Without + // it `head_etag` errors and the test would exercise the HEAD + // arm, not the download arm this test exists for. + let mut resp = axum::http::StatusCode::OK.into_response(); + resp.headers_mut() + .insert("etag", axum::http::HeaderValue::from_static("\"gen-1\"")); + resp + } else { + axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 2).await; + let store = RepoStore::for_testing_with_tigris( + PathBuf::from("/tmp/gitlawb-getfail-write"), + lock_pool, + TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint), + ); + + let err = match store + .acquire_write("did:key:z6MkGetFailWrite", "writerepo") + .await + { + Err(e) => e, + Ok(guard) => { + let _ = guard.release(false).await; + panic!("a failed download with no local copy must refuse the write"); + } + }; + assert!( + err.downcast_ref::().is_some(), + "the refusal must be typed so the handler layer maps it to a retryable 503, got {err:#}" + ); + + server.abort(); + } + + /// A failed under-lock GET must refuse even when a local copy exists. HEAD + /// only establishes the stored generation, not that the cached tree matches + /// it, so writing against stale-local + new commits can overwrite a newer + /// archive when the GET fails transiently. + #[sqlx::test] + async fn acquire_write_refuses_when_the_download_fails_with_a_local_copy(pool: PgPool) { + use axum::response::IntoResponse; + + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any(|method: axum::http::Method| async move { + if method == axum::http::Method::HEAD { + let mut resp = axum::http::StatusCode::OK.into_response(); + resp.headers_mut() + .insert("etag", axum::http::HeaderValue::from_static("\"gen-1\"")); + resp + } else { + axum::http::StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let owner = "did:key:z6MkStaleLocalWrite"; + let repo_name = "writerepo"; + let owner_slug = crate::db::normalize_owner_key(owner); + let base = tempfile::TempDir::new().unwrap(); + let repo_path = base + .path() + .join(owner_slug) + .join(format!("{repo_name}.git")); + std::fs::create_dir_all(repo_path.parent().unwrap()).unwrap(); + crate::git::store::init_bare(&repo_path).expect("seed bare repo"); + + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 2).await; + let store = RepoStore::for_testing_with_tigris( + base.path().to_path_buf(), + lock_pool, + TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint), + ); + + let err = match store.acquire_write(owner, repo_name).await { + Err(e) => e, + Ok(guard) => { + let _ = guard.release(false).await; + panic!( + "a failed download with a local copy must refuse rather than write against an unverified tree" + ); + } + }; + assert!( + err.downcast_ref::().is_some(), + "the refusal must be typed so the handler layer maps it to a retryable 503, got {err:#}" + ); + + server.abort(); + } + + /// The transfer bound is a knob, so it gets the same parse/default/reject-zero + /// coverage its sibling lock-pool-size knob has. + #[test] + fn lock_held_transfer_timeout_defaults_and_rejects_zero() { + use clap::Parser; + assert_eq!( + crate::config::Config::parse_from(["gitlawb-node"]).lock_held_transfer_timeout_secs, + 300 + ); + assert!(crate::config::Config::try_parse_from([ + "gitlawb-node", + "--lock-held-transfer-timeout-secs", + "0" + ]) + .is_err()); + } + + /// P1a: the non-owner pre-check must refresh from a NON-MUTATING snapshot. + /// A snapshot download must unpack into a throwaway temp dir and leave the + /// live repo path untouched, so an unlocked pre-check cannot delete or swap + /// the directory under a concurrent guarded write. + /// + /// Real S3 server (not a mock): upload an archive, then `read_snapshot` it, + /// and assert the snapshot path is a fresh temp dir distinct from the live + /// path, that the live path was never created, and that the snapshot reads + /// the same content. + #[sqlx::test] + async fn read_snapshot_is_non_mutating(pool: PgPool) { + use axum::response::IntoResponse; + + // A real in-process S3-compatible server via the SDK against an axum + // router is more plumbing than this test needs; instead, upload through + // the real Tigris client against an axum server that stores the object + // in memory, then snapshot through the same store. + // + // Simpler and equally load-bearing: build the archive bytes, serve them + // with a real HTTP server that answers HEAD 200 and GET with the bytes, + // then call read_snapshot and assert the live path is untouched and the + // snapshot content matches. + let mut archive_bytes = Vec::new(); + { + let dir = + std::env::temp_dir().join(format!("gitlawb-snap-src-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(dir.join("objects/info")).unwrap(); + std::fs::create_dir_all(dir.join("refs/heads")).unwrap(); + std::fs::write(dir.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + std::fs::write(dir.join("objects/info/packs"), "").unwrap(); + let encoder = zstd::stream::Encoder::new(&mut archive_bytes, 3).unwrap(); + let mut tar = tar::Builder::new(encoder); + tar.append_dir_all(".", &dir).unwrap(); + tar.into_inner().unwrap().finish().unwrap(); + std::fs::remove_dir_all(&dir).unwrap(); + } + let archive = std::sync::Arc::new(archive_bytes); + + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any(move |method: axum::http::Method| { + let archive = archive.clone(); + async move { + match method { + axum::http::Method::HEAD => axum::http::StatusCode::OK.into_response(), + axum::http::Method::GET => { + use axum::body::Body; + ( + [(axum::http::header::CONTENT_TYPE, "application/zstd")], + Body::from(archive.as_ref().clone()), + ) + .into_response() + } + _ => axum::http::StatusCode::METHOD_NOT_ALLOWED.into_response(), + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let opts = (*pool.connect_options()).clone(); + let lock_pool = no_reap_pool(&opts, 2).await; + let store = RepoStore::for_testing_with_tigris( + PathBuf::from("/tmp/gitlawb-snapshot-nonmut"), + lock_pool, + TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint), + ); + + let owner_did = "did:key:z6MkSnap"; + let (owner_slug, live_path) = store.local_path(owner_did, "snaprepo").unwrap(); + assert!( + !live_path.exists(), + "the live path must not exist before the snapshot" + ); + + let snap = store + .read_snapshot(owner_did, "snaprepo") + .await + .expect("snapshot reads the archive"); + let snap_path = snap.path().to_path_buf(); + let live_path_buf = live_path.as_path().to_path_buf(); + assert_ne!( + snap_path, live_path_buf, + "the snapshot must unpack into a temp dir, not the live path" + ); + assert!( + snap_path.starts_with(live_path.parent().unwrap()), + "the snapshot temp dir must live under the repo parent" + ); + assert!( + !live_path.exists(), + "the live path must remain untouched by a snapshot read" + ); + assert_eq!( + std::fs::read_to_string(snap_path.join("HEAD")).unwrap(), + "ref: refs/heads/main\n", + "the snapshot must contain the archive's content" + ); + drop(snap); + assert!( + !snap_path.exists(), + "dropping the snapshot must clean up its temp dir" + ); + let _ = owner_slug; + + server.abort(); + } + + /// Build a store whose release-side upload lands on `mock` and gives up + /// after 200ms, so a parked PUT reliably exceeds the bound. One lock-pool + /// connection on purpose: with a single slot the backend pid is a direct + /// observable for whether `release` pooled its session or closed it. + async fn timed_out_upload_store( + mock: &S3Mock, + opts: &sqlx::postgres::PgConnectOptions, + repos_dir: &str, + ) -> RepoStore { + RepoStore::new( + PathBuf::from(repos_dir), + Some(TigrisClient::for_testing_with_endpoint( + "test-bucket", + mock.endpoint(), + )), + no_reap_pool(opts, 1).await, + std::time::Duration::from_millis(200), + ) + } + + /// Seed the minimum bare-repo shape so the upload has something to archive. + fn seed_bare_repo(path: &Path) { + std::fs::create_dir_all(path.join("objects/info")).unwrap(); + std::fs::create_dir_all(path.join("refs/heads")).unwrap(); + std::fs::write(path.join("HEAD"), "ref: refs/heads/main\n").unwrap(); + } + + /// A timed-out release upload must still unlock on its OWN session and hand + /// that session back to the pool. The timeout says nothing about the lock: + /// holding it cannot fence a late PUT (`release` takes `mut self`, so the + /// guard drops and `Drop` frees the session the moment `release` returns), + /// and what actually protects a successor is the conditional PUT. + /// + /// Observable: the backend pid. On a one-connection pool a session that was + /// closed forces the next checkout onto a fresh backend, while a confirmed + /// unlock returns the same one. So an equal pid is the proof that the unlock + /// ran, returned true, and the connection was pooled rather than torn down. + #[sqlx::test] + async fn timed_out_release_upload_unlocks_on_its_own_session(pool: PgPool) { + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let store = timed_out_upload_store(&mock, &opts, "/tmp/gitlawb-u4-timeout-session").await; + + let mut guard = store + .acquire_write("did:key:z6MkU4TimeoutSess", "timedrepo") + .await + .expect("acquire"); + seed_bare_repo(&guard.local_path); + let pid_before = guard.backend_pid_for_test().await; + + // Park the upload so it is still in flight when the 200ms bound fires. + mock.park_next_put(); + let _ = guard.release(true).await; + assert_eq!( + mock.put_attempts().len(), + 1, + "the release upload must have reached the mock and parked, got {:?}", + mock.put_attempts() + ); + + let pid_after = { + let mut c = store.lock_pool.acquire().await.unwrap(); + let pid: (i32,) = sqlx::query_as("SELECT pg_backend_pid()") + .fetch_one(&mut *c) + .await + .unwrap(); + pid.0 + }; + assert_eq!( + pid_before, pid_after, + "a timed-out upload must not change the unlock decision: the guard must \ + unlock on its own session and return that connection to the pool, so the \ + next checkout lands on the same backend" + ); + + mock.open_gate(); + mock.shutdown(); + } + + /// Admission after the same timed-out upload: a successor must be let in + /// promptly. Useful as a property, but it is NOT what pins the removal of + /// the skip-unlock branch, because the lock frees within milliseconds under + /// either shape (the session closes as soon as `release` returns). + #[sqlx::test] + async fn successor_is_admitted_promptly_after_a_timed_out_release(pool: PgPool) { + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let store = timed_out_upload_store(&mock, &opts, "/tmp/gitlawb-u4-timeout-admit") + .await + .with_lock_acquire_deadline(std::time::Duration::from_secs(10)); + + let guard = store + .acquire_write("did:key:z6MkU4TimeoutAdmit", "timedrepo") + .await + .expect("acquire"); + seed_bare_repo(&guard.local_path); + + mock.park_next_put(); + let _ = guard.release(true).await; + + let started = std::time::Instant::now(); + let successor = store + .acquire_write("did:key:z6MkU4TimeoutAdmit", "timedrepo") + .await + .expect("a successor must be admitted after a timed-out release"); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "the successor waited {}ms; a timed-out upload must not park the next writer", + started.elapsed().as_millis() + ); + let _ = successor.release(false).await; + + mock.open_gate(); + mock.shutdown(); + } + + /// P2: the lock-acquire deadline must bound EVERY await in the retry loop, + /// not just the sleep between attempts. A pool checkout that would exceed + /// the deadline must shed as `RepoBusy` rather than wait out the pool's own + /// acquire timeout past the promised wall-clock cap. + /// + /// Observable: hold every lock-pool slot from an independent store, then + /// acquire with a short deadline. The pool checkout will not complete within + /// the deadline, so `acquire_write` must refuse as `RepoBusy` once the + /// deadline fires — not hang for the pool's 5s acquire timeout. + #[sqlx::test] + async fn pool_checkout_past_the_deadline_sheds_as_repo_busy(pool: PgPool) { + let opts = (*pool.connect_options()).clone(); + + // Exhaust every slot of the lock pool. The checkouts must come from the + // pool the store will use, not from independent connections: a separate + // `PgConnection::connect_with` consumes no slot, so the store's checkout + // would succeed immediately and the deadline would never be reached. + const N: u32 = 2; + let lock_pool = no_reap_pool(&opts, N).await; + let mut holders = Vec::new(); + for _ in 0..N { + holders.push(lock_pool.acquire().await.expect("hold a lock-pool slot")); + } + + // The store shares that exhausted pool (`PgPool` is a handle to one + // inner pool, so the clone is the same set of slots). + let store = + RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-deadline"), lock_pool.clone()) + .with_lock_acquire_deadline(std::time::Duration::from_millis(400)); + + // The pool is exhausted, so the checkout cannot complete within the + // deadline; the deadline must fire and shed as RepoBusy rather than let + // the pool's own 5s acquire timeout run. + let started = std::time::Instant::now(); + let err = match store + .acquire_write("did:key:z6MkDeadline", "deadline-repo") + .await + { + Err(e) => e, + Ok(guard) => { + let _ = guard.release(false).await; + panic!("with the pool exhausted, the deadline must shed, not succeed"); + } + }; + assert!( + err.downcast_ref::().is_some(), + "a checkout past the deadline must shed as RepoBusy, got {err:#}" + ); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "the refusal must come from the deadline, not the pool's own 5s acquire timeout" + ); + + // Release the holders so the test's pool can be torn down cleanly. + drop(holders); + } + + // ── conditional-semantics S3 mock (#279) ─────────────────────────────── + + /// A captured PUT: the body and the conditional headers as they arrived. + /// + /// Capture is deliberately separate from evaluation. When tokio drops an + /// SDK future the client can tear the TCP connection down and the server + /// side handler task is cancelled with it, so a parked handler that resumes + /// on its own is not something a test can depend on. Replaying what arrived + /// models the real S3 arm we care about (body fully transmitted, commit + /// decided later) with no timing in it. + #[derive(Clone, Debug)] + struct CapturedPut { + key: String, + body: Vec, + if_match: Option, + if_none_match: Option, + attempt: Option, + } + + /// One stored object. + #[derive(Clone, Debug)] + struct MockObject { + body: Vec, + etag: String, + /// The `x-amz-meta-gitlawb-attempt` it was written with. This is what + /// makes "are the published bytes MINE" answerable, and the mock has to + /// model it or no reconciliation test proves anything. + attempt: Option, + } + + /// One PUT as the mock judged it, for tests that assert on attempt counts. + /// `status` is `None` while a PUT is parked: it arrived and was logged, but + /// no precondition has been evaluated for it yet. + #[derive(Clone, Debug, PartialEq)] + struct PutAttempt { + if_match: Option, + if_none_match: Option, + status: Option, + } + + #[derive(Default)] + struct MockState { + /// Keyed by request path. A single-slot store was enough while every test + /// drove one repo; fork creation touches the SOURCE key and the FORK key + /// in one request, and collapsing them would make an assertion about one + /// silently read the other. + objects: HashMap, + /// The key the last successful PUT wrote, so the single-key accessors + /// below keep meaning what they meant when this mock served one key. + last_key: Option, + next_etag: u64, + puts: Vec, + /// Set by `park_next_put`, consumed by the next arriving PUT. + park_next_put: bool, + /// The response-loss arm, set by `commit_then_lose_next_put_response` / + /// `fail_next_put_after_delivery`. The request is fully delivered and the + /// client is told it failed; `Some(true)` also COMMITS it first, which is + /// the state that makes the failure a lie rather than a truth. + lose_next_response: Option, + /// Set by `roll_generation_after_next_heads`, decremented per HEAD. + roll_after_heads: u32, + captured: Option, + /// Conditional DELETEs the mock accepted, so a compensation test can + /// assert that a guarded delete did NOT run. + deletes: u32, + } + + /// An in-process S3-compatible server with REAL conditional semantics. + /// + /// The fence tests downstream are only worth anything if a precondition can + /// actually fail here, so this helper carries its own semantics tests below. + struct S3Mock { + endpoint: String, + state: Arc>, + gate: Arc, + server: tokio::task::JoinHandle<()>, + } + + /// S3 quotes ETags. Compare unquoted so a value that round-tripped through + /// the SDK (which surfaces `e_tag()` with the quotes intact) matches what + /// the mock minted. + fn unquote_etag(raw: &str) -> &str { + raw.trim().trim_matches('"') + } + + /// The conditional evaluation, in one place so a live PUT and a replayed + /// one cannot drift apart. Returns the status, and on success the fresh + /// ETag. Preconditions are read against the state passed in, which is + /// always the state as of the CALL, never as of capture. + fn evaluate_put( + st: &mut MockState, + key: &str, + body: Vec, + if_match: Option<&str>, + if_none_match: Option<&str>, + attempt: Option<&str>, + ) -> (u16, Option) { + let refuse = |st: &mut MockState| { + st.puts.push(PutAttempt { + if_match: if_match.map(str::to_string), + if_none_match: if_none_match.map(str::to_string), + status: Some(412), + }); + (412u16, None) + }; + + if let Some(want) = if_match { + // An absent object matches nothing, so If-Match cannot pass. + match st.objects.get(key).map(|o| o.etag.as_str()) { + Some(have) if unquote_etag(have) == unquote_etag(want) => {} + _ => return refuse(st), + } + } + if if_none_match.map(str::trim) == Some("*") && st.objects.contains_key(key) { + return refuse(st); + } + + // A fresh ETag per successful PUT, from a counter rather than a content + // hash: two writers can publish byte-identical archives, and an ETag + // that repeated across them would let a fence pass on a generation it + // never observed. + st.next_etag += 1; + let etag = format!("\"mock-etag-{}\"", st.next_etag); + st.objects.insert( + key.to_string(), + MockObject { + body, + etag: etag.clone(), + attempt: attempt.map(str::to_string), + }, + ); + st.last_key = Some(key.to_string()); + st.puts.push(PutAttempt { + if_match: if_match.map(str::to_string), + if_none_match: if_none_match.map(str::to_string), + status: Some(200), + }); + (200, Some(etag)) + } + + impl S3Mock { + async fn start() -> Self { + use axum::response::IntoResponse; + + let state = Arc::new(std::sync::Mutex::new(MockState::default())); + let gate = Arc::new(tokio::sync::Notify::new()); + + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any({ + let state = state.clone(); + let gate = gate.clone(); + move |method: axum::http::Method, + uri: axum::http::Uri, + headers: axum::http::HeaderMap, + body: axum::body::Bytes| { + let state = state.clone(); + let gate = gate.clone(); + async move { + let key = uri.path().trim_start_matches('/').to_string(); + let header = |name: &str| { + headers + .get(name) + .and_then(|v| v.to_str().ok()) + .map(str::to_string) + }; + match method { + axum::http::Method::PUT => { + let if_match = header("if-match"); + let if_none_match = header("if-none-match"); + let attempt = + header(&format!("x-amz-meta-{ATTEMPT_METADATA_KEY}")); + + // A parked PUT records what arrived and then + // waits. The client will usually be gone by + // the time the gate opens, which is exactly + // why the deterministic arm is the replay. + let parked = { + let mut st = state.lock().unwrap(); + if st.park_next_put { + st.park_next_put = false; + st.puts.push(PutAttempt { + if_match: if_match.clone(), + if_none_match: if_none_match.clone(), + status: None, + }); + st.captured = Some(CapturedPut { + key: key.clone(), + body: body.to_vec(), + if_match: if_match.clone(), + if_none_match: if_none_match.clone(), + attempt: attempt.clone(), + }); + true + } else { + false + } + }; + if parked { + gate.notified().await; + return axum::http::StatusCode::OK.into_response(); + } + + let (status, etag, lost_response) = { + #[allow(clippy::needless_late_init)] + let mut st = state.lock().unwrap(); + let lost_response = st.lose_next_response.take(); + if lost_response == Some(false) { + st.puts.push(PutAttempt { + if_match: if_match.clone(), + if_none_match: if_none_match.clone(), + status: Some(500), + }); + return axum::http::StatusCode::INTERNAL_SERVER_ERROR + .into_response(); + } + let (status, etag) = evaluate_put( + &mut st, + &key, + body.to_vec(), + if_match.as_deref(), + if_none_match.as_deref(), + attempt.as_deref(), + ); + (status, etag, lost_response) + }; + // The response-loss arm: the write is COMMITTED + // above and the client is told it failed. This + // is the state an S3-compatible store reaches + // when it durably records a conditional PUT and + // then loses or corrupts the response. + if lost_response == Some(true) && status == 200 { + return axum::http::StatusCode::INTERNAL_SERVER_ERROR + .into_response(); + } + match etag { + Some(etag) => ( + axum::http::StatusCode::OK, + [(axum::http::header::ETAG, etag)], + ) + .into_response(), + None => axum::http::StatusCode::from_u16(status) + .unwrap() + .into_response(), + } + } + axum::http::Method::DELETE => { + let if_match = header("if-match"); + let mut st = state.lock().unwrap(); + let Some(existing) = st.objects.get(&key).cloned() else { + return axum::http::StatusCode::NO_CONTENT.into_response(); + }; + let stale = if_match.as_deref().is_some_and(|want| { + unquote_etag(&existing.etag) != unquote_etag(want) + }); + if stale { + return axum::http::StatusCode::PRECONDITION_FAILED + .into_response(); + } + st.objects.remove(&key); + st.deletes += 1; + axum::http::StatusCode::NO_CONTENT.into_response() + } + axum::http::Method::HEAD | axum::http::Method::GET => { + let mut st = state.lock().unwrap(); + let answered = st.objects.get(&key).cloned(); + // Fault injection for the two-consecutive- + // losses arm, and the only deterministic way + // to sit BETWEEN a caller's HEAD and the + // conditional PUT it derives from it. The + // gate cannot do this: a parked PUT is + // captured rather than evaluated and answers + // 200, so it can never produce a refusal. + // + // Only the generation moves, not the bytes, + // which is a real state a store reaches (two + // writers can publish byte-identical + // archives) and keeps the stored object a + // valid archive for whoever downloads next. + // `evaluate_put` is untouched, and this logs + // no PutAttempt, so attempt counts still + // count only the caller's own PUTs. + if method == axum::http::Method::HEAD + && st.roll_after_heads > 0 + && st.objects.contains_key(&key) + { + st.roll_after_heads -= 1; + st.next_etag += 1; + let rolled = format!("\"mock-etag-{}\"", st.next_etag); + if let Some(obj) = st.objects.get_mut(&key) { + obj.etag = rolled; + } + } + match answered { + Some(obj) => { + let mut resp = ( + axum::http::StatusCode::OK, + [(axum::http::header::ETAG, obj.etag)], + axum::body::Body::from(obj.body), + ) + .into_response(); + if let Some(attempt) = obj.attempt { + resp.headers_mut().insert( + axum::http::HeaderName::from_static( + "x-amz-meta-gitlawb-attempt", + ), + axum::http::HeaderValue::from_str(&attempt) + .expect("attempt id is ascii"), + ); + } + resp + } + None => axum::http::StatusCode::NOT_FOUND.into_response(), + } + } + _ => axum::http::StatusCode::METHOD_NOT_ALLOWED.into_response(), + } + } + } + }), + ); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + Self { + endpoint, + state, + gate, + server, + } + } + + fn endpoint(&self) -> &str { + &self.endpoint + } + + /// The object the last successful PUT wrote. Every test that uses this + /// drives a single key; a test touching two keys reads them by key. + fn last(&self) -> Option { + let st = self.state.lock().unwrap(); + let key = st.last_key.clone()?; + st.objects.get(&key).cloned() + } + + fn current_etag(&self) -> Option { + self.last().map(|o| o.etag) + } + + fn object(&self) -> Option> { + self.last().map(|o| o.body) + } + + /// The stored object under a specific repo key. + fn object_for(&self, owner_slug: &str, repo_name: &str) -> Option { + let key = format!("test-bucket/repos/v1/{owner_slug}/{repo_name}.tar.zst"); + self.state.lock().unwrap().objects.get(&key).cloned() + } + + fn put_attempts(&self) -> Vec { + self.state.lock().unwrap().puts.clone() + } + + /// Answer each of the next `n` HEADs from the current state, then + /// immediately move the object to a new generation. A caller that HEADs + /// to pick up a precondition and then PUTs on it is therefore fencing + /// on a generation that is already gone, which is the only way to drive + /// two consecutive lost preconditions deterministically. + fn roll_generation_after_next_heads(&self, n: u32) { + self.state.lock().unwrap().roll_after_heads = n; + } + + /// Park the next arriving PUT so the caller's transfer bound elapses + /// with the request in flight (the abandoned-writer arm). + fn park_next_put(&self) { + self.state.lock().unwrap().park_next_put = true; + } + + /// Accept and COMMIT the next PUT, then answer 500. The response-loss + /// arm: the write is durable and the client is told it failed. + fn commit_then_lose_next_put_response(&self) { + self.state.lock().unwrap().lose_next_response = Some(true); + } + + /// Deliver the next PUT in full and then fail it WITHOUT committing. The + /// client's knowledge is identical to the arm above — that is the whole + /// point — so only asking the store can tell the two apart. + fn fail_next_put_after_delivery(&self) { + self.state.lock().unwrap().lose_next_response = Some(false); + } + + /// The attempt id stamped on whatever the last successful PUT stored. + fn stored_attempt(&self) -> Option { + self.last().and_then(|o| o.attempt) + } + + /// How many DELETEs the mock actually carried out. + fn deletes(&self) -> u32 { + self.state.lock().unwrap().deletes + } + + /// Let a parked handler go. Only the socket-level arm needs this; the + /// deterministic assertion is `replay_captured`. + fn open_gate(&self) { + self.gate.notify_waiters(); + } + + fn captured_put(&self) -> Option { + self.state.lock().unwrap().captured.clone() + } + + /// Re-run the captured PUT through the SAME evaluation the handler uses, + /// against the state as it is NOW. + fn replay_captured(&self) -> u16 { + let mut st = self.state.lock().unwrap(); + let captured = st.captured.clone().expect("a PUT was captured"); + evaluate_put( + &mut st, + &captured.key, + captured.body, + captured.if_match.as_deref(), + captured.if_none_match.as_deref(), + captured.attempt.as_deref(), + ) + .0 + } + + fn shutdown(&self) { + self.server.abort(); + } + } + + /// An SDK client aimed at the mock. Built here rather than through + /// `TigrisClient` because these tests exercise raw conditional PUTs, which + /// the storage client does not expose. + fn mock_s3_client(endpoint: &str) -> aws_sdk_s3::Client { + use aws_sdk_s3::config::{retry::RetryConfig, Credentials, Region}; + + let config = aws_sdk_s3::config::Config::builder() + .endpoint_url(endpoint) + .credentials_provider(Credentials::new("test", "test", None, None, "test")) + .region(Region::new("auto")) + .retry_config(RetryConfig::disabled()) + .behavior_version_latest() + .build(); + aws_sdk_s3::Client::from_conf(config) + } + + /// PUT through the SDK, returning either the fresh ETag or the HTTP status + /// the mock refused with. + async fn mock_put( + client: &aws_sdk_s3::Client, + body: &[u8], + if_match: Option<&str>, + if_none_match: Option<&str>, + ) -> Result { + mock_put_as(client, body, if_match, if_none_match, None).await + } + + /// The same, stamped with an attempt id, so a test can seed an object that + /// belongs to a NAMED attempt (its own, or a foreign one). + async fn mock_put_as( + client: &aws_sdk_s3::Client, + body: &[u8], + if_match: Option<&str>, + if_none_match: Option<&str>, + attempt: Option<&str>, + ) -> Result { + let mut req = client + .put_object() + .bucket("test-bucket") + .key("repos/v1/owner/repo.tar.zst") + .body(aws_sdk_s3::primitives::ByteStream::from(body.to_vec())); + if let Some(attempt) = attempt { + req = req.metadata(ATTEMPT_METADATA_KEY, attempt); + } + if let Some(v) = if_match { + req = req.if_match(v); + } + if let Some(v) = if_none_match { + req = req.if_none_match(v); + } + match req.send().await { + Ok(out) => Ok(out + .e_tag() + .expect("a successful PUT returns an ETag") + .to_string()), + Err(e) => Err(e + .raw_response() + .map(|r| r.status().as_u16()) + .unwrap_or_else(|| panic!("expected an HTTP response from the mock, got {e:?}"))), + } + } + + /// HEAD through the SDK, reported the way `TigrisClient::exists` reports it: + /// `Ok(false)` for a not-found, `Ok(true)` for a hit whose ETag is present. + async fn mock_head(client: &aws_sdk_s3::Client) -> Result { + match client + .head_object() + .bucket("test-bucket") + .key("repos/v1/owner/repo.tar.zst") + .send() + .await + { + Ok(out) => { + out.e_tag().ok_or("a HEAD hit must carry an ETag")?; + Ok(true) + } + Err(e) if e.as_service_error().is_some_and(|e| e.is_not_found()) => Ok(false), + Err(e) => Err(format!("unexpected HEAD failure: {e}")), + } + } + + /// 1. A stale If-Match must be refused, and the refusal must not write. + #[tokio::test] + async fn mock_refuses_a_wrong_if_match_and_leaves_the_object_unchanged() { + let mock = S3Mock::start().await; + let client = mock_s3_client(mock.endpoint()); + + let etag = mock_put(&client, b"first", None, None) + .await + .expect("the seeding PUT succeeds"); + + let status = mock_put(&client, b"second", Some("\"not-the-current-etag\""), None) + .await + .expect_err("a stale If-Match must be refused"); + assert_eq!(status, 412, "a stale If-Match must answer 412"); + assert_eq!( + mock.object().as_deref(), + Some(b"first".as_slice()), + "a refused PUT must leave the stored object unchanged" + ); + assert_eq!( + mock.current_etag().as_deref().map(unquote_etag), + Some(unquote_etag(&etag)), + "a refused PUT must leave the ETag unchanged" + ); + + mock.shutdown(); + } + + /// 2. The matching If-Match is the write that must go through. + #[tokio::test] + async fn mock_accepts_a_matching_if_match_and_rotates_the_etag() { + let mock = S3Mock::start().await; + let client = mock_s3_client(mock.endpoint()); + + let first = mock_put(&client, b"first", None, None).await.expect("seed"); + let second = mock_put(&client, b"second", Some(&first), None) + .await + .expect("a matching If-Match must succeed"); + + assert_ne!( + unquote_etag(&first), + unquote_etag(&second), + "a successful conditional PUT must mint a fresh ETag" + ); + assert_eq!( + mock.object().as_deref(), + Some(b"second".as_slice()), + "the accepted body must be what is stored" + ); + assert_eq!( + mock.current_etag().as_deref().map(unquote_etag), + Some(unquote_etag(&second)), + "HEAD/GET must report the ETag the PUT returned" + ); + + mock.shutdown(); + } + + /// 3. If-None-Match `*` is the create-only fence, so an existing object + /// must refuse it. + #[tokio::test] + async fn mock_refuses_if_none_match_star_against_an_existing_object() { + let mock = S3Mock::start().await; + let client = mock_s3_client(mock.endpoint()); + + mock_put(&client, b"first", None, None).await.expect("seed"); + let status = mock_put(&client, b"second", None, Some("*")) + .await + .expect_err("create-only against an existing object must be refused"); + + assert_eq!(status, 412, "If-None-Match * on an existing object is 412"); + assert_eq!( + mock.object().as_deref(), + Some(b"first".as_slice()), + "the refused create-only PUT must not overwrite" + ); + + mock.shutdown(); + } + + /// 4. The same fence must ADMIT the first writer, or the fresh-repo path + /// could never publish. + #[tokio::test] + async fn mock_accepts_if_none_match_star_against_an_empty_store() { + let mock = S3Mock::start().await; + let client = mock_s3_client(mock.endpoint()); + + // HEAD both ways, because `exists()` reads a not-found as "fresh repo" + // and any other status as a hard refusal. A mock that answered 200 on + // an empty store would send every fresh-repo test down the wrong arm. + assert!( + !mock_head(&client).await.expect("HEAD on an empty store"), + "an absent object must HEAD 404" + ); + + let etag = mock_put(&client, b"first", None, Some("*")) + .await + .expect("create-only against an empty store must succeed"); + assert_eq!(mock.object().as_deref(), Some(b"first".as_slice())); + assert_eq!( + mock.current_etag().as_deref().map(unquote_etag), + Some(unquote_etag(&etag)) + ); + assert!( + mock_head(&client).await.expect("HEAD after the create"), + "a stored object must HEAD 200 with the ETag the PUT returned" + ); + + mock.shutdown(); + } + + /// 5. Identical bytes must still produce a new ETag. Without this, an + /// If-Match fence would pass on a generation it never observed. + #[tokio::test] + async fn mock_mints_a_distinct_etag_per_successful_put() { + let mock = S3Mock::start().await; + let client = mock_s3_client(mock.endpoint()); + + let first = mock_put(&client, b"same", None, None).await.expect("first"); + let second = mock_put(&client, b"same", Some(&first), None) + .await + .expect("second"); + + assert_ne!( + unquote_etag(&first), + unquote_etag(&second), + "successive successful PUTs of identical bytes must still differ in ETag" + ); + + mock.shutdown(); + } + + /// 6. The whole point of capture-and-replay: the commit is judged when it + /// is replayed, not when the bytes arrived. A capture that was valid on + /// arrival must lose to a write that landed in between. + #[tokio::test] + async fn mock_judges_a_replayed_put_against_the_state_at_replay_time() { + let mock = S3Mock::start().await; + let client = mock_s3_client(mock.endpoint()); + + let first = mock_put(&client, b"first", None, None).await.expect("seed"); + + // Park the abandoned writer's PUT. Its If-Match is valid at ARRIVAL. + mock.park_next_put(); + let parked = tokio::time::timeout( + std::time::Duration::from_millis(300), + mock_put(&client, b"abandoned", Some(&first), None), + ) + .await; + assert!( + parked.is_err(), + "the parked PUT must still be in flight when the caller's bound elapses" + ); + let captured = mock + .captured_put() + .expect("the parked PUT must be captured at arrival"); + assert_eq!(captured.body, b"abandoned".to_vec()); + assert_eq!( + captured.if_match.as_deref().map(unquote_etag), + Some(unquote_etag(&first)), + "the capture must record the conditional headers as they arrived" + ); + assert_eq!(captured.if_none_match, None); + + // A successor commits while the capture sits parked. + let second = mock_put(&client, b"successor", Some(&first), None) + .await + .expect("the successor's PUT is the one that lands"); + + // Replaying now must be judged against the successor's state. + assert_eq!( + mock.replay_captured(), + 412, + "a replayed PUT must be evaluated against the state at replay time, \ + not the state it was captured against" + ); + assert_eq!( + mock.object().as_deref(), + Some(b"successor".as_slice()), + "the refused replay must not clobber the successor's object" + ); + assert_eq!( + mock.current_etag().as_deref().map(unquote_etag), + Some(unquote_etag(&second)) + ); + // Seed, parked, successor, replay. The log is what later tests assert + // attempt counts against, so it is checked here rather than trusted. + let attempts = mock.put_attempts(); + assert_eq!( + attempts.len(), + 4, + "every PUT attempt must be logged, got {attempts:?}" + ); + assert_eq!( + attempts.iter().map(|a| a.status).collect::>(), + vec![Some(200), None, Some(200), Some(412)], + "the parked attempt is logged undecided; the replay is the 412" + ); + assert_eq!( + attempts[1].if_match.as_deref().map(unquote_etag), + Some(unquote_etag(&first)), + "the parked attempt must be logged with the headers it arrived with" + ); + + mock.open_gate(); + mock.shutdown(); + } + + // ── conditional upload through TigrisClient (#279) ───────────────────── + + /// A tiny directory for `upload` to compress. What is inside does not + /// matter to a precondition test, only that a PUT carrying a body happens. + fn payload_dir(marker: &str) -> TempDir { + let dir = TempDir::new().unwrap(); + std::fs::write(dir.path().join("HEAD"), marker.as_bytes()).unwrap(); + dir + } + + /// A router that answers every request with one fixed status. This is NOT a + /// second semantics mock: it exists only to pin how a status the real mock + /// never produces (409, 404, 500) is classified. + async fn start_fixed_status_stub(status: u16) -> (String, tokio::task::JoinHandle<()>) { + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any( + move || async move { axum::http::StatusCode::from_u16(status).unwrap() }, + ), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + (endpoint, server) + } + + /// The one place the header itself is asserted: a matching If-Match must + /// succeed AND must actually have travelled as an If-Match header. The + /// store-level tests deliberately assert behavior rather than headers, so + /// if this assertion is not here, nothing pins the wire format. + #[tokio::test] + async fn upload_if_match_with_the_current_etag_publishes_and_sends_the_header() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + let seeded = mock_put(&mock_s3_client(mock.endpoint()), b"seed", None, None) + .await + .expect("seeding PUT"); + + let dir = payload_dir("winner"); + client + .upload( + "owner", + "repo", + dir.path(), + UploadPrecondition::IfMatch(seeded.clone()), + ) + .await + .expect("a matching If-Match must publish"); + + let last = mock + .put_attempts() + .last() + .cloned() + .expect("the upload must reach the mock"); + assert_eq!( + last.if_match.as_deref().map(unquote_etag), + Some(unquote_etag(&seeded)), + "the upload must carry the ETag it was fenced on as If-Match" + ); + assert_eq!(last.if_none_match, None); + assert_eq!(last.status, Some(200)); + + mock.shutdown(); + } + + #[tokio::test] + async fn upload_if_match_with_a_stale_etag_is_precondition_lost() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + mock_put(&mock_s3_client(mock.endpoint()), b"seed", None, None) + .await + .expect("seeding PUT"); + + let dir = payload_dir("loser"); + let err = client + .upload( + "owner", + "repo", + dir.path(), + UploadPrecondition::IfMatch("\"stale\"".to_string()), + ) + .await + .expect_err("a stale If-Match must be refused"); + assert!( + matches!(err, UploadError::PreconditionLost { status: 412 }), + "a stale If-Match must classify as a lost precondition, got {err:?}" + ); + assert_eq!( + mock.object().as_deref(), + Some(b"seed".as_slice()), + "the refused upload must not have written" + ); + + mock.shutdown(); + } + + #[tokio::test] + async fn upload_if_absent_into_an_empty_store_publishes() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + + let dir = payload_dir("first"); + client + .upload("owner", "repo", dir.path(), UploadPrecondition::IfAbsent) + .await + .expect("create-only into an empty store must publish"); + + let last = mock.put_attempts().last().cloned().expect("one attempt"); + assert_eq!(last.if_none_match.as_deref(), Some("*")); + assert_eq!(last.if_match, None); + assert!(mock.object().is_some(), "the create must have stored bytes"); + + mock.shutdown(); + } + + #[tokio::test] + async fn upload_if_absent_over_an_existing_object_is_precondition_lost() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + mock_put(&mock_s3_client(mock.endpoint()), b"seed", None, None) + .await + .expect("seeding PUT"); + + let dir = payload_dir("late-backfill"); + let err = client + .upload("owner", "repo", dir.path(), UploadPrecondition::IfAbsent) + .await + .expect_err("create-only over an existing object must be refused"); + assert!( + matches!(err, UploadError::PreconditionLost { status: 412 }), + "got {err:?}" + ); + assert_eq!( + mock.object().as_deref(), + Some(b"seed".as_slice()), + "a refused backfill must not clobber what is already published" + ); + + mock.shutdown(); + } + + #[tokio::test] + async fn upload_unconditional_overwrites_regardless() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + mock_put(&mock_s3_client(mock.endpoint()), b"seed", None, None) + .await + .expect("seeding PUT"); + + let dir = payload_dir("overwrite"); + client + .upload( + "owner", + "repo", + dir.path(), + UploadPrecondition::Unconditional, + ) + .await + .expect("an unconditional upload must succeed regardless of state"); + + let last = mock.put_attempts().last().cloned().expect("one attempt"); + assert_eq!(last.if_match, None, "no precondition may be sent"); + assert_eq!(last.if_none_match, None); + assert_ne!( + mock.object().as_deref(), + Some(b"seed".as_slice()), + "the unconditional upload must have replaced the seed" + ); + + mock.shutdown(); + } + + /// Tigris answers a create-only conflict with 409 rather than 412, so that + /// status has to classify as a lost precondition too, but ONLY when the + /// request was create-only. + #[tokio::test] + async fn upload_classifies_409_under_if_absent_as_precondition_lost() { + let (endpoint, server) = start_fixed_status_stub(409).await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + let dir = payload_dir("conflict"); + + let err = client + .upload("owner", "repo", dir.path(), UploadPrecondition::IfAbsent) + .await + .expect_err("409 must be an error"); + assert!( + matches!(err, UploadError::PreconditionLost { status: 409 }), + "409 under IfAbsent is a lost precondition, got {err:?}" + ); + + server.abort(); + } + + /// The P1 raw-response case: a conditional PUT refused with an UNPARSABLE + /// 409/412 body (malformed XML, premature close). The SDK cannot map that to + /// a modeled service error, so it surfaces as `SdkError::ResponseError`, and + /// the status has to be read off the raw response, not off a + /// `ServiceError`-only match. A lost precondition reported as `Other` here + /// would make `RepoWriteGuard::release` log-and-succeed instead of taking the + /// supersede retry, acknowledging a write that was definitively not + /// published. + #[tokio::test] + async fn upload_classifies_an_unparsable_409_as_precondition_lost() { + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any(|| async { + ( + axum::http::StatusCode::CONFLICT, + "this is not xml, so the sdk cannot model an error from it", + ) + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + let client = TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + let dir = payload_dir("unparsable-conflict"); + + let err = client + .upload("owner", "repo", dir.path(), UploadPrecondition::IfAbsent) + .await + .expect_err("409 must be an error"); + assert!( + matches!(err, UploadError::PreconditionLost { status: 409 }), + "an unparsable 409 under IfAbsent is still a lost precondition, got {err:?}" + ); + + server.abort(); + } + + #[tokio::test] + async fn upload_classifies_an_unparsable_412_as_precondition_lost() { + let app = axum::Router::new().route( + "/{*key}", + axum::routing::any(|| async { + (axum::http::StatusCode::PRECONDITION_FAILED, "also not xml") + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + let client = TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + let dir = payload_dir("unparsable-stale"); + + let err = client + .upload( + "owner", + "repo", + dir.path(), + UploadPrecondition::IfMatch("\"stale\"".to_string()), + ) + .await + .expect_err("412 must be an error"); + assert!( + matches!(err, UploadError::PreconditionLost { status: 412 }), + "an unparsable 412 is a lost precondition, got {err:?}" + ); + + server.abort(); + } + + /// P2 (fork orphan): `release_after_write` must surface a refused create-only + /// upload as `PreconditionLost`, not swallow it as success. The fork handler + /// relies on that to refuse creating a DB record whose archive is shadowed by + /// an orphan (a failed `create_repo` left bytes under the key, or another + /// writer got there first). Without the propagation, the fork reports success + /// and every other node fetches the unrelated archive. + #[tokio::test] + async fn release_after_write_refuses_when_the_key_already_exists() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + // Seed an orphan under the fork's would-be key. + mock_put(&mock_s3_client(mock.endpoint()), b"orphan", None, None) + .await + .expect("seeding the orphan archive"); + + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + // release_after_write("owner", "repo") uploads from local_path = + // /owner/repo.git, so the bare repo must exist there for the + // compress to have anything to read. + let local = repos_dir.join("owner").join("repo.git"); + std::fs::create_dir_all(local.parent().unwrap()).unwrap(); + store::init_bare(&local).expect("a bare repo to upload"); + + let store = RepoStore::new( + repos_dir, + Some(client), + sqlx::PgPool::connect_lazy(&std::env::var("DATABASE_URL").unwrap()).unwrap(), + Duration::from_secs(300), + ); + // The upload key is owner-slug/repo: mock_put seeded + // "repos/v1/owner/repo.tar.zst", and an owner_did of "owner" has no + // colons so its slug is exactly "owner" and the IfAbsent upload is + // refused against the seeded key. + let err = store + .release_after_write("owner", "repo", &PublishAttemptId::new()) + .await + .expect_err("a create-only upload over an existing key must be refused"); + assert!( + matches!(err, UploadError::PreconditionLost { .. }), + "the fork upload must surface the lost precondition, got {err:?}" + ); + assert_eq!( + mock.object().as_deref(), + Some(b"orphan".as_slice()), + "the refused upload must not have replaced the orphan" + ); + + mock.shutdown(); + } + + /// MUST-NOT. A 404 is permanent (no such bucket, a misrouted endpoint), so + /// reporting it as a lost precondition would tell a client to retry + /// something that can never succeed. Archive keys are never deleted by the + /// write path, so a racing delete cannot produce this. + /// + /// It must land as `NotPublished` rather than `Ambiguous`: a 4xx is an answer + /// the server gave BEFORE storing anything, so it does prove the write did + /// not commit, which is what licenses a caller to compensate. + #[tokio::test] + async fn upload_classifies_404_as_a_definite_non_publication() { + let (endpoint, server) = start_fixed_status_stub(404).await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + + for precondition in [ + UploadPrecondition::IfAbsent, + UploadPrecondition::IfMatch("\"whatever\"".to_string()), + ] { + let dir = payload_dir("gone"); + let err = client + .upload("owner", "repo", dir.path(), precondition.clone()) + .await + .expect_err("404 must be an error"); + assert!( + matches!(err, UploadError::NotPublished(_)), + "404 under {precondition:?} must NOT be a lost precondition, got {err:?}" + ); + assert!( + err.proves_not_published(), + "a 404 is a definite refusal, so compensation is licensed" + ); + } + + server.abort(); + } + + /// THE P2 SPLIT. A 5xx says the server FAILED, not that it did not commit: + /// an S3-compatible store can accept and durably record a conditional PUT and + /// then fail while producing the response. Classifying that as a definite + /// failure is what let guarded writes invalidate their cache and fork + /// creation drop its only local clone for a write that had landed. + #[tokio::test] + async fn upload_classifies_500_as_ambiguous_not_definite_failure() { + let (endpoint, server) = start_fixed_status_stub(500).await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + + for precondition in [ + UploadPrecondition::IfAbsent, + UploadPrecondition::IfMatch("\"whatever\"".to_string()), + UploadPrecondition::Unconditional, + ] { + let dir = payload_dir("boom"); + let err = client + .upload("owner", "repo", dir.path(), precondition.clone()) + .await + .expect_err("500 must be an error"); + assert!( + matches!(err, UploadError::Ambiguous { .. }), + "500 under {precondition:?} must be ambiguous, got {err:?}" + ); + assert!( + !err.proves_not_published(), + "a 5xx must never license destructive compensation, got {err:?}" + ); + } + + server.abort(); + } + + #[tokio::test] + async fn head_etag_reports_the_current_etag_and_none_when_absent() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + + assert_eq!( + client.head_etag("owner", "repo").await.expect("HEAD"), + None, + "an absent object must read as None, not an error" + ); + + let seeded = mock_put(&mock_s3_client(mock.endpoint()), b"seed", None, None) + .await + .expect("seeding PUT"); + let got = client + .head_etag("owner", "repo") + .await + .expect("HEAD") + .expect("a present object must report an ETag"); + assert_eq!( + unquote_etag(&got), + unquote_etag(&seeded), + "head_etag must report the ETag the last successful PUT minted" + ); + + mock.shutdown(); + } + + // ── the fenced release publish (#279) ────────────────────────────────── + + /// A store whose acquire-side refresh and release-side publish both land on + /// `mock`. The transfer bound is generous on purpose: these tests are about + /// the fence arms, and a short bound would let the timeout arm answer first. + async fn fenced_store( + mock: &S3Mock, + opts: &sqlx::postgres::PgConnectOptions, + repos_dir: &Path, + ) -> RepoStore { + RepoStore::new( + repos_dir.to_path_buf(), + Some(TigrisClient::for_testing_with_endpoint( + "test-bucket", + mock.endpoint(), + )), + no_reap_pool(opts, 2).await, + std::time::Duration::from_secs(30), + ) + } + + /// A client aimed at the same key the store under test publishes to, so a + /// test can seed the archive or land an interfering publish of its own. + fn mock_tigris(mock: &S3Mock) -> TigrisClient { + TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()) + } + + /// The slug `local_path` derives from a DID, needed because a test seeds + /// and reads the archive key directly. + fn owner_slug_of(owner_did: &str) -> String { + owner_did.replace([':', '/'], "_") + } + + /// A bare-repo-shaped directory carrying `marker`, so a test can tell whose + /// tree is stored without comparing compressed bytes. + fn marked_repo(path: &Path, marker: &str) { + seed_bare_repo(path); + std::fs::write(path.join("MARKER"), marker).unwrap(); + } + + /// The marker inside whatever archive is currently stored under the key. + async fn stored_marker(mock: &S3Mock, owner_slug: &str, repo_name: &str) -> String { + let out = TempDir::new().unwrap(); + let validated = super::validated_repo_disk_path(out.path(), "did:key:stored", "stored") + .expect("test repo path must validate"); + mock_tigris(mock) + .download(owner_slug, repo_name, &validated, None) + .await + .expect("the stored archive must be readable"); + std::fs::read_to_string(validated.join("MARKER")) + .expect("the stored archive must be marked") + } + + /// A process-wide sink for warn-level tracing output, installed once. + /// + /// Global rather than per test on purpose. `tracing`'s scoped default is + /// thread-local, and these events fire inside futures the test runtime may + /// move between threads, so a scoped subscriber would drop them silently + /// and every log assertion would go vacuous. Tests instead give their repo + /// a unique name and read back only the lines carrying it. + fn log_sink() -> Arc>> { + static LOG_SINK: std::sync::OnceLock>>> = + std::sync::OnceLock::new(); + LOG_SINK + .get_or_init(|| { + let sink = Arc::new(std::sync::Mutex::new(Vec::new())); + let writer = sink.clone(); + // `try_init`, because another test may already have installed a + // subscriber; the assertions below fail loudly if nothing was + // captured, so a silent no-op here cannot pass for a green run. + let _ = tracing_subscriber::fmt() + .with_writer(move || SinkWriter(writer.clone())) + .with_ansi(false) + .with_max_level(tracing::Level::WARN) + .try_init(); + sink + }) + .clone() + } + + struct SinkWriter(Arc>>); + + impl std::io::Write for SinkWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + /// Captured warn lines naming `repo_name`, joined back into one string. + fn warn_lines_for(repo_name: &str) -> String { + let raw = log_sink().lock().unwrap().clone(); + String::from_utf8_lossy(&raw) + .lines() + .filter(|l| l.contains(repo_name)) + .collect::>() + .join("\n") + } + + /// An uncontended write publishes, and what lands is the writer's tree. + /// + /// This is the must-not-spuriously-fence negative, so it asserts ONLY the + /// outcome and the stored bytes, never which precondition header travelled. + /// Forcing the carried precondition back to `Unconditional` has to leave it + /// green, or it is a second copy of the fix rather than a guard against it; + /// the header itself is pinned by + /// `upload_if_match_with_the_current_etag_publishes_and_sends_the_header`. + #[sqlx::test] + async fn uncontended_write_publishes_the_writers_tree(pool: PgPool) { + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkFenceUncontended"; + let slug = owner_slug_of(owner); + + // Seed an archive so the acquire takes the download arm, which is the + // ordinary case: the repo already exists in object storage. + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload( + &slug, + "repo", + seed.path(), + UploadPrecondition::Unconditional, + ) + .await + .expect("seeding the archive"); + + let guard = store.acquire_write(owner, "repo").await.expect("acquire"); + std::fs::write(guard.local_path.join("MARKER"), "writer").unwrap(); + guard + .release(true) + .await + .into_result() + .expect("an uncontended write must publish"); + + assert_eq!( + stored_marker(&mock, &slug, "repo").await, + "writer", + "an uncontended write must publish the writer's tree" + ); + + mock.shutdown(); + } + + /// The first write to an empty bucket, the absent-at-acquire path. Nothing + /// is stored when the lock is taken, so the publish is the one that creates + /// the key, and the writer's tree is what lands. + #[sqlx::test] + async fn first_write_to_an_empty_bucket_publishes_the_writers_tree(pool: PgPool) { + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkFenceFirstWrite"; + let slug = owner_slug_of(owner); + + let guard = store.acquire_write(owner, "repo").await.expect("acquire"); + marked_repo(&guard.local_path, "writer"); + guard + .release(true) + .await + .into_result() + .expect("the first write into an empty key must publish"); + + assert_eq!( + stored_marker(&mock, &slug, "repo").await, + "writer", + "the first write must publish the writer's tree into the empty key" + ); + + mock.shutdown(); + } + + /// THE INIT RACE, and the reason the fence needs a supersede-retry at all. + /// + /// `init` uploads a freshly created EMPTY repo create-only in the + /// background. A user who pushes immediately after creating a repo takes + /// the lock, sees nothing stored, and is fenced create-only too; the + /// background upload then wins the empty key and the push's publish loses. + /// A fence with no retry would turn every such push into a refusal. + /// + /// The retry is sound because the loss is DEFINITE and this writer still + /// holds the lock: what landed underneath was published without it, so this + /// tree supersedes it. + #[sqlx::test] + async fn a_lost_fence_republishes_once_and_the_writers_tree_wins(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkFenceInitRace"; + let repo = "fence-init-race-repo"; + let slug = owner_slug_of(owner); + + // Nothing is stored yet, so the acquire records the absent case. + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + marked_repo(&guard.local_path, "writer"); + + // init's create-only background upload lands after that observation. + let empty = TempDir::new().unwrap(); + marked_repo(empty.path(), "empty-init"); + mock_tigris(&mock) + .upload(&slug, repo, empty.path(), UploadPrecondition::IfAbsent) + .await + .expect("the background init upload wins the empty key"); + let before = mock.put_attempts().len(); + assert_eq!(before, 1, "only the init upload has run so far"); + + guard + .release(true) + .await + .into_result() + .expect("the supersede-retry must leave the release reporting success"); + + let attempts = mock.put_attempts(); + assert_eq!( + attempts.len(), + before + 2, + "the release must attempt exactly twice, the fenced publish and one \ + supersede-retry, got {attempts:?}" + ); + assert_eq!( + attempts[before].status, + Some(412), + "the create-only publish must lose to what landed underneath, got {attempts:?}" + ); + assert_eq!( + attempts[before + 1].status, + Some(200), + "the supersede-retry must publish, got {attempts:?}" + ); + assert_eq!( + stored_marker(&mock, &slug, repo).await, + "writer", + "the lock holder's tree must be what is stored after the retry" + ); + assert!( + warn_lines_for(repo).contains("republishing"), + "the fired fence must be visible in the log, got {:?}", + warn_lines_for(repo) + ); + + mock.shutdown(); + } + + /// Two consecutive definite losses: the retry is bounded at ONE, so the + /// release refuses instead of escalating, and the refusal reaches the + /// caller rather than being logged and swallowed. + /// + /// The second loss is arranged by replacing the object right after the + /// re-HEAD answers, so the retry fences on a generation that is already + /// gone. That is fault injection at the only point where it can be + /// deterministic; the mock's PUT gate cannot do it, because a parked PUT is + /// captured rather than evaluated and answers 200. + #[sqlx::test] + async fn a_second_consecutive_loss_refuses_and_never_attempts_a_third(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkFenceDoubleLoss"; + let repo = "fence-double-loss-repo"; + let slug = owner_slug_of(owner); + + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional) + .await + .expect("seeding the archive"); + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + std::fs::write(guard.local_path.join("MARKER"), "writer").unwrap(); + + // An unlocked publish lands after the acquire, so the carried fence is + // already stale before the release runs. + let orphan = TempDir::new().unwrap(); + marked_repo(orphan.path(), "orphan"); + mock_tigris(&mock) + .upload( + &slug, + repo, + orphan.path(), + UploadPrecondition::Unconditional, + ) + .await + .expect("an unconditional publish always lands"); + let before = mock.put_attempts().len(); + assert_eq!(before, 2, "the seed and the orphan have run so far"); + + // ... and the generation the retry HEADs for moves on before its PUT + // can use it, so the second attempt loses too. + mock.roll_generation_after_next_heads(1); + let local_path = guard.local_path.clone(); + let outcome = guard.release(true).await; + + assert!( + matches!(outcome, ReleaseOutcome::Fenced), + "a publish refused twice must be reported to the caller, got {outcome:?}" + ); + assert!( + !local_path.exists(), + "a fenced publish must invalidate the local read cache" + ); + let attempts = mock.put_attempts(); + assert_eq!( + attempts.len(), + before + 2, + "a release must attempt at most TWO publishes, never a third, got {attempts:?}" + ); + assert_eq!( + (attempts[before].status, attempts[before + 1].status), + (Some(412), Some(412)), + "both attempts must have been refused by the store, got {attempts:?}" + ); + let logged = warn_lines_for(repo); + assert!( + logged.contains("republishing"), + "the first loss must log the retry, got {logged:?}" + ); + assert!( + logged.contains("refusing the write"), + "the second loss must log its own distinct refusal, got {logged:?}" + ); + + mock.shutdown(); + } + + /// Driven from the client side at a publishing site: a refused publish must + /// render as the retryable 503 and never as a success body. + /// + /// `create_issue` bumps the author's trust score AFTER releasing the guard, + /// so an unchanged score is what proves the short-circuit actually precedes + /// the post-release effects rather than merely being written above them. A + /// `?` placed after the bump would leave the status assertion green and + /// this one red. + #[sqlx::test] + async fn a_fenced_publish_renders_as_503_and_skips_the_post_release_effects(pool: PgPool) { + use tower::ServiceExt; + + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let owner = "did:key:z6MkFenceHandlerAuthor"; + let repo = "fence-handler-repo"; + let slug = owner_slug_of(owner); + + let mut state = crate::test_support::test_state(pool.clone()).await; + state.repo_store = fenced_store(&mock, &opts, repos.path()).await; + let now = chrono::Utc::now(); + state + .db + .create_repo(&crate::db::RepoRecord { + id: uuid::Uuid::new_v4().to_string(), + name: repo.to_string(), + owner_did: owner.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/tmp/{repo}"), + forked_from: None, + machine_id: None, + }) + .await + .expect("seed repo"); + // The trust bump only moves a row that already exists, so the author has + // to be registered or the observable would be vacuously unchanged. + state + .db + .register_agent(owner, &[]) + .await + .expect("register the author"); + let score_before = state.db.get_trust_score(owner).await.expect("trust score"); + + // A real bare repo, on disk and published, so the acquire refresh has a + // valid archive to download and the handler's git work succeeds. + let local = repos.path().join(&slug).join(format!("{repo}.git")); + store::init_bare(&local).expect("init the bare repo"); + mock_tigris(&mock) + .upload(&slug, repo, &local, UploadPrecondition::Unconditional) + .await + .expect("publish the archive"); + + // Move the generation on after BOTH of the handler's HEADs: the one in + // `acquire_write`, so the fence it carries is stale by the time it + // publishes, and the one the supersede-retry does, so the retry loses + // too and the release refuses. + mock.roll_generation_after_next_heads(2); + + let router = axum::Router::new() + .route( + "/api/v1/repos/{owner}/{repo}/issues", + axum::routing::post(crate::api::issues::create_issue), + ) + .with_state(state.clone()); + let resp = router + .oneshot(crate::test_support::signed_request_as( + owner, + axum::http::Method::POST, + &format!("/api/v1/repos/{owner}/{repo}/issues"), + axum::body::Body::from(r#"{"title":"t","body":"b"}"#), + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "a publish the store refused must be a retryable 503, not a success" + ); + let body = axum::body::to_bytes(resp.into_body(), 64 * 1024) + .await + .expect("body"); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("repo_write_fenced"), + "the 503 must carry its own code so a client can tell it from contention, got {body}" + ); + assert!( + !body.contains(repo) && !body.contains(&slug), + "the body must be fixed and must not name the repo or owner, got {body}" + ); + assert_eq!( + state.db.get_trust_score(owner).await.expect("trust score"), + score_before, + "the post-release trust bump must not run when the publish was refused" + ); + + mock.shutdown(); + } + + // ── the abandoned writer's late PUT (#279) ───────────────────────────── + + /// A store whose under-lock transfer bound is short, so a parked PUT + /// actually runs the release past its budget instead of making the test sit + /// out `fenced_store`'s 30s. The acquire-side refresh shares the bound, + /// which is fine here: it moves a few KiB against an in-process mock. + async fn fenced_store_with_bound( + mock: &S3Mock, + opts: &sqlx::postgres::PgConnectOptions, + repos_dir: &Path, + bound: std::time::Duration, + ) -> RepoStore { + RepoStore::new( + repos_dir.to_path_buf(), + Some(TigrisClient::for_testing_with_endpoint( + "test-bucket", + mock.endpoint(), + )), + no_reap_pool(opts, 2).await, + bound, + ) + } + + /// The statuses of every logged PUT attempt, which is what the + /// abandoned-writer tests assert their attempt counts on. + fn attempt_statuses(mock: &S3Mock) -> Vec> { + mock.put_attempts().iter().map(|a| a.status).collect() + } + + /// THE HEADLINE ARM. An abandoned writer's PUT that lands after a successor + /// has published must be refused by the store, and the successor's archive + /// must survive it. + /// + /// This is the whole point of the change. Dropping the future of an + /// in-flight PUT does not cancel the request the server is already + /// processing, so the advisory lock cannot fence it: A's release returns, + /// the lock frees, B acquires and publishes, and A's bytes are still on + /// their way to a store that has already moved on. Only the conditional PUT + /// decides that race, and it decides it at COMMIT time, not at arrival. + /// + /// The mock models that by capturing A's PUT when it arrives and evaluating + /// it on replay, against the state as of the replay. That is deliberately + /// not a timing test: a parked handler whose client has gone away is + /// cancelled with the connection, so a test that waited for it to resume on + /// its own would be waiting on nothing. + #[sqlx::test] + async fn an_abandoned_writers_late_put_loses_to_the_successor(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkFenceLatePut"; + let repo = "fence-late-put-repo"; + let slug = owner_slug_of(owner); + + // Seed the key, so A's acquire observes a generation and carries + // If-Match on it. This is the ordinary case: the repo already exists. + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional) + .await + .expect("seeding the archive"); + let seeded = mock.current_etag().expect("the seed minted an ETag"); + + // Writer A takes the lock, writes its tree, and has its publish parked + // past the transfer bound. + let guard_a = store.acquire_write(owner, repo).await.expect("A acquires"); + std::fs::write(guard_a.local_path.join("MARKER"), "writer-a").unwrap(); + mock.park_next_put(); + let started = std::time::Instant::now(); + let outcome_a = guard_a.release(true).await; + assert!( + started.elapsed() >= bound, + "A's release must have run out its transfer bound with the PUT in flight" + ); + // A timeout is UNKNOWABLE rather than failed, so the release frees the + // lock but must not report durability to the caller. That is exactly why + // the fence has to live in the store: nothing here knows A's bytes are + // still coming. + assert!( + matches!(outcome_a, ReleaseOutcome::UploadUnknowable), + "an abandoned publish must report unknowable durability, got {outcome_a:?}" + ); + assert!( + outcome_a.into_result().is_err(), + "unknowable upload must not report success to the caller" + ); + + assert!( + mock.captured_put().is_some(), + "A's PUT must have arrived and been captured before the bound elapsed" + ); + + // Writer B acquires the freed lock and publishes for real. + let b_started = std::time::Instant::now(); + let guard_b = store + .acquire_write(owner, repo) + .await + .expect("B must acquire once A's release frees the lock"); + assert!( + b_started.elapsed() < std::time::Duration::from_secs(5), + "B's acquire must be prompt, not blocked behind A's abandoned transfer" + ); + std::fs::write(guard_b.local_path.join("MARKER"), "writer-b").unwrap(); + guard_b + .release(true) + .await + .into_result() + .expect("B's publish is the one that must land"); + let after_b = mock.current_etag().expect("B's publish minted an ETag"); + assert_ne!( + unquote_etag(&after_b), + unquote_etag(&seeded), + "B's publish must have moved the generation on" + ); + + // NOW A's bytes reach the store's commit point. + assert_eq!( + mock.replay_captured(), + 412, + "A's late PUT must be refused: the generation it fenced on is gone" + ); + assert_eq!( + stored_marker(&mock, &slug, repo).await, + "writer-b", + "the successor's archive must survive the abandoned writer's late PUT" + ); + assert_eq!( + mock.current_etag().as_deref().map(unquote_etag), + Some(unquote_etag(&after_b)), + "a refused PUT must not rotate the generation either" + ); + + // Seed, A's parked PUT, B's publish, the deliberate replay. A never + // attempted a second PUT of its own: the timeout arm takes no + // compensating action precisely because the outcome is unknowable. + assert_eq!( + attempt_statuses(&mock), + vec![Some(200), None, Some(200), Some(412)], + "got {:?}", + mock.put_attempts() + ); + + // The header detail comes LAST on purpose. Asserting it up front would + // make a lost fence red here, on a wire-format check, rather than on the + // outcome above, and the outcome is what this test is for. + let captured = mock.captured_put().expect("A's PUT was captured"); + assert_eq!( + captured.if_match.as_deref().map(unquote_etag), + Some(unquote_etag(&seeded)), + "A's in-flight PUT must carry the generation it observed under the lock" + ); + assert_eq!(captured.if_none_match, None); + + mock.open_gate(); + mock.shutdown(); + } + + /// The create-only arm of the same race, and the one whose real-world + /// failure mode is SILENT: an ignored If-None-Match just returns 200, so a + /// publish that should have been fenced lands with no error anywhere. + #[sqlx::test] + async fn an_abandoned_writers_late_create_only_put_loses_to_the_successor(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkFenceLateCreate"; + let repo = "fence-late-create-repo"; + let slug = owner_slug_of(owner); + + // Nothing stored, so A's acquire records the absent case and is fenced + // create-only. + let guard_a = store.acquire_write(owner, repo).await.expect("A acquires"); + marked_repo(&guard_a.local_path, "writer-a"); + mock.park_next_put(); + let outcome_a = guard_a.release(true).await; + assert!( + matches!(outcome_a, ReleaseOutcome::UploadUnknowable), + "an abandoned publish must report unknowable durability, got {outcome_a:?}" + ); + + assert!( + mock.captured_put().is_some(), + "A's PUT must have arrived and been captured before the bound elapsed" + ); + + // B wins the empty key. + let guard_b = store + .acquire_write(owner, repo) + .await + .expect("B must acquire once A's release frees the lock"); + std::fs::write(guard_b.local_path.join("MARKER"), "writer-b").unwrap(); + guard_b + .release(true) + .await + .into_result() + .expect("B's create must land"); + + assert_eq!( + mock.replay_captured(), + 412, + "A's late create-only PUT must be refused now that the key exists" + ); + assert_eq!( + stored_marker(&mock, &slug, repo).await, + "writer-b", + "the successor's archive must survive the abandoned create-only PUT" + ); + assert_eq!( + attempt_statuses(&mock), + vec![None, Some(200), Some(412)], + "got {:?}", + mock.put_attempts() + ); + + // Last, for the same reason as the If-Match arm: the outcome is the + // claim, the header is the detail. + let captured = mock.captured_put().expect("A's PUT was captured"); + assert_eq!( + captured.if_none_match.as_deref(), + Some("*"), + "A's in-flight PUT must carry the create-only fence it observed" + ); + assert_eq!(captured.if_match, None); + + mock.open_gate(); + mock.shutdown(); + } + + /// THE CONTROL, and it is what makes the two arms above attributable. + /// + /// Same abandonment, same replay, but no successor publishes in between, so + /// the generation A fenced on is still current when its bytes commit and + /// the PUT must LAND. Without this, a green headline test would prove only + /// that replays are rejected, not that STALENESS is what rejects them. + /// + /// It must therefore stay green when the carried precondition is forced + /// back to `Unconditional`: it asserts an outcome the fence does not + /// change, which is the whole reason it can attribute the others' red. + #[sqlx::test] + async fn an_abandoned_put_still_on_the_current_generation_lands(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkFenceControl"; + let repo = "fence-control-repo"; + let slug = owner_slug_of(owner); + + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional) + .await + .expect("seeding the archive"); + + let guard_a = store.acquire_write(owner, repo).await.expect("A acquires"); + std::fs::write(guard_a.local_path.join("MARKER"), "writer-a").unwrap(); + mock.park_next_put(); + let outcome_a = guard_a.release(true).await; + assert!( + matches!(outcome_a, ReleaseOutcome::UploadUnknowable), + "an abandoned publish must report unknowable durability, got {outcome_a:?}" + ); + + assert_eq!( + mock.replay_captured(), + 200, + "with nothing published in between, the abandoned PUT is still current \ + and must be accepted" + ); + assert_eq!( + stored_marker(&mock, &slug, repo).await, + "writer-a", + "the accepted late PUT must be what is stored" + ); + assert_eq!( + attempt_statuses(&mock), + vec![Some(200), None, Some(200)], + "got {:?}", + mock.put_attempts() + ); + + mock.open_gate(); + mock.shutdown(); + } + + // ── attempt identity and publication stage (#285) ────────────────────── + + /// The mock has to MODEL attempt metadata or every reconciliation test below + /// is vacuous: a HEAD that never echoes what the PUT stamped would make + /// `attempt_landed` answer `false` unconditionally and the fix would look + /// like it worked by doing nothing. + #[tokio::test] + async fn mock_round_trips_the_attempt_metadata_a_put_stamped() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + + let dir = payload_dir("stamped"); + let attempt = PublishAttemptId::new(); + let receipt = client + .upload_tracked( + "owner", + "repo", + dir.path(), + UploadPrecondition::IfAbsent, + attempt.clone(), + None, + ) + .await + .expect("the create must publish"); + assert_eq!(receipt.attempt, attempt); + assert!( + receipt.etag.is_some(), + "an acknowledged PUT carries the generation it minted" + ); + assert_eq!(mock.stored_attempt().as_deref(), Some(attempt.as_str())); + + let stored = client + .head_generation("owner", "repo") + .await + .expect("HEAD") + .expect("an object is stored"); + assert!( + stored.belongs_to(&attempt), + "HEAD must report the attempt the PUT stamped, got {stored:?}" + ); + assert!( + !stored.belongs_to(&PublishAttemptId::new()), + "a different attempt must not match" + ); + assert!(client + .attempt_landed("owner", "repo", &attempt) + .await + .expect("reconcile")); + + mock.shutdown(); + } + + /// P2 FINDING 4, THE HEADLINE. A server that accepts the COMPLETE conditional + /// PUT, commits it, and then loses or corrupts the response before the SDK + /// can return success. + /// + /// Classifying that as a definite failure is what made fork creation drop its + /// only local clone and skip the DB insert for a write that HAD landed, after + /// which every retry saw the orphan object under `If-None-Match: *` and + /// returned `RepoExists` — the fork name unusable until an operator cleaned + /// up. The outcome must be ambiguous, and it must be RECONCILABLE: the + /// attempt id travelled with the bytes, so the client can ask the store what + /// it holds. + #[tokio::test] + async fn a_put_that_commits_and_loses_its_response_is_ambiguous_and_reconcilable() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + + let dir = payload_dir("committed-but-unreported"); + let attempt = PublishAttemptId::new(); + mock.commit_then_lose_next_put_response(); + let stage = PublishStageCell::new(); + let err = client + .upload_tracked( + "owner", + "repo", + dir.path(), + UploadPrecondition::IfAbsent, + attempt.clone(), + Some(&stage), + ) + .await + .expect_err("the client never saw a success response"); + + assert!( + matches!(err, UploadError::Ambiguous { .. }), + "a lost response is not proof of failure, got {err:?}" + ); + assert!( + !err.proves_not_published(), + "compensating this would delete a write that landed" + ); + assert_eq!( + stage.get(), + PublishStage::Ambiguous { + attempt: attempt.clone() + }, + "the stage must record the attempt whose fate is unresolved" + ); + + // The write IS durable, and the attempt id is what proves it. + assert!( + client + .attempt_landed("owner", "repo", &attempt) + .await + .expect("reconcile"), + "the store committed this attempt's bytes, so reconciliation must recover it" + ); + assert_eq!(mock.stored_attempt().as_deref(), Some(attempt.as_str())); + + mock.shutdown(); + } + + /// The other half of "closes or corrupts": a peer that reads the whole + /// request and then drops the socket without answering at all. There is no + /// HTTP status to classify on, so the SdkError variant is all there is — and + /// only a construction failure proves the request never left. + #[tokio::test] + async fn a_closed_response_with_no_http_status_is_ambiguous() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let server = tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + tokio::spawn(async move { + use tokio::io::AsyncReadExt; + // Consume what the client sends — the request IS delivered — + // then hang up without a response. + let mut buf = vec![0u8; 64 * 1024]; + let _ = tokio::time::timeout( + std::time::Duration::from_millis(250), + sock.read(&mut buf), + ) + .await; + drop(sock); + }); + } + }); + + let client = TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint); + let dir = payload_dir("no-answer"); + let err = client + .upload("owner", "repo", dir.path(), UploadPrecondition::IfAbsent) + .await + .expect_err("a dropped connection must not read as success"); + assert!( + matches!(err, UploadError::Ambiguous { .. }), + "a request delivered with no response read must stay ambiguous, got {err:?}" + ); + assert!(!err.proves_not_published()); + + server.abort(); + } + + /// The conditional delete must be atomic, not merely narrowed. A successor + /// publishing between the ownership HEAD and the DELETE is the exact race a + /// "look it up again first" guard leaves open, and `roll_generation_after_ + /// next_heads` is the seam that sits in that window deterministically. + #[tokio::test] + async fn a_generation_that_moves_between_the_head_and_the_delete_is_not_deleted() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + let attempt = PublishAttemptId::new(); + + mock_put_as( + &mock_s3_client(mock.endpoint()), + b"ours", + None, + None, + Some(attempt.as_str()), + ) + .await + .expect("seed this attempt's object"); + + // The ownership HEAD sees our attempt and our ETag; the store then moves + // on before the DELETE fenced on that ETag arrives. + mock.roll_generation_after_next_heads(1); + let outcome = client + .delete_if_attempt_matches("owner", "repo", &attempt) + .await + .expect("a refused conditional delete is an outcome, not an error"); + assert_eq!( + outcome, + AttemptDelete::NotOurs, + "a delete whose generation moved under it must be refused, not retried blind" + ); + assert_eq!(mock.deletes(), 0, "nothing may have been deleted"); + assert!(mock.object().is_some(), "the object must survive"); + + mock.shutdown(); + } + + /// The must-do direction: an attempt's OWN orphan is still cleanable, or a + /// failed fork would tombstone its name forever. + #[tokio::test] + async fn an_attempts_own_object_is_deleted_and_a_foreign_one_is_not() { + let mock = S3Mock::start().await; + let client = TigrisClient::for_testing_with_endpoint("test-bucket", mock.endpoint()); + let mine = PublishAttemptId::new(); + let theirs = PublishAttemptId::new(); + + assert_eq!( + client + .delete_if_attempt_matches("owner", "repo", &mine) + .await + .expect("delete"), + AttemptDelete::Absent, + "an empty key is nothing to clean up" + ); + + mock_put_as( + &mock_s3_client(mock.endpoint()), + b"theirs", + None, + None, + Some(theirs.as_str()), + ) + .await + .expect("seed a foreign object"); + assert_eq!( + client + .delete_if_attempt_matches("owner", "repo", &mine) + .await + .expect("delete"), + AttemptDelete::NotOurs + ); + assert!(mock.object().is_some(), "a foreign object must survive"); + + mock_put_as( + &mock_s3_client(mock.endpoint()), + b"mine", + None, + None, + Some(mine.as_str()), + ) + .await + .expect("replace it with ours"); + assert_eq!( + client + .delete_if_attempt_matches("owner", "repo", &mine) + .await + .expect("delete"), + AttemptDelete::Deleted + ); + assert!(mock.object().is_none(), "our own orphan must be removable"); + + mock.shutdown(); + } + + /// P2 FINDING 4, the fork path. `release_after_write` is fork creation's + /// publish, and a lost response there must reach the handler as ambiguous so + /// it can recover the committed attempt instead of dropping its clone. + #[sqlx::test] + async fn fork_publish_that_loses_its_response_stays_recoverable(pool: PgPool) { + let mock = S3Mock::start().await; + let tmp = TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + let local = repos_dir.join("owner").join("repo.git"); + std::fs::create_dir_all(local.parent().unwrap()).unwrap(); + store::init_bare(&local).expect("a bare repo to upload"); + + let store = RepoStore::new( + repos_dir, + Some(mock_tigris(&mock)), + pool.clone(), + Duration::from_secs(30), + ); + let attempt = PublishAttemptId::new(); + mock.commit_then_lose_next_put_response(); + let err = store + .release_after_write("owner", "repo", &attempt) + .await + .expect_err("the client saw no success"); + assert!( + !err.proves_not_published(), + "the fork must not treat a lost response as licence to delete its clone, got {err:?}" + ); + assert!( + store + .fork_attempt_landed("owner", "repo", &attempt) + .await + .expect("reconcile"), + "the archive IS published under this attempt, so the fork must be recoverable \ + rather than fenced behind its own orphan" + ); + + mock.shutdown(); + } + + /// P1 FINDING 3, the object half, driven through the REAL recovery path. + /// + /// The interleaving: the background recovery observed no row for this fork + /// name (the DB below has none), a successor then committed and published + /// under that name, and only afterwards did the recovery resume its cleanup. + /// Before the attempt guard, that cleanup deleted the successor's archive and + /// directory after the successor had already answered 201. + #[sqlx::test] + async fn fork_recovery_resuming_after_a_successor_deletes_neither_object_nor_path( + pool: PgPool, + ) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let tmp = TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + let store = RepoStore::new( + repos_dir.clone(), + Some(mock_tigris(&mock)), + pool.clone(), + Duration::from_secs(30), + ); + + let failed = PublishAttemptId::new(); + let successor = PublishAttemptId::new(); + + // The successor owns both resources by the time cleanup resumes. + mock_put_as( + &mock_s3_client(mock.endpoint()), + b"successor-archive", + None, + None, + Some(successor.as_str()), + ) + .await + .expect("the successor published"); + let disk_path = repos_dir.join("owner").join("repo.git"); + std::fs::create_dir_all(&disk_path).unwrap(); + std::fs::write(disk_path.join("HEAD"), b"successor").unwrap(); + claim_fork_disk_path(&disk_path, &successor); + + // The failed attempt resumes its compensation. + store + .compensate_fork_archive("owner", "repo", &disk_path, &failed) + .await; + + assert_eq!( + mock.deletes(), + 0, + "a failed attempt must not delete the successor's archive" + ); + assert_eq!( + mock.stored_attempt().as_deref(), + Some(successor.as_str()), + "the successor's object must still be what is stored" + ); + assert!( + disk_path.exists(), + "a failed attempt must not delete the successor's repository directory" + ); + assert_eq!( + std::fs::read_to_string(disk_path.join("HEAD")).unwrap(), + "successor" + ); + + mock.shutdown(); + } + + /// The must-do direction of the same guard: an attempt that still owns both + /// resources must be able to clean up after itself, or a failed fork leaves + /// an orphan that fences its own name. + #[sqlx::test] + async fn fork_compensation_removes_what_this_attempt_still_owns(pool: PgPool) { + let mock = S3Mock::start().await; + let tmp = TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + let store = RepoStore::new( + repos_dir.clone(), + Some(mock_tigris(&mock)), + pool.clone(), + Duration::from_secs(30), + ); + + let attempt = PublishAttemptId::new(); + mock_put_as( + &mock_s3_client(mock.endpoint()), + b"my-orphan", + None, + None, + Some(attempt.as_str()), + ) + .await + .expect("this attempt published"); + let disk_path = repos_dir.join("owner").join("repo.git"); + std::fs::create_dir_all(&disk_path).unwrap(); + claim_fork_disk_path(&disk_path, &attempt); + + store + .compensate_fork_archive("owner", "repo", &disk_path, &attempt) + .await; + + assert!( + mock.object().is_none(), + "this attempt's orphan must be gone" + ); + assert!(!disk_path.exists(), "this attempt's clone must be gone"); + + mock.shutdown(); + } + + // ── the quarantined generation (#285 finding 2) ──────────────────────── + + /// A store whose Tigris compression parks on `gate`, so a publish can be + /// stalled at `PublishStage::PreparingArchive` — the window in which no + /// request has been constructed, let alone sent. + async fn compression_gated_store( + mock: &S3Mock, + opts: &sqlx::postgres::PgConnectOptions, + repos_dir: &Path, + bound: std::time::Duration, + gate: Arc, + ) -> RepoStore { + RepoStore::new( + repos_dir.to_path_buf(), + Some(mock_tigris(mock).with_compress_gate(gate)), + no_reap_pool(opts, 2).await, + bound, + ) + } + + /// P1 FINDING 1, at the release boundary. A publish whose bound expires while + /// the archive is still being COMPRESSED never constructed a request, so it + /// is a definite non-publication — not the unknowable in-flight PUT the + /// timeout arm used to report for every stall alike. + /// + /// The observable is the mock's PUT log: zero attempts. That is what + /// distinguishes this from the parked-PUT tests above, where exactly one + /// request arrived. + #[sqlx::test] + async fn a_bound_that_expires_before_dispatch_is_a_definite_failure(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let gate = Arc::new(crate::git::tigris::BlockingGate::shut()); + let owner = "did:key:z6MkStageCompress"; + let repo = "stage-compress-repo"; + + // The acquire-side refresh must run BEFORE the gate closes: it does no + // compression of its own, but the store is shared and holding the gate + // early would prove nothing about the release. + let store = compression_gated_store( + &mock, + &opts, + repos.path(), + std::time::Duration::from_millis(600), + Arc::clone(&gate), + ) + .await; + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + marked_repo(&guard.local_path, "writer"); + let local_path = guard.local_path.clone(); + + let outcome = guard.release(true).await; + + assert!( + matches!(outcome, ReleaseOutcome::UploadFailed), + "a bound that expired before any PUT was dispatched is a DEFINITE failure, \ + got {outcome:?}" + ); + assert!( + mock.put_attempts().is_empty(), + "no request may have reached the store, got {:?}", + mock.put_attempts() + ); + assert!( + !local_path.exists(), + "a definite non-publication must invalidate the local write cache; only an \ + unresolved dispatch may leave the tree in place" + ); + assert!( + warn_lines_for(repo).contains("before any PUT was dispatched"), + "the distinct verdict must be visible in the log, got {:?}", + warn_lines_for(repo) + ); + + // Let the parked blocking thread finish so it is not stranded. + gate.open(); + mock.shutdown(); + } + + /// P1 FINDING 2. A bounded release whose PUT is in flight leaves the writer's + /// tree at the ORDINARY live path with no confirmed generation behind it. The + /// writer gets its 503 — and then a same-node read must not hand that tree + /// out as an ordinary successful read. + /// + /// Deleting the tree is deliberately not the fix and is asserted against: the + /// PUT may have landed, and this can be the only local copy of it. + #[sqlx::test] + async fn an_unresolved_publish_quarantines_the_tree_and_a_later_read_refuses(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkQuarantineRead"; + let repo = "quarantine-read-repo"; + let slug = owner_slug_of(owner); + + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional) + .await + .expect("seeding the archive"); + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + std::fs::write(guard.local_path.join("MARKER"), "unresolved").unwrap(); + let local_path = guard.local_path.clone(); + mock.park_next_put(); + let outcome = guard.release(true).await; + + assert!( + matches!(outcome, ReleaseOutcome::UploadUnknowable), + "the PUT was consumed and left unresolved, got {outcome:?}" + ); + assert!( + outcome.into_result().is_err(), + "the writer must get a retryable refusal, not a 2xx" + ); + assert!( + local_path.exists(), + "the tree must NOT be deleted: the PUT may have landed and this could be the \ + only local copy" + ); + + // THE READ. Pre-fix this returned the live path on filesystem existence + // alone and served the unresolved refs indefinitely. + let read = store.acquire(owner, repo).await; + let err = read.expect_err("an unconfirmed tree must not be served as an ordinary read"); + assert!( + err.downcast_ref::().is_some(), + "the refusal must be the retryable one, got {err:#}" + ); + assert!( + local_path.exists(), + "refusing must not delete the tree either" + ); + + mock.open_gate(); + mock.shutdown(); + } + + /// The release valve, and what keeps the quarantine from being a permanent + /// outage: once the abandoned PUT actually commits, the stored object carries + /// THIS attempt's id, reconciliation says so, and the read is served. + #[sqlx::test] + async fn a_quarantined_tree_is_served_once_the_store_confirms_the_attempt(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkQuarantineLift"; + let repo = "quarantine-lift-repo"; + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + marked_repo(&guard.local_path, "unresolved"); + let local_path = guard.local_path.clone(); + mock.park_next_put(); + assert!(matches!( + guard.release(true).await, + ReleaseOutcome::UploadUnknowable + )); + assert!( + store.acquire(owner, repo).await.is_err(), + "refused while unresolved" + ); + + // The abandoned PUT reaches the store's commit point and lands. + assert_eq!(mock.replay_captured(), 200); + + let served = store + .acquire(owner, repo) + .await + .expect("a confirmed attempt must lift the quarantine"); + assert_eq!(served, local_path.as_path()); + assert!( + store.acquire(owner, repo).await.is_ok(), + "the marker must have been cleared, not re-evaluated on every read" + ); + + mock.open_gate(); + mock.shutdown(); + } + + /// THE CONTROL. An ordinary confirmed write must leave no quarantine behind, + /// or every read after every push would refuse. It asserts an outcome the + /// quarantine does not change, which is what lets the two tests above + /// attribute their red. + #[sqlx::test] + async fn a_confirmed_publish_leaves_the_tree_readable(pool: PgPool) { + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkQuarantineControl"; + let repo = "quarantine-control-repo"; + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + marked_repo(&guard.local_path, "clean"); + guard + .release(true) + .await + .into_result() + .expect("an uncontended publish lands"); + + store + .acquire(owner, repo) + .await + .expect("a confirmed write must be readable with no reconciliation at all"); + + mock.shutdown(); + } + + /// P2 FINDING 4, at a GUARDED WRITE. The review names this arm explicitly: + /// "guarded issue/push writes also take definite-failure cache and + /// compensation paths despite not knowing whether their generation landed." + /// + /// The store here accepts the complete PUT, commits it, and then loses the + /// response. Pre-fix that arrived as `UploadError::Other`, which `release` + /// read as `UploadFailed` and answered by deleting the local tree and running + /// the caller's compensator (`create_issue` deletes the issue ref it just + /// wrote) — undoing a write that IS durable in object storage, so the next + /// reader downloads an archive containing the "undone" work. + /// + /// The correct outcome is a retryable refusal with the tree kept and + /// quarantined, and — because the attempt id travelled with the bytes — a + /// later read that RECONCILES and is served. + #[sqlx::test] + async fn a_guarded_write_whose_response_is_lost_is_not_compensated(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let store = fenced_store(&mock, &opts, repos.path()).await; + let owner = "did:key:z6MkLostResponseWrite"; + let repo = "lost-response-repo"; + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + marked_repo(&guard.local_path, "writer"); + let local_path = guard.local_path.clone(); + + let compensated = Arc::new(std::sync::atomic::AtomicBool::new(false)); + mock.commit_then_lose_next_put_response(); + let outcome = { + let compensated = Arc::clone(&compensated); + guard + .release_compensating(true, move |_path| { + compensated.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }) + .await + }; + + assert!( + matches!(outcome, ReleaseOutcome::UploadUnknowable), + "a lost response is not a definite failure, got {outcome:?}" + ); + assert!( + outcome.into_result().is_err(), + "the writer still must not be told it succeeded" + ); + assert!( + !compensated.load(std::sync::atomic::Ordering::SeqCst), + "RED: the caller's undo ran for a write that IS published — create_issue would \ + have deleted the issue ref that other nodes will fetch" + ); + assert!( + local_path.exists(), + "RED: the local tree was invalidated for a write that landed" + ); + + // ...and because the attempt travelled with the bytes, the quarantine is + // answerable rather than a standing outage. + store + .acquire(owner, repo) + .await + .expect("the committed attempt must reconcile and be served"); + + mock.shutdown(); + } + + /// A write that follows an unresolved one heals the path: the under-lock + /// refresh replaces the tree with the stored generation, so the quarantine it + /// inherited is answered rather than carried forever. + #[sqlx::test] + async fn the_next_write_clears_an_inherited_quarantine(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let opts = (*pool.connect_options()).clone(); + let repos = TempDir::new().unwrap(); + let bound = std::time::Duration::from_millis(750); + let store = fenced_store_with_bound(&mock, &opts, repos.path(), bound).await; + let owner = "did:key:z6MkQuarantineHeal"; + let repo = "quarantine-heal-repo"; + let slug = owner_slug_of(owner); + + let seed = TempDir::new().unwrap(); + marked_repo(seed.path(), "seed"); + mock_tigris(&mock) + .upload(&slug, repo, seed.path(), UploadPrecondition::Unconditional) + .await + .expect("seeding the archive"); + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + std::fs::write(guard.local_path.join("MARKER"), "unresolved").unwrap(); + mock.park_next_put(); + assert!(matches!( + guard.release(true).await, + ReleaseOutcome::UploadUnknowable + )); + assert!(store.acquire(owner, repo).await.is_err()); + + // A successor takes the freed lock; its refresh downloads the confirmed + // archive over the quarantined tree. + let successor = store.acquire_write(owner, repo).await.expect("successor"); + std::fs::write(successor.local_path.join("MARKER"), "successor").unwrap(); + successor + .release(true) + .await + .into_result() + .expect("the successor publishes"); + + store + .acquire(owner, repo) + .await + .expect("the healed path must read normally again"); + + mock.open_gate(); + mock.shutdown(); + } + + // ── fork creation through the handler (#285 findings 3 and 4) ────────── + + /// A state whose repo store publishes to `mock`, plus a PUBLIC source repo + /// that is already on disk and already in object storage — so the fork's own + /// create-only PUT is the first and only PUT the handler makes, and a + /// one-shot response-loss flag can be aimed at it. + async fn fork_state( + mock: &S3Mock, + pool: &PgPool, + repos_dir: &Path, + source_owner: &str, + source_name: &str, + ) -> crate::state::AppState { + let opts = (*pool.connect_options()).clone(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.repo_store = fenced_store(mock, &opts, repos_dir).await; + // `fork_repo` derives the clone's destination from `config.repos_dir`, + // not from the store, so the two have to agree or the validated join + // rejects the default (relative) config path. + let mut config = (*state.config).clone(); + config.repos_dir = repos_dir.to_path_buf(); + state.config = Arc::new(config); + + let now = chrono::Utc::now(); + state + .db + .create_repo(&crate::db::RepoRecord { + id: uuid::Uuid::new_v4().to_string(), + name: source_name.to_string(), + owner_did: source_owner.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/unused/{source_name}"), + forked_from: None, + machine_id: None, + }) + .await + .expect("seed the source repo row"); + + let slug = owner_slug_of(source_owner); + let source_path = repos_dir.join(&slug).join(format!("{source_name}.git")); + store::init_bare(&source_path).expect("a real bare source repo"); + // Publish the SOURCE key so `acquire` finds it already migrated and does + // not lazily upload it; the fork's PUT must be the only one. + mock_tigris(mock) + .upload( + &slug, + source_name, + &source_path, + UploadPrecondition::Unconditional, + ) + .await + .expect("seed the source archive"); + state + } + + async fn do_fork( + state: &crate::state::AppState, + source_owner: &str, + source_name: &str, + forker: &str, + fork_name: &str, + ) -> std::result::Result<(axum::http::StatusCode, String), crate::error::AppError> { + crate::api::repos::fork_repo( + axum::extract::State(state.clone()), + axum::Extension(crate::auth::AuthenticatedDid(forker.to_string())), + axum::extract::Path(( + crate::db::normalize_owner_key(source_owner).to_string(), + source_name.to_string(), + )), + axum::http::HeaderMap::new(), + axum::Json(crate::api::repos::ForkRepoRequest { + name: Some(fork_name.to_string()), + }), + ) + .await + .map(|(status, body)| (status, body.0.id)) + } + + /// P2 FINDING 4, THE USER-VISIBLE FAILURE MODE. The fork's create-only PUT + /// commits, and the response is lost before the SDK can report success. + /// + /// Pre-fix that arrived as a definite failure: `ForkCloneGuard` removed the + /// only local clone and no DB row was inserted, leaving an orphan object + /// under the fork's key. Every retry then sent `If-None-Match: *`, saw the + /// orphan and answered `RepoExists` — the fork name unusable until an + /// operator cleaned up. The attempt id stamped into the object is what turns + /// "did my request succeed" (undecidable) into "are the published bytes mine" + /// (decidable), so the committed attempt is RECOVERED. + #[sqlx::test] + async fn a_fork_whose_publish_lost_its_response_is_recovered_not_fenced(pool: PgPool) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let repos = TempDir::new().unwrap(); + let source_owner = "did:key:z6MkForkSourceAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let forker = "did:key:z6MkForkerLostRespAAAAAAAAAAAAAAAAAAAAA"; + let state = fork_state(&mock, &pool, repos.path(), source_owner, "src").await; + + mock.commit_then_lose_next_put_response(); + let (status, id) = do_fork(&state, source_owner, "src", forker, "recovered") + .await + .expect("a committed publish must not be reported as a failure"); + + assert_eq!(status, axum::http::StatusCode::CREATED); + let row = state + .db + .get_repo(crate::db::normalize_owner_key(forker), "recovered") + .await + .expect("lookup") + .expect("the fork row must have been inserted"); + assert_eq!(row.id, id); + + // The object is the fork's, stamped with the row that owns it. + let stored = mock + .object_for(&owner_slug_of(forker), "recovered") + .expect("the fork archive is published"); + assert_eq!( + stored.attempt.as_deref(), + Some(row.id.as_str()), + "the published archive must name the row that owns it" + ); + assert!( + repos + .path() + .join(owner_slug_of(forker)) + .join("recovered.git") + .exists(), + "the fork's clone must still be on disk" + ); + + // ...and the name is NOT fenced: it resolves to a real repository. + assert!(state + .db + .get_repo_by_id(&row.id) + .await + .expect("lookup") + .is_some()); + + mock.shutdown(); + } + + /// The other half of the ambiguity, where the client's knowledge is + /// IDENTICAL: the PUT was delivered in full and failed, and this time it did + /// not commit. + /// + /// Nothing may be destroyed on this path either — the request could still be + /// in flight — so the answer is a retryable refusal with the only local clone + /// left in place, not a deletion. + #[sqlx::test] + async fn a_fork_whose_publish_is_unresolved_keeps_its_clone_and_refuses_retryably( + pool: PgPool, + ) { + let _sink = log_sink(); + let mock = S3Mock::start().await; + let repos = TempDir::new().unwrap(); + let source_owner = "did:key:z6MkForkSourceBBBBBBBBBBBBBBBBBBBBBBBBBB"; + let forker = "did:key:z6MkForkerUnresolvedAAAAAAAAAAAAAAAAAAA"; + let state = fork_state(&mock, &pool, repos.path(), source_owner, "src").await; + + mock.fail_next_put_after_delivery(); + let err = do_fork(&state, source_owner, "src", forker, "unresolved") + .await + .expect_err("an unresolved publish must not report success"); + assert!( + matches!(err, crate::error::AppError::RepoUnavailable), + "the refusal must be the retryable one, got {err:?}" + ); + + assert!( + repos + .path() + .join(owner_slug_of(forker)) + .join("unresolved.git") + .exists(), + "RED: the only local copy of a write that MAY have landed was deleted" + ); + assert_eq!( + mock.deletes(), + 0, + "nothing may be deleted while the outcome is unresolved" + ); + assert!( + state + .db + .get_repo(crate::db::normalize_owner_key(forker), "unresolved") + .await + .expect("lookup") + .is_none(), + "no row may be inserted for a publish that is not confirmed" + ); + + mock.shutdown(); + } + + /// THE CONTROL. An ordinary fork must still work end to end, or the two + /// assertions above would pass on a handler that simply never forks. + #[sqlx::test] + async fn an_ordinary_fork_publishes_and_commits(pool: PgPool) { + let mock = S3Mock::start().await; + let repos = TempDir::new().unwrap(); + let source_owner = "did:key:z6MkForkSourceCCCCCCCCCCCCCCCCCCCCCCCCCC"; + let forker = "did:key:z6MkForkerPlainAAAAAAAAAAAAAAAAAAAAAAAA"; + let state = fork_state(&mock, &pool, repos.path(), source_owner, "src").await; + + let (status, id) = do_fork(&state, source_owner, "src", forker, "plain") + .await + .expect("an uncontended fork must succeed"); + assert_eq!(status, axum::http::StatusCode::CREATED); + let stored = mock + .object_for(&owner_slug_of(forker), "plain") + .expect("the fork archive is published"); + assert_eq!(stored.attempt.as_deref(), Some(id.as_str())); + + mock.shutdown(); + } } diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 5617b4198..9ca16e88f 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -870,6 +870,7 @@ pub fn merge_branch( } /// Resolve a repo disk path: {repos_dir}/{owner_slug}/{repo_name}.git +#[allow(dead_code)] // exercised from test_support and state tests; production uses validated_repo_disk_path pub fn repo_disk_path(repos_dir: &Path, owner_did: &str, repo_name: &str) -> PathBuf { // Sanitize the DID for use as a directory name let owner_slug = owner_did.replace([':', '/'], "_"); diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index cf7abfd5f..6a0d3006c 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -5,17 +5,121 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; use std::sync::{Arc, Mutex, OnceLock}; use anyhow::{Context, Result}; use aws_sdk_s3::Client as S3Client; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; + +// The publication vocabulary lives in `git::publish`, which carries no +// object-storage types, and is re-exported here so existing `git::tigris::…` +// imports keep resolving. #79 deletes this file; `git::publish` is what survives +// the swap, and the only backend-aware code below is `classify_dispatch`. +pub use super::publish::{ + DispatchKnowledge, PublishAttemptId, PublishStage, PublishStageCell, StoredGeneration, + UploadError, UploadPrecondition, UploadReceipt, ATTEMPT_METADATA_KEY, +}; + +/// What happened to a conditional, attempt-guarded delete. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AttemptDelete { + /// The object was this attempt's and is gone. + Deleted, + /// Nothing is stored under the key. + Absent, + /// Something is stored, and it is NOT this attempt's work. Left alone. + NotOurs, +} + +/// THE ONLY BACKEND-AWARE CLASSIFIER in the publish path. +/// +/// It answers one question: does this failure PROVE the request never committed? +/// Anything short of proof is [`DispatchKnowledge::MaybeSent`], because the +/// caller's next move on a definite failure is destructive (delete the object, +/// drop the only local clone, invalidate the cache) and being wrong there +/// destroys published state. +/// +/// - An HTTP status means the server answered. A 4xx is a refusal it took +/// BEFORE storing anything, so it proves non-publication. A 5xx does not: the +/// store can commit and then fail to report it. +/// - No status at all means no response was read. Only a construction failure +/// proves the request never left; Smithy's timeout, dispatch and response +/// errors all allow that the bytes arrived and committed while the answer was +/// lost, corrupted, or never parsed. +/// - `SdkError` is `#[non_exhaustive]`, so the fallback arm must be the CAUTIOUS +/// one. A future variant defaulting to "definitely failed" would silently +/// re-open exactly the class this closes. +/// +/// #79 replaces `tigris.rs` with a `BlobStore` layer; this function is the piece +/// each backend reimplements, and nothing above it changes. +fn classify_put_failure( + err: &aws_sdk_s3::error::SdkError, + status: Option, +) -> DispatchKnowledge { + if let Some(status) = status { + return if (400..500).contains(&status) { + DispatchKnowledge::NeverSent + } else { + DispatchKnowledge::MaybeSent + }; + } + match err { + aws_sdk_s3::error::SdkError::ConstructionFailure(_) => DispatchKnowledge::NeverSent, + _ => DispatchKnowledge::MaybeSent, + } +} /// Wrapper around the S3 client with the configured bucket. #[derive(Clone)] pub struct TigrisClient { s3: S3Client, bucket: String, + /// Test-only seam: when set, the blocking compression inside `upload_tracked` + /// parks on this barrier. That is the ONLY window in which a publish attempt + /// can be cancelled with `PublishStage::PreparingArchive` still holding, and + /// it is unreachable from outside without a seam because compression of a + /// test-sized repo finishes in microseconds. + #[cfg(test)] + compress_gate: Option>, +} + +/// A gate a BLOCKING thread parks on until a test opens it. +/// +/// A condvar rather than a held `MutexGuard`: the test's assertions run while the +/// gate is shut, and holding a `std::sync::MutexGuard` across those awaits is +/// both a lint violation and a real hazard. Here the test owns no guard at all — +/// it flips a flag and notifies. +#[cfg(test)] +#[derive(Default)] +pub struct BlockingGate { + open: Mutex, + opened: std::sync::Condvar, +} + +#[cfg(test)] +impl BlockingGate { + /// A gate that starts SHUT. + pub fn shut() -> Self { + Self::default() + } + + fn wait(&self) { + let mut open = self.open.lock().expect("compression gate poisoned"); + while !*open { + open = self + .opened + .wait(open) + .expect("compression gate poisoned while waiting"); + } + } + + /// Let every parked compression through. Call this at teardown so the + /// blocking thread is not stranded for the life of the process. + pub fn open(&self) { + *self.open.lock().expect("compression gate poisoned") = true; + self.opened.notify_all(); + } } impl TigrisClient { @@ -28,28 +132,48 @@ impl TigrisClient { Ok(Self { s3, bucket: bucket.to_string(), + #[cfg(test)] + compress_gate: None, }) } - /// Test-only constructor with an explicit S3 endpoint, region, and static - /// credentials — no env-var reads, so parallel tests cannot race each other's - /// `AWS_*` environment the way the env-based `new` would. Lets a test point - /// the client at a non-routable endpoint to exercise acquire-stall paths. + /// Build a client pointed at an arbitrary endpoint, for tests. + /// + /// The production constructor reads the endpoint and credentials from the + /// environment, which a test cannot steer without mutating process-global + /// state. This takes both explicitly so a test can aim the client at a + /// closed port and get a prompt transport error out of `exists()`. + /// + /// `RetryConfig::disabled()` is load-bearing, not tidiness: the SDK's default + /// policy retries a connection refusal with backoff, which turns each failing + /// call into seconds of waiting. #[cfg(test)] - pub(crate) async fn for_testing_with_endpoint(bucket: &str, endpoint_url: &str) -> Self { - let creds = aws_sdk_s3::config::Credentials::new("test", "test", None, None, "test"); - let config = aws_config::defaults(aws_config::BehaviorVersion::latest()) - .endpoint_url(endpoint_url) - .region(aws_config::Region::new("auto")) - .credentials_provider(creds) - .load() - .await; + pub fn for_testing_with_endpoint(bucket: &str, endpoint: &str) -> Self { + use aws_sdk_s3::config::{retry::RetryConfig, Credentials, Region}; + + let config = aws_sdk_s3::config::Config::builder() + .endpoint_url(endpoint) + .credentials_provider(Credentials::new("test", "test", None, None, "test")) + .region(Region::new("auto")) + .retry_config(RetryConfig::disabled()) + .behavior_version_latest() + .build(); Self { - s3: S3Client::new(&config), + s3: S3Client::from_conf(config), bucket: bucket.to_string(), + compress_gate: None, } } + /// Test-only: park this client's blocking compression on `gate` until the + /// holder releases it, so a cancellation can be aimed at + /// [`PublishStage::PreparingArchive`] rather than at the PUT. + #[cfg(test)] + pub fn with_compress_gate(mut self, gate: Arc) -> Self { + self.compress_gate = Some(gate); + self + } + /// S3 key for a given repo: `repos/v1/{owner_slug}/{repo_name}.tar.zst` fn repo_key(owner_slug: &str, repo_name: &str) -> String { format!("repos/v1/{owner_slug}/{repo_name}.tar.zst") @@ -77,34 +201,325 @@ impl TigrisClient { } } - /// Upload a local bare repo directory to Tigris as a tar.zst archive. - pub async fn upload(&self, owner_slug: &str, repo_name: &str, local_path: &Path) -> Result<()> { + /// Read the ETag of a repo archive, or `None` when nothing is stored under + /// the key. The ETag identifies the generation a later conditional upload + /// can fence itself on. + /// + /// Separate from `exists` rather than folded into it: `exists` has callers + /// that only want the boolean, and widening its return type would churn + /// every one of them for no benefit. + pub async fn head_etag(&self, owner_slug: &str, repo_name: &str) -> Result> { + Ok(self + .head_generation(owner_slug, repo_name) + .await? + .map(|g| g.etag) + .unwrap_or(None)) + } + + /// The full stored generation: the ETag AND the attempt that published it. + /// + /// The attempt half is what makes a lost response recoverable. An ETag alone + /// answers "has the generation moved", which every writer sees the same way; + /// the attempt id answers "are the published bytes MINE", which is the + /// question a client whose PUT response vanished actually needs answered. + pub async fn head_generation( + &self, + owner_slug: &str, + repo_name: &str, + ) -> Result> { + let key = Self::repo_key(owner_slug, repo_name); + match self + .s3 + .head_object() + .bucket(&self.bucket) + .key(&key) + .send() + .await + { + Ok(out) => { + let etag = out + .e_tag() + .context(format!("tigris HEAD {key}: hit carried no ETag"))? + .to_string(); + let attempt = out + .metadata() + .and_then(|m| m.get(ATTEMPT_METADATA_KEY)) + .cloned(); + Ok(Some(StoredGeneration { + etag: Some(etag), + attempt, + })) + } + Err(e) => { + if e.as_service_error().is_some_and(|e| e.is_not_found()) { + Ok(None) + } else { + Err(anyhow::anyhow!("tigris HEAD {key}: {e}")) + } + } + } + } + + /// Did `attempt`'s bytes land? The reconciliation an ambiguous dispatch owes + /// before anything downstream may treat it as confirmation. + /// + /// `Ok(false)` is deliberately NOT "the PUT failed": an abandoned request may + /// still be in flight, so a negative answer licenses refusing and retrying, + /// never deleting. Only `Ok(true)` upgrades an unresolved attempt to + /// published. + pub async fn attempt_landed( + &self, + owner_slug: &str, + repo_name: &str, + attempt: &PublishAttemptId, + ) -> Result { + Ok(self + .head_generation(owner_slug, repo_name) + .await? + .is_some_and(|g| g.belongs_to(attempt))) + } + + /// Upload a local bare repo directory to Tigris as a tar.zst archive, + /// fenced by `precondition`, under a freshly minted attempt identity. + pub async fn upload( + &self, + owner_slug: &str, + repo_name: &str, + local_path: &Path, + precondition: UploadPrecondition, + ) -> std::result::Result { + self.upload_tracked( + owner_slug, + repo_name, + local_path, + precondition, + PublishAttemptId::new(), + None, + ) + .await + } + + /// The full form: a caller-chosen attempt identity, and a stage cell the + /// upload reports its progress into. + /// + /// The stage cell is the answer to "a dropped future never returns an + /// outcome". Compression runs inside `spawn_blocking` and the conditional PUT + /// is not even constructed until it finishes, so a handler cancelled during + /// compression definitely never attempted publication — but nothing could + /// observe that, because the only report was the return value of a future + /// that no longer exists. Marking [`PublishStage::PreparingArchive`] before + /// the blocking call and [`PublishStage::PutDispatched`] immediately before + /// `send()` makes that boundary readable from outside. + pub async fn upload_tracked( + &self, + owner_slug: &str, + repo_name: &str, + local_path: &Path, + precondition: UploadPrecondition, + attempt: PublishAttemptId, + stage: Option<&PublishStageCell>, + ) -> std::result::Result { let key = Self::repo_key(owner_slug, repo_name); - debug!(key = %key, path = %local_path.display(), "uploading repo to tigris"); + debug!(key = %key, path = %local_path.display(), attempt = %attempt, "uploading repo to tigris"); + + if let Some(stage) = stage { + stage.set(PublishStage::PreparingArchive); + } - // Create tar.zst in memory - let archive_bytes = tokio::task::spawn_blocking({ + // Create tar.zst in memory. Nothing has been sent at this point and + // nothing can be: the request below is not constructed until this + // returns. A failure or a cancellation here is a definite + // non-publication. + let compressed = { let local_path = local_path.to_path_buf(); - move || compress_repo(&local_path) - }) - .await - .context("tar task panicked")? - .context("compressing repo")?; + #[cfg(test)] + let gate = self.compress_gate.clone(); + tokio::task::spawn_blocking(move || { + // Test-only seam: park INSIDE the blocking compression, which is + // the window a handler can be cancelled in while + // `PublishStage::PreparingArchive` still holds. + #[cfg(test)] + if let Some(gate) = gate { + // Blocks until the test opens the gate. + gate.wait(); + } + compress_repo(&local_path) + }) + .await + }; + let archive_bytes = match compressed { + Ok(Ok(bytes)) => bytes, + Ok(Err(e)) => { + if let Some(stage) = stage { + stage.set(PublishStage::Refused); + } + return Err(UploadError::NotPublished(e.context("compressing repo"))); + } + Err(e) => { + if let Some(stage) = stage { + stage.set(PublishStage::Refused); + } + return Err(UploadError::NotPublished( + anyhow::Error::new(e).context("tar task panicked"), + )); + } + }; let body = aws_sdk_s3::primitives::ByteStream::from(archive_bytes); - self.s3 + let mut req = self + .s3 .put_object() .bucket(&self.bucket) .key(&key) + // The attempt identity travels WITH the bytes. That is what makes a + // lost response recoverable: the client can HEAD the key afterwards + // and ask whether the published object is its own work, which is + // decidable, instead of asking whether its request succeeded, which + // is not. + .metadata(ATTEMPT_METADATA_KEY, attempt.as_str()) .body(body) - .content_type("application/zstd") - .send() - .await - .context(format!("tigris PUT {key}"))?; + .content_type("application/zstd"); + match &precondition { + UploadPrecondition::IfMatch(etag) => req = req.if_match(etag), + UploadPrecondition::IfAbsent => req = req.if_none_match("*"), + UploadPrecondition::Unconditional => {} + } - info!(key = %key, "uploaded repo to tigris"); - Ok(()) + // From HERE the bytes may reach the store. Everything downstream that + // could destroy state has to treat this stage as "maybe published". + if let Some(stage) = stage { + stage.set(PublishStage::PutDispatched { + attempt: attempt.clone(), + }); + } + + let sent = req.send().await; + let out = match sent { + Ok(out) => out, + Err(e) => { + // `PutObjectError` models no PreconditionFailed variant (its arms are + // EncryptionTypeMismatch, InvalidRequest, InvalidWriteOffset, + // TooManyParts, Unhandled), so a refused precondition arrives as + // `Unhandled` and matching the enum would classify it as a generic + // failure. The raw HTTP status off the response is the only place the + // answer actually lives. + // + // Read it via `raw_response()`, not a `ServiceError`-only match: the + // SDK exposes the raw response for BOTH `ServiceError` and + // `ResponseError`, and a refused conditional PUT whose error body the + // SDK cannot parse (malformed XML, premature close) surfaces as + // `ResponseError`. Matching only `ServiceError` would classify that + // unparsable 409/412 as a generic failure, and `RepoWriteGuard::release` + // would log-and-succeed instead of taking the supersede retry, + // acknowledging a write that was definitively not published. + let status = e.raw_response().map(|raw| raw.status().as_u16()); + // 412 is always a lost precondition. 409 is one only when we asked + // for create-only, which is how S3-compatible stores report "the key + // already exists". Everything else, 404 included, is a real failure: + // archive keys are never deleted by the write path, so a 404 here + // means something permanent like a missing bucket or a misrouted + // endpoint, and reporting that as a lost precondition would tell a + // caller to expect a successor that does not exist. + let lost = match status { + Some(412) => true, + Some(409) => matches!(precondition, UploadPrecondition::IfAbsent), + _ => false, + }; + if lost { + if let Some(stage) = stage { + stage.set(PublishStage::Refused); + } + return Err(UploadError::PreconditionLost { + status: status.expect("a lost precondition came from a status"), + }); + } + let knowledge = classify_put_failure(&e, status); + let ctx = anyhow::Error::new(e).context(format!("tigris PUT {key}")); + return Err(match knowledge { + DispatchKnowledge::NeverSent => { + if let Some(stage) = stage { + stage.set(PublishStage::Refused); + } + UploadError::NotPublished(ctx) + } + DispatchKnowledge::MaybeSent => { + warn!( + key = %key, + attempt = %attempt, + status = ?status, + "tigris PUT failed WITHOUT proving it did not commit — the outcome \ + is ambiguous and must be reconciled, not compensated" + ); + if let Some(stage) = stage { + stage.set(PublishStage::Ambiguous { + attempt: attempt.clone(), + }); + } + UploadError::Ambiguous { + attempt: attempt.clone(), + source: ctx, + } + } + }); + } + }; + + let etag = out.e_tag().map(str::to_string); + if let Some(stage) = stage { + stage.set(PublishStage::Published { + attempt: attempt.clone(), + etag: etag.clone(), + }); + } + info!(key = %key, attempt = %attempt, "uploaded repo to tigris"); + Ok(UploadReceipt { attempt, etag }) + } + + /// Delete a repo archive ONLY while it is still `attempt`'s work. + /// + /// The unconditional delete this replaces used the logical owner/name as its + /// cleanup authority, so a failed attempt compensating late could erase the + /// object a SUCCESSOR had already published under the same name and already + /// returned 201 for. A second name lookup before the delete only narrows that + /// window; reading the attempt id off the object and fencing the delete on + /// the generation it came from closes it. + pub async fn delete_if_attempt_matches( + &self, + owner_slug: &str, + repo_name: &str, + attempt: &PublishAttemptId, + ) -> Result { + let Some(generation) = self.head_generation(owner_slug, repo_name).await? else { + return Ok(AttemptDelete::Absent); + }; + if !generation.belongs_to(attempt) { + return Ok(AttemptDelete::NotOurs); + } + let key = Self::repo_key(owner_slug, repo_name); + let mut req = self.s3.delete_object().bucket(&self.bucket).key(&key); + // The If-Match guard is what makes this atomic rather than merely + // narrowed: between the HEAD above and this call a successor can publish, + // and the store refusing on the moved generation is the only thing that + // stops the delete landing on their object. + if let Some(etag) = generation.etag.as_deref() { + req = req.if_match(etag); + } + match req.send().await { + Ok(_) => Ok(AttemptDelete::Deleted), + Err(e) => { + // A refused conditional delete means the generation moved: the + // object is no longer ours, which is a successful outcome for a + // guard whose whole job is not to touch somebody else's bytes. + if e.raw_response() + .map(|raw| raw.status().as_u16()) + .is_some_and(|s| s == 412 || s == 409) + { + return Ok(AttemptDelete::NotOurs); + } + Err(anyhow::anyhow!("tigris conditional DELETE {key}: {e}")) + } + } } /// Download a repo archive from Tigris and extract to local disk. @@ -112,10 +527,32 @@ impl TigrisClient { &self, owner_slug: &str, repo_name: &str, - local_path: &Path, + local_path: &super::repo_store::ValidatedRepoDiskPath, + swap_authority: Option>, ) -> Result<()> { + self.download_to(owner_slug, repo_name, local_path, true, swap_authority) + .await + .map(|_| ()) + } + + /// Download a repo archive from Tigris and extract it, returning the + /// directory that was populated. + /// + /// `publish` controls whether the extract is swapped into `target` in place + /// (the live-path mutation used by writes; returns `target`) or unpacked + /// into a fresh temp directory under `target`'s parent (a non-mutating + /// snapshot read; returns the temp dir, which the caller owns and cleans + /// up). The snapshot form never touches the live repo path. + pub async fn download_to( + &self, + owner_slug: &str, + repo_name: &str, + target: &super::repo_store::ValidatedRepoDiskPath, + publish: bool, + swap_authority: Option>, + ) -> Result { let key = Self::repo_key(owner_slug, repo_name); - debug!(key = %key, path = %local_path.display(), "downloading repo from tigris"); + debug!(key = %key, path = %target.as_path().display(), "downloading repo from tigris"); let resp = self .s3 @@ -133,31 +570,59 @@ impl TigrisClient { .context("reading tigris response body")? .into_bytes(); - // Extract tar.zst to local path - tokio::task::spawn_blocking({ - let local_path = local_path.to_path_buf(); - move || decompress_repo(&data, &local_path) + // The snapshot temp dir is decided HERE, in the async layer, before the + // extraction runs. Cleanup ownership moves into the blocking task so a + // cancelled async future cannot drop it while extraction is still running. + let snapshot_tmp = if publish { + None + } else { + let parent = target.parent().context("snapshot path has no parent")?; + std::fs::create_dir_all(parent).context("creating parent dir")?; + let file_name = target + .file_name() + .context("snapshot path has no file name")? + .to_string_lossy(); + Some(parent.join(format!( + ".{file_name}.tmp-snapshot.{}", + uuid::Uuid::new_v4() + ))) + }; + + // Extract tar.zst to a directory. + let extracted = tokio::task::spawn_blocking({ + let target = target.clone(); + let snapshot_tmp = snapshot_tmp.clone(); + move || -> Result { + let result = (|| -> Result { + if publish { + decompress_repo(&data, &target, swap_authority.as_ref())?; + return Ok(DownloadExtract::Published(())); + } + // Non-mutating snapshot: unpack into the temp dir decided above. + // The live repo path is never touched. + let tmp_dir = snapshot_tmp.expect("snapshot path was decided above"); + std::fs::create_dir_all(&tmp_dir).context("creating temp extract dir")?; + let unpack = (|| -> Result<()> { + let decoder = zstd::stream::Decoder::new(&data[..])?; + let mut archive = tar::Archive::new(decoder); + archive.unpack(&tmp_dir).context("unpacking tar.zst")?; + Ok(()) + })(); + if let Err(e) = unpack { + let _ = std::fs::remove_dir_all(&tmp_dir); + return Err(e); + } + Ok(DownloadExtract::Snapshot(TempSnapshotDir { path: tmp_dir })) + })(); + result + } }) .await .context("extract task panicked")? .context("extracting repo")?; - info!(key = %key, path = %local_path.display(), "downloaded repo from tigris"); - Ok(()) - } - - /// Delete a repo archive from Tigris. - #[allow(dead_code)] - pub async fn delete(&self, owner_slug: &str, repo_name: &str) -> Result<()> { - let key = Self::repo_key(owner_slug, repo_name); - self.s3 - .delete_object() - .bucket(&self.bucket) - .key(&key) - .send() - .await - .context(format!("tigris DELETE {key}"))?; - Ok(()) + info!(key = %key, path = %target.as_path().display(), "downloaded repo from tigris"); + Ok(extracted) } } @@ -180,7 +645,7 @@ fn compress_repo(repo_path: &Path) -> Result> { /// `decompress_repo`. Concurrent extractions unpack into isolated temp dirs in /// parallel, but the final `remove_dir_all` + `rename` must not interleave for /// the same `local_path`, or they race to a nondeterministic overwrite/failure. -fn publish_lock(local_path: &Path) -> Arc> { +pub(crate) fn publish_lock(local_path: &Path) -> Arc> { // KNOWN LIMITATION: this map is never evicted — one (PathBuf, Arc) // entry accrues per distinct repo path for the process lifetime. Bounded by // the number of repos a node hosts, so it's negligible for normal use, but @@ -193,6 +658,35 @@ fn publish_lock(local_path: &Path) -> Arc> { .clone() } +/// Owns a snapshot temp dir from extraction through handoff to [`RepoSnapshot`]. +/// Dropped when the `spawn_blocking` join result is abandoned, so cancellation +/// before the outer future resumes still removes the directory. +pub(crate) struct TempSnapshotDir { + path: PathBuf, +} + +impl TempSnapshotDir { + pub(crate) fn into_repo_snapshot(self) -> super::repo_store::RepoSnapshot { + let path = self.path.clone(); + std::mem::forget(self); + super::repo_store::RepoSnapshot::from_owned_path(path) + } +} + +impl Drop for TempSnapshotDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.path); + } +} + +/// What [`TigrisClient::download_to`] produced. +pub enum DownloadExtract { + /// Published into the validated live repo path. + Published(()), + /// Unpacked into a throwaway temp dir that cleans up on drop until adopted. + Snapshot(TempSnapshotDir), +} + /// Decompress a tar.zst byte vector into a local directory. /// /// Extraction is atomic with respect to `local_path`: the archive is unpacked @@ -200,11 +694,16 @@ fn publish_lock(local_path: &Path) -> Arc> { /// fully succeeds. A corrupt or truncated archive therefore can never clobber a /// good existing copy at `local_path` — on failure we discard the temp dir and /// leave `local_path` exactly as it was. -fn decompress_repo(data: &[u8], local_path: &Path) -> Result<()> { - let parent = local_path.parent().context("repo path has no parent")?; +fn decompress_repo( + data: &[u8], + local_path: &super::repo_store::ValidatedRepoDiskPath, + swap_authority: Option<&Arc>, +) -> Result<()> { + let live = local_path.as_path(); + let parent = live.parent().context("repo path has no parent")?; std::fs::create_dir_all(parent).context("creating parent dir")?; - let file_name = local_path + let file_name = live .file_name() .context("repo path has no file name")? .to_string_lossy(); @@ -229,17 +728,218 @@ fn decompress_repo(data: &[u8], local_path: &Path) -> Result<()> { return Err(e); } - // Swap the freshly-extracted repo into place. rename within the same parent - // is effectively atomic, but most platforms refuse to rename onto a - // non-empty dir, so remove the old copy first. Serialize this per repo path: - // concurrent extractions unpack into isolated temp dirs, but their swaps - // must not interleave or they race to a nondeterministic overwrite/failure. - let lock = publish_lock(local_path); - let _publish = lock.lock().expect("publish lock poisoned"); - if local_path.exists() { - std::fs::remove_dir_all(local_path).context("removing stale repo dir")?; - } - std::fs::rename(&tmp_dir, local_path).context("swapping extracted repo into place")?; + // Swap through the validated-path helper so CodeQL sees the barrier before the + // remove/rename sink (`rust/path-injection`). + super::repo_store::swap_extracted_into_validated_repo(local_path, &tmp_dir, swap_authority)?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use aws_sdk_s3::primitives::ByteStream; + use futures::FutureExt; + + /// The envs the probe needs, all of them, or it does not run. + /// + /// `AWS_ENDPOINT_URL_S3` is included on purpose: without it the SDK resolves + /// to real AWS S3, and a probe that passed there would say nothing about + /// Tigris. + fn probe_env() -> Option { + if std::env::var("GITLAWB_TIGRIS_PROBE").ok().as_deref() != Some("1") { + return None; + } + for name in [ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_ENDPOINT_URL_S3", + ] { + if std::env::var(name).is_err() { + eprintln!("tigris conditional-write probe: {name} is unset, skipping"); + return None; + } + } + match std::env::var("GITLAWB_TIGRIS_BUCKET") { + Ok(b) if !b.is_empty() => Some(b), + _ => { + eprintln!( + "tigris conditional-write probe: GITLAWB_TIGRIS_BUCKET is unset, skipping" + ); + None + } + } + } + + /// One conditional PUT, reported as the status that REFUSED it, or `None` + /// when the store accepted the write. + /// + /// Accepted is the interesting answer here, not an error: it means the + /// endpoint ignored the header we fenced on. + async fn conditional_put( + s3: &S3Client, + bucket: &str, + key: &str, + body: &'static [u8], + if_match: Option<&str>, + if_none_match: Option<&str>, + ) -> Result, String> { + let mut req = s3 + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(body)); + if let Some(v) = if_match { + req = req.if_match(v); + } + if let Some(v) = if_none_match { + req = req.if_none_match(v); + } + match req.send().await { + Ok(_) => Ok(None), + Err(e) => match e.raw_response() { + // Same raw-response rule as `upload`: a refused conditional PUT + // whose body the SDK cannot parse surfaces as `ResponseError`, and + // the status has to come off the raw response for both variants. + // Without it, an unparsable 409/412 would report "no HTTP + // response" here and skip the supersede retry. + Some(raw) => Ok(Some(raw.status().as_u16())), + None => Err(format!("conditional PUT {key}: no HTTP response: {e}")), + }, + } + } + + /// The probe body, written to RETURN its failures rather than panic on + /// them, so the caller's cleanup is reached on every arm. + async fn conditional_write_probe(s3: &S3Client, bucket: &str, key: &str) -> Result<(), String> { + // 1. A plain PUT under a throwaway key. + let seeded = s3 + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"probe-one")) + .send() + .await + .map_err(|e| format!("seeding PUT {key}: {e}"))?; + + // 2. Its ETag, which is the generation the next arm fences against. + let etag = seeded + .e_tag() + .ok_or_else(|| format!("seeding PUT {key} returned no ETag"))? + .to_string(); + + // 3. A deliberately wrong If-Match. A store honoring it answers 412. + let wrong = format!("\"{}\"", "0".repeat(32)); + if etag.trim_matches('"') == wrong.trim_matches('"') { + return Err(format!( + "the seeded ETag {etag} collides with the deliberately wrong one, \ + so this arm would prove nothing" + )); + } + match conditional_put(s3, bucket, key, b"probe-two", Some(&wrong), None).await? { + Some(412) => {} + Some(status) => { + return Err(format!( + "a stale If-Match must be refused with 412, the endpoint answered {status}" + )) + } + None => { + return Err( + "a stale If-Match was ACCEPTED: this endpoint does not honor If-Match, so \ + the release fence cannot hold here" + .to_string(), + ) + } + } + + // 4. If-None-Match `*` over the object that now exists. This arm matters + // MORE than the one above. An ignored If-Match eventually surfaces as + // odd behavior, because a stale writer overwrites and someone notices + // the lost tree. An ignored If-None-Match just returns 200, so a publish + // that should have been fenced lands with no error anywhere: the silent + // no-op the bucket-type caveat on this test describes. + match conditional_put(s3, bucket, key, b"probe-three", None, Some("*")).await? { + // Either status is a pass, and the asymmetry with the If-Match arm + // above mirrors `upload`'s classifier exactly: 412 is always a lost + // precondition, and 409 is one too when we asked for create-only. + // AWS documents 409 for a create-only conflict racing a delete, so a + // store answering it is enforcing the precondition and we already + // handle it. Pinning 412 alone here would fail the probe against a + // backend that is behaving correctly, which sends whoever runs it + // chasing a fault that is not there. + Some(412) | Some(409) => {} + Some(status) => { + return Err(format!( + "create-only over an existing object must be refused with 412 or 409, \ + the endpoint answered {status}" + )) + } + None => { + return Err( + "If-None-Match * was ACCEPTED over an existing object: this endpoint does \ + not honor create-only, so a fenced publish lands silently" + .to_string(), + ) + } + } + + Ok(()) + } + + /// Probe the REAL Tigris endpoint for the conditional-write semantics the + /// release fence depends on. + /// + /// UNTIL THIS IS RUN AGAINST REAL CREDENTIALS, the fence is verified against + /// vendor documentation and an in-process mock, not against the backend it + /// runs on. The mock implements the semantics we believe Tigris has; it + /// cannot tell us whether Tigris actually has them. + /// + /// The bucket matters, not just the endpoint. Tigris documents conditional + /// operations as supported on Single-region and Multi-region buckets only. + /// Global and Dual-region buckets are eventually consistent, and a + /// conditional PUT evaluated against a stale replica would make the fence a + /// silent no-op rather than an error. So point `GITLAWB_TIGRIS_BUCKET` at a + /// throwaway bucket of the SAME type production uses. + /// + /// Ignored by default and additionally gated on `GITLAWB_TIGRIS_PROBE=1`, + /// because it writes to a real bucket and costs real requests. Run with: + /// `GITLAWB_TIGRIS_PROBE=1 cargo test -p gitlawb-node --bin gitlawb-node + /// tigris_honors_conditional_writes -- --ignored --nocapture` + #[tokio::test] + #[ignore = "writes to a real Tigris bucket; needs GITLAWB_TIGRIS_PROBE=1 plus credentials"] + async fn tigris_honors_conditional_writes() { + let Some(bucket) = probe_env() else { + eprintln!( + "tigris conditional-write probe: skipped. Set GITLAWB_TIGRIS_PROBE=1, \ + GITLAWB_TIGRIS_BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and \ + AWS_ENDPOINT_URL_S3 to run it." + ); + return; + }; + + let config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; + let s3 = S3Client::new(&config); + // A fresh key per run, so a probe that somehow orphaned an object on an + // earlier run cannot change what this one observes. + let key = format!("probe/conditional-write-{}.bin", uuid::Uuid::new_v4()); + + // CLEANUP MUST RUN ON EVERY ARM, and a failing assertion is precisely + // the case this probe exists to catch, so the delete cannot sit after + // the checks. The body returns its failures rather than panicking, and + // `catch_unwind` covers the panic an SDK call could still raise; either + // way the delete below is reached before the verdict is re-raised. + let outcome = std::panic::AssertUnwindSafe(conditional_write_probe(&s3, &bucket, &key)) + .catch_unwind() + .await; + + if let Err(e) = s3.delete_object().bucket(&bucket).key(&key).send().await { + eprintln!("tigris conditional-write probe: cleanup of {key} failed: {e}"); + } + + match outcome { + Ok(Ok(())) => {} + Ok(Err(msg)) => panic!("tigris conditional-write probe: {msg}"), + Err(payload) => std::panic::resume_unwind(payload), + } + } +} diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 5d4579a3b..5ee09aa54 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -676,7 +676,7 @@ async fn warm_candidates( &repo.owner_did, &repo.name, ) { - Ok(p) if p.is_dir() => out.push((repo, created_at_key, p)), + Ok(p) if p.is_dir() => out.push((repo, created_at_key, p.into_path_buf())), Ok(_) => {} Err(e) => { tracing::warn!(repo_id = %repo.id, err = %e, "sweep discovery: rejected unsafe repo path"); diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 66bfa0961..e6b3d4015 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -59,32 +59,6 @@ struct DbStartupStatus { next_retry_secs: AtomicU64, } -/// Hard ceiling on the advisory-lock pool's `max_connections`. -/// -/// `max_concurrent_git_pushes` is validated all the way up to 1_048_576, and the lock -/// pool used to derive its size straight from that knob, so raising the push cap -/// silently raised the node's Postgres connection ceiling with no CLI error and no -/// relation to the server's own `max_connections` (#173 F4). The node's total budget is -/// now bounded: `db_max_connections` (default 48) + at most this. -const LOCK_POOL_MAX_CONNECTIONS: u32 = 64; - -/// Connections the lock pool keeps above the push cap. Covers the three non-push -/// `acquire_write` callers (`api/issues.rs` x2, `api/pulls.rs`), which hold no -/// concurrency permit, so a push never queues here for a connection where it did not -/// before. -const LOCK_POOL_PUSH_HEADROOM: u8 = 8; - -/// Size the advisory-lock pool for a given push cap: the cap plus -/// [`LOCK_POOL_PUSH_HEADROOM`], clamped to [`LOCK_POOL_MAX_CONNECTIONS`]. Past the -/// clamp a push may wait for a lock-pool connection, which is a bounded wait that sheds -/// a clean 503 (see `LockPoolBusy`), not an unbounded hang. -fn lock_pool_size(max_concurrent_git_pushes: usize) -> u32 { - u32::try_from(max_concurrent_git_pushes) - .unwrap_or(u32::MAX) - .saturating_add(u32::from(LOCK_POOL_PUSH_HEADROOM)) - .min(LOCK_POOL_MAX_CONNECTIONS) -} - #[tokio::main] async fn main() -> Result<()> { tracing_subscriber::fmt() @@ -116,8 +90,6 @@ async fn main() -> Result<()> { // Load or generate the node's identity keypair let keypair = load_or_create_keypair(&config)?; - // Sealing key for the legacy-scan continuation tokens, DERIVED from the identity - // just loaded so it is the same key after a restart (see `derive_scan_token_key`). let scan_token_key = AppState::derive_scan_token_key(&keypair); let node_did = keypair.did(); @@ -316,20 +288,20 @@ async fn main() -> Result<()> { None }; - // Repo write locks run on their own pool, never the main query pool: each push - // holds its connection for the whole receive-pack, so a burst of concurrent - // pushes drawing from the main pool would park that many connections for the - // duration of their receive-packs and starve every other query. That holds - // whatever the two pools are sized at, which is why the separation is - // structural rather than a consequence of the defaults; config validate() - // separately requires db_max_connections >= max_concurrent_git_pushes + 8. See - // build_lock_pool for the cancellation semantics (#173). - let lock_pool = git::repo_store::build_lock_pool( - db.pool(), - lock_pool_size(config.max_concurrent_git_pushes), + // Advisory-lock connections come from their own pool: a write guard pins one + // for its whole lifetime, and sharing the application pool would let a push + // burst starve ordinary request handlers. + let lock_pool = db::Db::lock_pool( + &config.database_url, + config.db_lock_pool_max_connections, std::time::Duration::from_secs(config.db_acquire_timeout_secs), + )?; + let repo_store = git::repo_store::RepoStore::new( + config.repos_dir.clone(), + tigris, + lock_pool, + std::time::Duration::from_secs(config.lock_held_transfer_timeout_secs), ); - let repo_store = git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, lock_pool); // Per-DID limiter for the creation endpoints. Keyed on the authenticated // DID (attacker-varied), so bound its key set to cap memory. @@ -374,6 +346,23 @@ async fn main() -> Result<()> { tracing::warn!("GITLAWB_PUSH_RATE_LIMIT=0 — per-IP push rate limiting disabled"); } + // close_issue drives a full archive download for non-owner author pre-checks. + // Keep it on its own bucket so it cannot drain the receive-pack limit. + let close_issue_limit = std::env::var("GITLAWB_CLOSE_ISSUE_RATE_LIMIT") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(120); + let close_issue_rate_limiter = rate_limit::RateLimiter::new_bounded( + close_issue_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + if close_issue_limit == 0 { + tracing::warn!( + "GITLAWB_CLOSE_ISSUE_RATE_LIMIT=0 — per-IP close_issue rate limiting disabled" + ); + } + // Which forwarded header the edge is trusted to set. Default None (trust // nothing, key on the socket peer). Fly nodes set GITLAWB_TRUSTED_PROXY=fly; // a node behind Caddy/NGINX sets it to x-forwarded-for. @@ -423,18 +412,10 @@ async fn main() -> Result<()> { rate_limiter, create_ip_rate_limiter, push_rate_limiter, + close_issue_rate_limiter, ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, - // The legacy-probe budget is operator-tunable via GITLAWB_IPFS_MAX_LEGACY_PROBES - // (R5), which is what `ipfs_legacy_probe_budget` reads. NOT - // GITLAWB_IPFS_MAX_REPOS_WALKED, which this comment used to name: that is the - // separate cap on expensive visibility walks, so an operator following the old - // text tuned the walk cap and left this fan-out unchanged. The history-walk - // ceiling above stays constant (a smaller value false-503s a provenanced - // request). Default 256 preserves the shipped behaviour. ipfs_max_legacy_probes: AppState::ipfs_legacy_probe_budget(&config), ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, - // Operator-tunable via GITLAWB_IPFS_MAX_LEGACY_SCAN_ROWS, read through the same - // helper shape as the probe budget so the knob cannot be a silent no-op. ipfs_max_legacy_scan_rows: AppState::ipfs_legacy_scan_row_budget(&config), ipfs_max_legacy_scan_rule_bytes: crate::api::ipfs::MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST, ipfs_scan_token_key: Arc::new(scan_token_key), @@ -513,10 +494,6 @@ async fn main() -> Result<()> { std::time::Duration::from_secs(3600), 200_000, ), - // Separate WORK-budget bucket for the resolver's per-probe/per-walk charges (R6). - // Its capacity is DERIVED from the route limit (no new knob) and floored at the - // legacy-probe budget, so one full default-config legacy scan never self-throttles - // mid-request while the route brake above stays the pure once-per-request cap. ipfs_work_rate_limiter: rate_limit::RateLimiter::new_bounded( AppState::ipfs_work_budget(&config), std::time::Duration::from_secs(3600), @@ -556,16 +533,14 @@ async fn main() -> Result<()> { // Periodic cleanup of expired rate limit entries + consumed-proof ledger { - let cleanup_state = state.clone(); + let sweep_state = state.clone(); let db = state.db.clone(); let mut shutdown_rx = state.subscribe_shutdown(); tokio::spawn(async move { loop { tokio::select! { _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { - // Sweep every per-IP/DID limiter (incl. the ipfs walk brake) - // so bounded maps shed stale keys instead of sitting at cap. - cleanup_state.sweep_rate_limiters().await; + sweep_state.sweep_rate_limiters().await; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs() as i64) @@ -708,44 +683,6 @@ async fn main() -> Result<()> { Ok(()) } -/// U4 (#173): spawn the periodic legacy provider-CID repair sweep. Releases before this -/// version stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`, -/// and this version's `/ipfs/{cid}` resolver withholds any row whose stored key is not -/// the raw-content CID. The opportunistic repair on the pin path only fires when a push -/// re-carries the object, which normal git negotiation makes it not do, so those rows -/// need a walk. DETACHED, never on the boot path: the caller keeps serving while this -/// runs, and the sweep's own batch bound plus inter-batch delay keep it off the DB's -/// critical path. Its cursor is durable, so a restart mid-walk resumes instead of -/// rewinding. It re-arms after every run, including a failed one, so it never returns -/// and there is no awaited value to log here; the shutdown watcher below is what ends -/// it. -/// -/// A named function rather than an inline block in `main` so the WIRING has a seam a -/// test can call: that the task is spawned at all, that it reads its batch and delay -/// from the config knobs rather than some other field, that the caller is not blocked -/// on it, and that the shutdown watcher actually ends it mid-walk. The sweep's own -/// behavior is covered elsewhere; this is the boot-path half. -fn spawn_legacy_cid_sweep(state: &AppState, config: &Config) -> tokio::task::JoinHandle<()> { - let db = state.db.clone(); - let repos_dir = config.repos_dir.clone(); - let git_bin = state.git_bin.clone(); - let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs); - let batch = config.pin_repair_sweep_batch; - let delay = std::time::Duration::from_secs(config.pin_repair_sweep_delay_secs); - let mut shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - tokio::select! { - _ = ipfs_pin::run_sweep_rearmed( - &repos_dir, &git_bin, git_timeout, batch, delay, - ipfs_pin::SWEEP_REARM_DELAY, &db, - ) => {} - // Shutdown mid-walk simply drops the run; the persisted cursor means the - // next boot picks up where this one stopped. - _ = shutdown_rx.changed() => {} - } - }) -} - fn spawn_shutdown_signal(tx: watch::Sender) { tokio::spawn(async move { #[cfg(unix)] @@ -1187,20 +1124,20 @@ mod rate_limiter_sweep_tests { state.rate_limiter = RateLimiter::new(10, window); state.create_ip_rate_limiter = RateLimiter::new(10, window); state.push_rate_limiter = RateLimiter::new(10, window); + state.close_issue_rate_limiter = RateLimiter::new(10, window); state.sync_trigger_rate_limiter = RateLimiter::new(10, window); state.peer_write_rate_limiter = RateLimiter::new(10, window); state.ipfs_rate_limiter = RateLimiter::new(10, window); - state.ipfs_work_rate_limiter = RateLimiter::new(10, window); let limiters = |s: &crate::state::AppState| { [ s.rate_limiter.clone(), s.create_ip_rate_limiter.clone(), s.push_rate_limiter.clone(), + s.close_issue_rate_limiter.clone(), s.sync_trigger_rate_limiter.clone(), s.peer_write_rate_limiter.clone(), s.ipfs_rate_limiter.clone(), - s.ipfs_work_rate_limiter.clone(), ] }; for l in limiters(&state) { @@ -1217,6 +1154,26 @@ mod rate_limiter_sweep_tests { } } +/// it. +fn spawn_legacy_cid_sweep(state: &AppState, config: &Config) -> tokio::task::JoinHandle<()> { + let db = state.db.clone(); + let repos_dir = config.repos_dir.clone(); + let git_bin = state.git_bin.clone(); + let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs); + let batch = config.pin_repair_sweep_batch; + let delay = std::time::Duration::from_secs(config.pin_repair_sweep_delay_secs); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + tokio::select! { + _ = ipfs_pin::run_sweep_rearmed( + &repos_dir, &git_bin, git_timeout, batch, delay, + ipfs_pin::SWEEP_REARM_DELAY, &db, + ) => {} + _ = shutdown_rx.changed() => {} + } + }) +} + async fn gossip_ping_round( db: &Db, client: &reqwest::Client, @@ -1393,140 +1350,6 @@ fn load_or_create_keypair(config: &Config) -> Result { } } -#[cfg(test)] -mod legacy_cid_sweep_wiring_tests { - use super::spawn_legacy_cid_sweep; - use sqlx::PgPool; - use std::time::Duration; - - /// Seed `count` `pinned_cids` rows whose keys are already canonical raw CIDv1, in a - /// known `sha256_hex` order. The sweep's own cost gate skips a raw-CIDv1 row without - /// reading bytes or resolving a repo, so each row is SCANNED (it advances the cursor) - /// and nothing else. That is what makes the cursor a clean readout of how far the - /// walk got, with no dependency on repos on disk. - async fn seed_scannable_rows(pool: &PgPool, count: usize) -> Vec { - let mut shas = Vec::new(); - for i in 1..=count { - let sha = format!("wire{i:02}"); - let cid = gitlawb_core::cid::Cid::from_git_object_bytes(sha.as_bytes()).to_string(); - assert!( - gitlawb_core::cid::is_raw_cidv1(&cid), - "the seeded key must hit the sweep's raw-CIDv1 skip, not a repair attempt" - ); - sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") - .bind(&sha) - .bind(&cid) - .bind("2020-01-01T00:00:00Z") - .execute(pool) - .await - .unwrap(); - shas.push(sha); - } - shas - } - - /// Poll the persisted sweep cursor until it reaches `want`, or give up. - async fn cursor_reaches(db: &crate::db::Db, want: &str, within: Duration) -> String { - let deadline = std::time::Instant::now() + within; - loop { - let c = db.pin_repair_cursor().await.unwrap(); - if c == want || std::time::Instant::now() >= deadline { - return c; - } - tokio::time::sleep(Duration::from_millis(20)).await; - } - } - - /// #173 U4, the BOOT-PATH half. The sweep's own logic (batching, cursor resumption, - /// terminal vs retryable skips) is covered in `test_support`; what this covers is the - /// wiring `main` performs, which nothing else executes: the task is spawned at all, - /// it takes its batch and delay from the two `pin_repair_sweep_*` knobs rather than - /// some other config field, the caller is not blocked on the walk, and the shutdown - /// watcher ends the run mid-walk. - /// - /// Six scannable rows, batch 2, delay 30s. One pass must land the cursor on exactly - /// the second row and the task must then still be alive in its inter-batch sleep, - /// which pins both knobs at once: a different batch stops at a different row, and a - /// delay that did not come from the knob either finishes the table or leaves the task - /// gone. Shutdown must then end it while four rows are still unwalked. - #[sqlx::test] - async fn the_boot_path_spawns_the_sweep_detached_with_its_configured_knobs(pool: PgPool) { - let state = crate::test_support::test_state(pool.clone()).await; - let shas = seed_scannable_rows(&pool, 6).await; - let repos_dir = tempfile::TempDir::new().unwrap(); - - let mut config = (*state.config).clone(); - config.repos_dir = repos_dir.path().to_path_buf(); - config.pin_repair_sweep_batch = 2; - // Far longer than this test runs, so a task still alive after the first pass can - // only be one that is honoring the configured inter-batch delay. - config.pin_repair_sweep_delay_secs = 30; - - let started = std::time::Instant::now(); - let handle = spawn_legacy_cid_sweep(&state, &config); - let spawn_cost = started.elapsed(); - - let cursor = cursor_reaches(&state.db, &shas[1], Duration::from_secs(10)).await; - assert_eq!( - cursor, shas[1], - "the spawned sweep must run and stop its first pass at the CONFIGURED batch \ - bound (2), leaving the cursor on the second row" - ); - assert!( - spawn_cost < Duration::from_secs(1), - "the sweep must be detached, not awaited on the boot path; the spawn took \ - {spawn_cost:?}" - ); - assert!( - !handle.is_finished(), - "with a 30s inter-batch delay the task must still be sleeping between passes, \ - not finished: a finished task means the delay was not the configured one" - ); - - state.shutdown(); - tokio::time::timeout(Duration::from_secs(10), handle) - .await - .expect("the shutdown watcher must end the sweep, and not after its 30s delay") - .expect("the sweep task must not panic"); - - assert_eq!( - state.db.pin_repair_cursor().await.unwrap(), - shas[1], - "shutdown must have ended the run MID-walk, with the remaining rows unwalked" - ); - } -} - -#[cfg(test)] -mod lock_pool_sizing_tests { - use super::{lock_pool_size, LOCK_POOL_MAX_CONNECTIONS, LOCK_POOL_PUSH_HEADROOM}; - - /// The default push cap gets its cap plus headroom, so no push ever queues for a - /// lock-pool connection where it did not before. - #[test] - fn default_push_cap_gets_headroom_over_the_cap() { - assert_eq!(lock_pool_size(32), 32 + u32::from(LOCK_POOL_PUSH_HEADROOM)); - assert_eq!(lock_pool_size(1), 1 + u32::from(LOCK_POOL_PUSH_HEADROOM)); - } - - /// #173 F4: `max_concurrent_git_pushes` is validated all the way to 1_048_576, so an - /// operator raising it used to raise the node's Postgres connection ceiling with it, - /// silently and without bound. The lock pool is CLAMPED instead. - #[test] - fn an_oversized_push_cap_is_clamped_not_propagated() { - assert_eq!(lock_pool_size(1_048_576), LOCK_POOL_MAX_CONNECTIONS); - assert_eq!(lock_pool_size(usize::MAX), LOCK_POOL_MAX_CONNECTIONS); - // The largest cap that still fits under the clamp keeps its full headroom. - let widest = (LOCK_POOL_MAX_CONNECTIONS - u32::from(LOCK_POOL_PUSH_HEADROOM)) as usize; - assert_eq!(lock_pool_size(widest), LOCK_POOL_MAX_CONNECTIONS); - assert_eq!( - lock_pool_size(widest - 1), - LOCK_POOL_MAX_CONNECTIONS - 1, - "values below the clamp must not be rounded up to it" - ); - } -} - #[cfg(test)] mod gossip_ssrf_tests { use super::{ diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 24607e5ad..7fcd1f5d3 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -79,6 +79,10 @@ pub struct AppState { /// brake a push flood from a DID farm (one throwaway DID per repo), so the /// push path throttles on the resolved client IP instead. pub push_rate_limiter: RateLimiter, + /// Per-client-IP rate limiter for `close_issue`'s pre-lock snapshot path. + /// Distinct from `push_rate_limiter` so a flood of close attempts cannot + /// drain the receive-pack budget for the same source IP. + pub close_issue_rate_limiter: RateLimiter, /// Per-client-IP ROUTE brake for `GET /ipfs/{cid}`: charged ONCE per request by the /// `rate_limit_by_ip` middleware (server.rs), never inside the handler. It bounds /// request RATE (the "requests per hour" contract of `GITLAWB_IPFS_RATE_LIMIT`) on @@ -342,6 +346,7 @@ impl AppState { self.rate_limiter.cleanup().await; self.create_ip_rate_limiter.cleanup().await; self.push_rate_limiter.cleanup().await; + self.close_issue_rate_limiter.cleanup().await; self.ipfs_rate_limiter.cleanup().await; self.ipfs_work_rate_limiter.cleanup().await; self.sync_trigger_rate_limiter.cleanup().await; diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 430c06002..e5c9b0c5b 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -103,6 +103,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + close_issue_rate_limiter: RateLimiter::new(120, Duration::from_secs(3600)), ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, @@ -557,6 +558,35 @@ mod tests { ); } + /// Fork disk paths must go through `validated_repo_disk_path` so a user-supplied + /// name cannot reach `remove_dir_all` on an escaped path (CodeQL path-injection). + #[sqlx::test] + async fn fork_rejects_name_that_fails_validated_disk_path(pool: PgPool) { + let owner = "did:key:zFORKOWNERAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let state = test_state(pool).await; + let repo = seed_repo(owner, "fork-src"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let router = Router::new() + .route( + "/api/v1/repos/{owner}/{repo}/fork", + axum::routing::post(crate::api::repos::fork_repo), + ) + .with_state(state.clone()); + let too_long = "a".repeat(101); + let uri = format!("/api/v1/repos/{owner}/fork-src/fork"); + let body = Body::from(format!(r#"{{"name":"{too_long}"}}"#)); + let resp = router + .oneshot(signed_request_as(owner, Method::POST, &uri, body)) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::BAD_REQUEST, + "fork names that fail validated_repo_disk_path must be rejected before clone" + ); + } + /// N13: the task handlers bind the acting DID to the signer. A caller signed /// as B claiming delegator_did A is rejected before any DB write (DB-free). #[sqlx::test] @@ -15936,65 +15966,6 @@ mod tests { use super::*; use crate::api::repos::drain_faults; - /// Process-wide tracing capture so a test can assert the give-up is logged at - /// ERROR. A global default subscriber can only be installed once per process, - /// so it is shared by every test here and assertions filter on the repo id, - /// which is a fresh uuid per test. - mod logcap { - use std::sync::{Arc, Mutex, OnceLock}; - use tracing::{Event, Level, Subscriber}; - use tracing_subscriber::layer::{Context, Layer}; - use tracing_subscriber::prelude::*; - - type Lines = Arc>>; - - fn lines() -> &'static Lines { - static LINES: OnceLock = OnceLock::new(); - LINES.get_or_init(|| Arc::new(Mutex::new(Vec::new()))) - } - - struct Capture; - impl Layer for Capture { - fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { - struct V(String); - impl tracing::field::Visit for V { - fn record_debug( - &mut self, - field: &tracing::field::Field, - value: &dyn std::fmt::Debug, - ) { - self.0.push_str(&format!(" {}={:?}", field.name(), value)); - } - } - let mut v = V(String::new()); - event.record(&mut v); - lines() - .lock() - .unwrap() - .push((*event.metadata().level(), v.0)); - } - } - - pub(super) fn install() { - static ONCE: OnceLock<()> = OnceLock::new(); - ONCE.get_or_init(|| { - let _ = tracing::subscriber::set_global_default( - tracing_subscriber::registry().with(Capture), - ); - }); - } - - pub(super) fn errors_containing(needle: &str) -> Vec { - lines() - .lock() - .unwrap() - .iter() - .filter(|(lvl, msg)| *lvl == Level::ERROR && msg.contains(needle)) - .map(|(_, msg)| msg.clone()) - .collect() - } - } - /// SCENARIO 1. The repo re-read fails once, then succeeds: the drain lap /// must still RUN, under the refreshed state, and pin the coalesced push's /// object. RED before the fix (the single `Err` returned `None`, the lap @@ -16060,11 +16031,11 @@ mod tests { } /// SCENARIO 2. Every re-read attempt fails: the loop must give up on a BOUND - /// (asserted as a literal, so raising or removing the bound goes RED) and log - /// the give-up at ERROR so the residual loss is observable rather than silent. + /// (asserted as a literal, so raising or removing the bound goes RED) and hit + /// the give-up path that logs at ERROR in production (asserted here via the + /// drain_faults seam, which fires on the same branch as that log). #[sqlx::test] async fn u2_sustained_repo_reread_failure_is_bounded_and_logged(pool: PgPool) { - logcap::install(); let state = test_state(pool).await; let owner = new_did(); let repo = seed_repo(&owner, "u2-bounded"); @@ -16115,11 +16086,10 @@ mod tests { !state.db.is_pinned(&obj2).await.unwrap(), "with the read never succeeding there is nothing fresh to act on" ); - let errs = logcap::errors_containing(&repo.id); assert!( - !errs.is_empty(), - "the exhausted drain re-read is logged at ERROR with the repo id, so \ - the residual work loss is observable; captured: {errs:?}" + drain_faults::reread_exhausted(&repo.id), + "the exhausted drain re-read must hit the give-up path that logs at \ + ERROR in production" ); assert!( state.encrypt_inflight.is_empty(), diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 48381e3dc..7ce405f6d 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -225,7 +225,8 @@ fn f4_release_keeps_conn_owned_until_unlock_resolves() { let repo_store = src("git/repo_store.rs"); let rel_start = repo_store - .find("pub async fn release(mut self") + .find("pub async fn release(self") + .or_else(|| repo_store.find("pub async fn release(mut self")) .expect("F4 gate: repo_store.rs no longer defines RepoWriteGuard::release"); let rel_end = repo_store[rel_start..] .find("impl Drop for RepoWriteGuard") diff --git a/crates/gl/src/whoami.rs b/crates/gl/src/whoami.rs index 6807aa167..7aa9bca2e 100644 --- a/crates/gl/src/whoami.rs +++ b/crates/gl/src/whoami.rs @@ -60,7 +60,7 @@ pub(crate) async fn run_to_writer(args: WhoamiArgs, w: &mut impl std::io::Write) } Ok(resp) => { let status = resp.status(); - let raw = read_body_capped(resp, 8 * 1024).await; + let raw = read_body_capped(resp, 8 * 1024).await.text; let msg = serde_json::from_str::(&raw) .ok() .and_then(|v| {