diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 4e327c429..9116e4027 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -11,12 +11,10 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::cert; use crate::error::{AppError, Result}; use crate::git::{smart_http, store, visibility_pack}; use crate::state::AppState; use crate::visibility::{visibility_check, withheld_globs, Decision}; -use crate::webhooks; /// The git all-zeros object id — the create/delete sentinel in a ref update. const ZERO_SHA: &str = "0000000000000000000000000000000000000000"; @@ -2245,14 +2243,516 @@ pub async fn git_receive_pack( let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit) .with_hold(std::sync::Arc::clone(&guard)) .with_lease(lease.clone()); - let receive_result = smart_http::receive_pack( + + // #26 Split PR 1: durable intent for this push, written BEFORE + // the receive_pack call. Every ref update the pusher intends to + // land gets a `prepared` row carrying the verified pusher DID, + // the raw RFC 9421 signature header, signature-input, and + // content-digest that authorized the push, plus the request id. + // + // The state is flipped to `applied` (Ok) or `cancelled` (Err) + // AFTER receive_pack returns. The drain reads only `applied` + // rows, so a row that never gets the post-Ok flip stays in + // `prepared` (handler crash / dropped future) or `cancelled` + // (receive_pack Err) and is never promoted to a push event, a + // certificate, or an anchor. + // + // Inserted AT THE LAST POSSIBLE MOMENT, immediately before the + // receive_pack call, so a rejection above (owner enforcement, + // branch protection, etc.) does not produce a `prepared` row + // that nothing will ever flip. + let signature_header = headers + .get("signature") + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_string(); + let signature_input = headers + .get("signature-input") + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_string(); + let content_digest = headers + .get("content-digest") + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_string(); + // #26 Split PR 1 — request-level intent row. Written BEFORE + // `smart_http::receive_pack` runs, in state `received`, carrying + // the raw HTTP body the handler will hand to git and the SHA-256 + // of it. The recovery drain (step 3) and the on-disk reconcile + // (already on this branch) both key off this row; a node crash + // between this write and the outcomes commit leaves the row in + // `received` and its children in `prepared`, which is the + // recoverable state. + let request_id = uuid::Uuid::new_v4().to_string(); + let now = Utc::now().to_rfc3339(); + let request_bytes_hash = { + use sha2::{Digest, Sha256}; + let mut h = Sha256::new(); + h.update(&body); + h.finalize().to_vec() + }; + let req_row = crate::db::ReceivePackRequest { + id: request_id.clone(), + repo_id: record.id.clone(), + pusher_did: auth.0.to_string(), + node_did: state.node_did.to_string(), + request_bytes: body.to_vec(), + request_bytes_hash, + state: crate::db::request_state::RECEIVED.to_string(), + git_exit_ok: None, + parsed_report: None, + accepted_ordinal: None, + attempt_count: 0, + last_error: None, + next_attempt_at: None, + created_at: now.clone(), + completed_at: None, + }; + if let Err(e) = state.db.insert_receive_pack_request(&req_row).await { + // A durable-intent write failure here means we cannot + // guarantee recovery for the upcoming git apply. Refuse the + // push with 503 rather than risk a ref landing with no + // recovery record. + tracing::error!( + err = %e, + repo = %name, + "failed to persist receive-pack request row; refusing push" + ); + return Err(AppError::Overloaded( + "durable intent write failed, retry shortly".into(), + )); + } + if let Err(e) = state + .db + .insert_pending_ref_transitions( + &request_id, + &record.id, + &state.node_did.to_string(), + auth.0.as_str(), + &ref_updates, + &signature_header, + &signature_input, + &content_digest, + ) + .await + { + // A durable-intent write failure here means we cannot + // guarantee recovery for the upcoming git apply. Refuse the + // push with 503 rather than risk a ref landing with no + // recovery record. + tracing::error!( + err = %e, + repo = %name, + "failed to persist durable post-receive intent; refusing push" + ); + return Err(AppError::Overloaded( + "durable intent write failed, retry shortly".into(), + )); + } + + // #26 Split PR 1 step 5 — write the per-request marker ref + // BEFORE calling `git receive-pack`. The marker's value is + // derived from `request_bytes_hash` via `marker_value_for`, + // which `git hash-object -w`s the first 20 bytes (yielding a + // 40-char SHA-1, the only thing `git update-ref` will accept). + // The reconcile reads it back via `git::store::read_ref` and + // compares against the request row via the same helper. + // Failure is non-fatal: the reconcile's marker gate will see no + // marker and quarantine the request; an operator can reclassify. + let marker_ref_name = format!("refs/gitlawb/requests/{request_id}"); + match crate::git::store::marker_value_for(&disk_path, &req_row.request_bytes_hash) { + Ok(marker_value) => { + let marker_write = std::process::Command::new("git") + .args(["update-ref", &marker_ref_name, &marker_value]) + .arg("--no-deref") + .current_dir(&disk_path) + .output(); + match marker_write { + Ok(out) if !out.status.success() => { + tracing::warn!( + request_id = %request_id, + repo = %name, + stderr = %String::from_utf8_lossy(&out.stderr), + "marker write returned non-zero; reconcile will quarantine this request" + ); + } + Err(e) => { + tracing::warn!( + err = %e, + request_id = %request_id, + repo = %name, + "marker write failed to spawn; reconcile will quarantine this request" + ); + } + _ => {} + } + } + Err(e) => { + tracing::warn!( + err = %e, + request_id = %request_id, + repo = %name, + "marker value computation failed; reconcile will quarantine this request" + ); + } + } + + // P1 (reviewer-1/2 round 3): use receive_pack_raw to get the raw + // stdout (which contains the report-status with per-ref ok/ng + // results) and the process exit status. This allows us to: + // 1. Parse per-ref results to distinguish proven rejections from + // uncertain outcomes on error. + // 2. On success, write effects and then DELETE outbox rows so they + // don't replay on restart. + let (receive_raw, exit_ok) = match smart_http::receive_pack_raw( &state.git_bin, &disk_path, body, git_timeout, Some(admission), ) - .await; + .await + { + Ok(r) => r, + Err(e) => { + // Timeout or spawn failure — the git process group was + // torn down. Mark all prepared rows as uncertain so the + // reconcile step can check them against disk at startup. + if let Err(ce) = state + .db + .mark_pending_ref_transitions_uncertain(&request_id) + .await + { + tracing::warn!( + err = %ce, + request_id = %request_id, + repo = %name, + "failed to mark pending ref transitions uncertain after receive-pack error" + ); + } + let app = git_service_app_error(&e); + match &app { + AppError::Timeout(_) => tracing::warn!(repo = %name, "git receive-pack timed out"), + AppError::BadRequest(msg) => { + tracing::warn!(repo = %name, err = %msg, "git receive-pack: bad client request") + } + _ => tracing::error!(repo = %name, err = %e, "git receive-pack failed"), + } + return Err(app); + } + }; + + // P1 (reviewer-1/2 round 4): complete the per-ref outcome + // model. The previous `all_refs_ok` ignored `unpack_ok` and + // treated an absent report as success, then ran the durable + // effects loop unconditionally. A mixed push where one ref is + // rejected still issued a signed certificate, an anchor job, a + // push event, a trust-score bump, and a webhook for the rejected + // ref. Git can report `unpack ok / ng refs/heads/main` on a + // zero exit (a non-fast-forward, a hook denial), and that path + // is exactly the one the previous logic missed. + // + // Build the set of ref names that the report-status proves + // landed. Every ref update the handler intended to land gets a + // fate: + // - in `ok_set` → row goes to `applied`, effects fire + // - in `report` but ng → row goes to `cancelled`, no effects + // - not in `report` at all → row goes to `uncertain` for reconcile + // + // If `unpack_ok == false`, no ref could have landed — all rows + // become `cancelled` and no effects fire for any ref. + // + // If the report is unparseable (client did not request + // report-status, or framing was malformed), every row becomes + // `uncertain` so the on-disk reflog proof can sort out which + // ones actually landed at the next startup. + let report = smart_http::parse_report_status(&receive_raw); + + // (unpack_ok, all_in_report_ok, ok_set, request_failed). + // + // The reviewer wanted: distinguish per-ref ok/ng from + // unparseable/incomplete reports. The two cases are: + // + // - report parsed: ok_set is exactly the refs the report + // named with `ok`. Refs the report did NOT mention are + // "unmentioned" and become `uncertain` for reconcile. + // - report absent (the client did not request report-status, + // or the framing was unreadable): no in-band ng signal, + // and the only ground truth we have is the process exit. + // A zero exit with no report is a successful push whose + // refs we cannot enumerate per-ref — the legacy + // "all_refs_ok = exit_ok" semantic. A non-zero exit with + // no report means we cannot prove which refs landed, so + // every row is `uncertain` for reconcile. + let (unpack_ok, all_in_report_ok, ok_set, request_failed) = match &report { + Some((unpack_ok, ref_results)) => { + let ok_set: std::collections::HashSet<&str> = ref_results + .iter() + .filter(|(_, ok)| *ok) + .map(|(name, _)| name.as_str()) + .collect(); + let all_in_report_ok = ref_results.iter().all(|(_, ok)| *ok); + (*unpack_ok, all_in_report_ok, ok_set, !exit_ok) + } + None if exit_ok => { + // No report but the process exited zero: every pushed + // ref is implicitly ok — the legacy semantic that + // receive-pack tests / clients without report-status + // rely on. + let ok_set: std::collections::HashSet<&str> = + ref_updates.iter().map(|u| u.ref_name.as_str()).collect(); + (true, true, ok_set, false) + } + None => { + // No report and a non-zero exit: we cannot prove which + // refs landed. Mark all rows `uncertain` for reconcile. + let ok_set: std::collections::HashSet<&str> = std::collections::HashSet::new(); + (false, true, ok_set, !exit_ok) + } + }; + + // Per-ref state flip. Rows go to `applied`, `cancelled`, or + // `uncertain` based on the report. The previous bulk helper + // `mark_pending_ref_transitions_applied(request_id)` marked + // EVERY row `applied` regardless of which refs git actually + // accepted — that is the bug that issued certs for rejected + // refs. + let pending_ref_names: Vec<&str> = ref_updates.iter().map(|u| u.ref_name.as_str()).collect(); + + // #26 Split PR 1: the push event id is keyed on + // `(request_id, accepted_ordinal)`. The `accepted_ordinal` is + // the ordinal (in `ref_updates`) of the FIRST ref the report + // proves landed; the v30 migration's `ordinal` column carries + // the position. No `first_ref_name` rewrite is needed because + // the identity is on the request, not on a mutable per-ref + // column. Compute it once here so the per-ref effects loop can + // stamp the request row at the right moment. + let accepted_ordinal: Option = ref_updates + .iter() + .position(|u| ok_set.contains(u.ref_name.as_str())) + .map(|i| i as i32); + + if !unpack_ok && report.is_some() { + // Unpack failed explicitly — every row is proven not to have + // landed. Mark all prepared rows for this request as + // `cancelled` (the only state from which reconcile and drain + // both refuse to promote). The drain will not pick these up; + // the next startup's reconcile will not promote them. + if let Err(e) = state + .db + .mark_pending_ref_transitions_cancelled_for_names(&request_id, &pending_ref_names) + .await + { + tracing::warn!( + err = %e, + request_id = %request_id, + repo = %name, + "failed to mark pending ref transitions cancelled (unpack fail)" + ); + } + } else if report.is_some() { + // Report parsed. Split rows by per-ref ok/ng, with anything + // NOT in the report (defensive: report is a subset of the + // pushed refs in some edge cases) falling to `uncertain`. + let mut ok_names: Vec<&str> = Vec::new(); + let mut ng_names: Vec<&str> = Vec::new(); + let mut unmentioned: Vec<&str> = Vec::new(); + let reported: std::collections::HashSet<&str> = report + .as_ref() + .map(|(_, rs)| rs.iter().map(|(n, _)| n.as_str()).collect()) + .unwrap_or_default(); + for name in &pending_ref_names { + if !reported.contains(name) { + unmentioned.push(*name); + } else if ok_set.contains(name) { + ok_names.push(*name); + } else { + ng_names.push(*name); + } + } + if !ng_names.is_empty() { + tracing::warn!( + request_id = %request_id, + repo = %name, + rejected_refs = ?ng_names, + "git report-status: some refs rejected; durable effects will skip them" + ); + } + if !ok_names.is_empty() { + if let Err(e) = state + .db + .mark_pending_ref_transitions_applied_for_names(&request_id, &ok_names) + .await + { + tracing::error!( + err = %e, + request_id = %request_id, + repo = %name, + "failed to mark pending ref transitions applied; recovery will re-derive" + ); + } + } + if !ng_names.is_empty() { + if let Err(e) = state + .db + .mark_pending_ref_transitions_cancelled_for_names(&request_id, &ng_names) + .await + { + tracing::warn!( + err = %e, + request_id = %request_id, + repo = %name, + "failed to mark pending ref transitions cancelled (per-ref ng)" + ); + } + } + if !unmentioned.is_empty() { + if let Err(e) = state + .db + .mark_pending_ref_transitions_uncertain_for_names(&request_id, &unmentioned) + .await + { + tracing::warn!( + err = %e, + request_id = %request_id, + repo = %name, + "failed to mark pending ref transitions uncertain (unmentioned in report)" + ); + } + } + } else if ok_set.len() == pending_ref_names.len() && !pending_ref_names.is_empty() { + // Implicit-ok path: report was absent but the process exit + // was zero, so every pushed ref is treated as landed. Mark + // all prepared rows `applied` so the drain (and the live + // per-ref effects below) can run. + if let Err(e) = state + .db + .mark_pending_ref_transitions_applied(&request_id) + .await + { + tracing::error!( + err = %e, + request_id = %request_id, + repo = %name, + "failed to mark pending ref transitions applied (implicit ok, no report)" + ); + } + } else { + // No report at all AND non-zero exit — every ref's fate is + // uncertain. The next startup reconcile will use the + // reflog proof to promote only those whose transition + // actually landed. + if let Err(e) = state + .db + .mark_pending_ref_transitions_uncertain(&request_id) + .await + { + tracing::warn!( + err = %e, + request_id = %request_id, + repo = %name, + "failed to mark pending ref transitions uncertain (no report)" + ); + } + } + + // #26 Split PR 1: transition the request row to + // `outcomes_committed` (with `parsed_report` and + // `accepted_ordinal` stamped) or `rejected_at_git` (when git + // returned non-zero with no parseable report). The drain + // (step 3) reads `outcomes_committed` rows; today the live + // path also runs the per-ref effects inline below so the + // request moves to `complete` is step-3 territory. + // + // The transition runs as a side-effect of the four-branch + // flip above: a parseable report always lands in + // `outcomes_committed`; the no-report non-zero-exit branch + // (the implicit `None =>` else) lands in `rejected_at_git`. + if let Some(parsed) = &report { + let parsed_json = serde_json::json!({ + "unpack_ok": parsed.0, + "ref_results": parsed.1.iter().map(|(n, ok)| serde_json::json!({ + "ref_name": n, + "ok": ok, + })).collect::>(), + }); + if let Err(e) = state + .db + .mark_request_outcomes_committed(&request_id, exit_ok, &parsed_json, accepted_ordinal) + .await + { + tracing::warn!( + err = %e, + request_id = %request_id, + repo = %name, + "failed to stamp request outcomes; the request row stays in `received` and the drain will not see it" + ); + } + } else if !exit_ok { + // No report AND non-zero exit: request goes to + // `rejected_at_git`. Children stay in `prepared` for the + // reconcile step to decide via on-disk SHA + reflog proof. + if let Err(e) = state + .db + .mark_request_rejected_at_git( + &request_id, + Some("git returned non-zero exit with no parseable report"), + ) + .await + { + tracing::warn!( + err = %e, + request_id = %request_id, + repo = %name, + "failed to mark request rejected_at_git; the request row stays in `received` and the drain will not see it" + ); + } + } else { + // No report but exit zero (implicit-ok): every ref + // accepted. The per-ref state flip above already marked + // every child `applied`; the request row goes to + // `outcomes_committed` with `parsed_report = null` and + // `accepted_ordinal = Some(0)`. + let parsed_json = serde_json::Value::Null; + if let Err(e) = state + .db + .mark_request_outcomes_committed(&request_id, exit_ok, &parsed_json, accepted_ordinal) + .await + { + tracing::warn!( + err = %e, + request_id = %request_id, + repo = %name, + "failed to stamp request outcomes (implicit-ok); the request row stays in `received`" + ); + } + } + + // On non-zero exit, return an error to the caller. The outbox + // rows have already been handled above (per-ref fates applied). + // The client-visible body does NOT include wire-supplied ref + // names — ref names can carry control bytes and the previous + // `format!("refs rejected: {rejected:?}")` embedded them in a + // 500 response. Server-side the names are logged above. + if request_failed { + let reclaimed = guard + .lock() + .expect("repo write-lock mutex poisoned") + .take() + .expect("the write lock is only taken here, and only once"); + reclaimed.release(false).await; + drop(lease); + + let body_msg = if !unpack_ok { + "git-receive-pack failed: unpack failed" + } else if !all_in_report_ok { + "git-receive-pack failed: refs rejected" + } else { + "git-receive-pack failed" + }; + return Err(AppError::Git(body_msg.to_string())); + } // #174 F2/U5: the post-receive replication tail runs in an independently owned // task. It parks on `git_encrypt_semaphore` (withheld / candidate / full-scan @@ -2296,15 +2796,38 @@ pub async fn git_receive_pack( // The alternative, detaching `release` and the tail together to keep the ordering, // would return 200 to the pusher before the durable copy lands, which is a larger // change to the client contract than the window it closes. - let push_succeeded = receive_result.is_ok(); + // P1 (reviewer-1/2 round 4): the durable effects only fire for + // refs the report-status proves landed. The previous code ran + // them unconditionally once past the `!exit_ok` return, so a + // `unpack ok / ng refs/heads/main` zero-exit push still signed + // a cert and queued an anchor for the rejected ref. + // + // `push_succeeded` is the request-level signal for the + // replication tail and the lock release. `any_ref_ok` is the + // request-scoped effects gate (push event, trust score, + // metrics): at least one ref must have landed for those to be + // meaningful. + let any_ref_ok = !ok_set.is_empty(); + let push_succeeded = exit_ok && any_ref_ok; + if push_succeeded { - tokio::spawn(post_receive_replication_tail( - state.clone(), - record.clone(), - ref_updates.clone(), - disk_path.clone(), - auth.0.to_string(), - )); + // Spawn the replication tail only for refs that landed. + // Filtering at spawn-time keeps the tail's input accurate + // even for a mixed push. + let landed_refs: Vec = ref_updates + .iter() + .filter(|u| ok_set.contains(u.ref_name.as_str())) + .cloned() + .collect(); + if !landed_refs.is_empty() { + tokio::spawn(post_receive_replication_tail( + state.clone(), + record.clone(), + landed_refs, + disk_path.clone(), + auth.0.to_string(), + )); + } } // Always release the advisory lock — even on error — to prevent stale locks @@ -2327,108 +2850,91 @@ pub async fn git_receive_pack( // the disconnect path this line is never reached: clone (a) rides the reaper (F3). drop(lease); - let result = receive_result.map_err(|e| { - let app = git_service_app_error(&e); - match &app { - AppError::Timeout(_) => tracing::warn!(repo = %name, "git receive-pack timed out"), - AppError::BadRequest(msg) => { - tracing::warn!(repo = %name, err = %msg, "git receive-pack: bad client request") - } - _ => tracing::error!(repo = %name, err = %e, "git receive-pack failed"), - } - app - })?; - - // Update the repo's updated_at timestamp after a successful push + // If no ref landed, return 200 with the receive-pack body but do + // NOT run any durable effects (no push event, no trust score, + // no metrics, no webhooks, no certs, no anchor jobs). The + // outbox rows have already been flipped to `cancelled` / + // `uncertain` for every ref above; the next startup reconcile + // will not promote them. + if !any_ref_ok { + return axum::response::Response::builder() + .status(axum::http::StatusCode::OK) + .header("Content-Type", "application/x-git-receive-pack-result") + .header("Cache-Control", "no-cache") + .body(axum::body::Body::from(receive_raw)) + .map_err(|e| AppError::Internal(anyhow::anyhow!("failed to build response: {e}"))); + } + + // #26 Split PR 1 step 3 — the per-ref effects fan-out moved into + // `apply_request_effects`. The live handler and the recovery + // drain call the same function, so the live and recovery paths + // produce identical artifact ids and the request row's + // `accepted_ordinal` is the single source of truth for the push + // event identity. A `Retry` outcome here means one or more + // per-ref effects failed transiently; the request is left in + // `effects_pending` for the drain to pick up on the next + // startup. A `Nothing` outcome means the request had no + // accepted ref (the four-branch flip above would have caught + // that case via `any_ref_ok`, so this is defensive). let _ = state.db.touch_repo(&record.id).await; - - // Record the successful push for metrics. The body has already been - // consumed by smart_http::receive_pack so we observe size up front. crate::metrics::record_push(&record.id); crate::metrics::observe_pack_size(body_len as f64); - // Record push event for trust score and issue a signed ref certificate. - // The route is behind `require_signature`, so the verified pusher identity is - // always present; use it directly rather than re-parsing the headers. - let did = auth.0.as_str(); - { - // Use the first new commit hash we parsed, fall back to timestamp - let commit_hash = ref_updates - .first() - .map(|u| u.new_sha.clone()) - .unwrap_or_else(|| Utc::now().timestamp().to_string()); - - let _ = state.db.record_push(did, &record.id, &commit_hash, 0).await; - if let Ok(push_count) = state.db.get_push_count(did).await { - // 0.05 base (from registration) + 0.05 per push, capped at 1.0 - // 1 push → 0.10, 5 pushes → 0.30, 19 pushes → 1.0 - let new_score = (push_count as f64 * 0.05 + 0.05).min(1.0); - let _ = state.db.update_trust_score(did, new_score).await; + match crate::durable_outbox::apply_request_effects(&state, &request_id).await { + Ok(crate::durable_outbox::EffectsOutcome::Done) => { + if let Err(e) = state.db.mark_request_complete(&request_id).await { + tracing::warn!( + err = %e, + request_id = %request_id, + repo = %name, + "live path: mark_request_complete failed; drain will pick up" + ); + } } - - // Issue a signed certificate for every ref this push advanced, each - // carrying that ref's real old→new transition. A multi-ref push must - // not collapse to a single cert covering only the first ref. - for update in &ref_updates { - match cert::issue_ref_certificate( - &state, - &record.id, - &update.ref_name, - &update.old_sha, - &update.new_sha, - did, - ) - .await + Ok(crate::durable_outbox::EffectsOutcome::Nothing) => { + // No accepted ref (defensive — `any_ref_ok` gates the + // call site, so this branch is unreachable in practice). + // Mark complete so the drain skips the request. + if let Err(e) = state.db.mark_request_complete(&request_id).await { + tracing::warn!( + err = %e, + request_id = %request_id, + repo = %name, + "live path: mark_request_complete (Nothing) failed" + ); + } + } + Ok(crate::durable_outbox::EffectsOutcome::Retry { last_error }) => { + let next_attempt_at = (Utc::now() + chrono::Duration::seconds(60)).to_rfc3339(); + if let Err(e) = state + .db + .mark_request_effects_pending(&request_id, &next_attempt_at, &last_error) + .await { - Ok(c) => { - tracing::info!(cert_id = %c.id, repo = %record.name, ref_name = %update.ref_name, pusher = %did, "issued ref certificate") - } - Err(e) => { - tracing::warn!(err = %e, ref_name = %update.ref_name, "failed to issue ref certificate") - } + tracing::warn!( + err = %e, + request_id = %request_id, + repo = %name, + "live path: mark_request_effects_pending failed; drain will retry" + ); } } - } - - // Fire push webhooks — one per ref update - if !ref_updates.is_empty() { - let base_url = state - .config - .public_url - .as_deref() - .unwrap_or("http://127.0.0.1:7545") - .trim_end_matches('/'); - let owner_short = crate::db::normalize_owner_key(&record.owner_did); - let clone_url = format!("{}/{}/{}.git", base_url, owner_short, record.name); - - for update in &ref_updates { - let payload = serde_json::json!({ - "ref": update.ref_name, - "before": update.old_sha, - "after": update.new_sha, - "created": update.old_sha == ZERO_SHA, - "forced": false, - "pusher": { - "did": did, - }, - "repository": { - "id": record.id, - "name": record.name, - "owner_did": record.owner_did, - "clone_url": clone_url, - }, - }); - webhooks::fire_event( - state.db.clone(), - state.http_client.clone(), - &record.id, - "push", - payload, + Err(e) => { + tracing::error!( + err = %e, + request_id = %request_id, + repo = %name, + "live path: apply_request_effects returned Err; request left for drain" ); } } - Ok(result) + axum::response::Response::builder() + .status(axum::http::StatusCode::OK) + .header("Content-Type", "application/x-git-receive-pack-result") + .header("Cache-Control", "no-cache") + .body(axum::body::Body::from(receive_raw)) + .map_err(|e| AppError::Internal(anyhow::anyhow!("failed to build response: {e}"))) } /// The detached post-receive replication tail (#174 F2): everything a landed push @@ -3147,10 +3653,10 @@ pub async fn get_icaptcha_proof( /// replication tail at the durability boundary while the certificate and webhook /// loops below still iterate their own copy (#174 U5). #[derive(Clone)] -struct RefUpdate { - old_sha: String, - new_sha: String, - ref_name: String, +pub(crate) struct RefUpdate { + pub(crate) old_sha: String, + pub(crate) new_sha: String, + pub(crate) ref_name: String, } /// Parse git receive-pack pkt-line ref updates from the request body. diff --git a/crates/gitlawb-node/src/cert.rs b/crates/gitlawb-node/src/cert.rs index 0ed50418e..69eacffb6 100644 --- a/crates/gitlawb-node/src/cert.rs +++ b/crates/gitlawb-node/src/cert.rs @@ -11,10 +11,41 @@ use uuid::Uuid; use crate::db::RefCertificate; use crate::state::AppState; -/// Issue a signed ref-update certificate for a successful push. +/// Issue a signed ref-update certificate for a successful push. The +/// live receive-pack handler calls this on every successful push. /// -/// Builds a canonical JSON payload, signs it with the node's Ed25519 key, -/// persists the certificate, and returns it. +/// `cert_id` is the deterministic id derived from `(request_id, +/// ref_name)` (see [`crate::db::ref_cert_id_for`]). It is required so +/// the recovery drain and the live handler produce the same primary +/// key: a live push followed by a recovery pass collapses to a +/// single cert row, and a re-push to the same `(repo, ref)` updates +/// the existing row's `old_sha` / `new_sha` / `pusher_did` / +/// `issued_at` / `signature` to the new transition while preserving +/// the original `id` (the `insert_ref_certificate` upsert is +/// keyed on `(repo_id, ref_name)` and only updates fields when the +/// new `issued_at` is strictly greater). +/// +/// #26 Split PR 1 P1-B: the live handler routes through this +/// function (the upsert), NOT through +/// [`issue_ref_certificate_idempotent`] (DO NOTHING). After the +/// reviewer-1 round-2 fix, the recovery drain also routes through +/// this function (P1: refresh a stale cert), so both paths use the +/// same deterministic `cert_id` and the same upsert. A re-pass is +/// always safe: +/// +/// - Live handler → live upsert: re-push updates the row, preserves +/// the original `id`. The contract pinned by +/// `insert_ref_certificate_upserts_on_repo_ref` is restored. +/// - Live handler → recovery: live's `ON CONFLICT (id) DO UPDATE` +/// preserves the original `id`; the recovery's same upsert is +/// a no-op for an equal-`issued_at` re-run and a refresh for a +/// strictly-newer one. +/// - Recovery → live handler: the recovery wrote a row with the +/// deterministic `id`; the live upsert (which preserves `id` and +/// only updates other fields when `issued_at` is strictly newer) +/// is a no-op for an equal-`issued_at` re-run and a refresh for +/// a strictly-newer one. +#[allow(dead_code)] // round-trip test in db/mod.rs pins the upsert contract; the live path and the drain use issue_ref_certificate_with_issued_at pub async fn issue_ref_certificate( state: &AppState, repo_id: &str, @@ -22,9 +53,131 @@ pub async fn issue_ref_certificate( old_sha: &str, new_sha: &str, pusher_did: &str, + cert_id: &str, +) -> Result { + issue_ref_certificate_with_issued_at( + state, repo_id, ref_name, old_sha, new_sha, pusher_did, cert_id, None, + ) + .await +} + +/// #26 Split PR 1 round 4 — variant that lets the caller stamp the +/// cert's `issued_at` with a transition-time timestamp instead of +/// `Utc::now()`. The recovery drain passes the persisted +/// `row.created_at` so a replay after a later live cert does not +/// outrank the live cert in the `EXCLUDED.issued_at > +/// ref_certificates.issued_at` upsert guard. +/// +/// The live handler uses the default `issue_ref_certificate` (no +/// override), which keeps `Utc::now()` — the reviewer's invariant +/// is that `issued_at` reflects the transition time, and for a +/// live push the transition time and the wall-clock are the same. +/// +/// `issued_at_override` is honored verbatim; passing a value not in +/// RFC 3339 form is a logic bug (the upsert will mis-order), so +/// callers must use the row's persisted `created_at`. +/// +/// # Clippy allow — too many arguments +/// This is the explicit "stamp a transition-time `issued_at`" +/// variant of `issue_ref_certificate`. The drain +/// (`durable_outbox::derive_one`) is the in-crate caller; the +/// test `replay_of_stale_row_does_not_overwrite_live_cert_b` pins +/// the contract that a recovery replay's `issued_at` does NOT +/// outrank a later live cert. Adding a struct-arg would be a +/// larger refactor for two callers (live + drain) and obscure the +/// parallel to `issue_ref_certificate` (which is `#[allow]`'d for +/// the same reason historically). +#[allow(clippy::too_many_arguments)] +pub async fn issue_ref_certificate_with_issued_at( + state: &AppState, + repo_id: &str, + ref_name: &str, + old_sha: &str, + new_sha: &str, + pusher_did: &str, + cert_id: &str, + issued_at_override: Option, +) -> Result { + let cert = build_ref_certificate( + state, + repo_id, + ref_name, + old_sha, + new_sha, + pusher_did, + Some(cert_id.to_string()), + issued_at_override, + ) + .await?; + state.db.insert_ref_certificate(&cert).await +} + +/// #26 Split PR 1 — idempotent variant. +/// +/// `cert_id` is the deterministic id derived from +/// `(request_id, ref_name)` so a recovery re-pass against the same +/// transition produces the same primary key. The insert uses +/// `ON CONFLICT (repo_id, ref_name) DO NOTHING` (the existing +/// `insert_ref_certificate_idempotent` helper), so the function +/// returns `None` if a live-path cert already exists for the +/// `(repo_id, ref_name)` pair, and `Some(cert)` if it wrote a new +/// one. +/// +/// Retained for any future caller that wants DO-NOTHING semantics +/// (e.g. an explicit "never overwrite" handler); the live and +/// recovery paths both use [`issue_ref_certificate`] (the upsert) +/// after the P1 fix in #26 Split 1 round 2. +#[allow(dead_code)] +pub async fn issue_ref_certificate_idempotent( + state: &AppState, + repo_id: &str, + ref_name: &str, + old_sha: &str, + new_sha: &str, + pusher_did: &str, + cert_id: &str, +) -> Result> { + let cert = build_ref_certificate( + state, + repo_id, + ref_name, + old_sha, + new_sha, + pusher_did, + Some(cert_id.to_string()), + None, + ) + .await?; + state.db.insert_ref_certificate_idempotent(&cert).await +} + +/// Shared cert construction: build the JSON payload, sign it with the +/// node key, and assemble the `RefCertificate` row. `cert_id_override` +/// lets the recovery path plug in a deterministic id; the live path +/// passes `None` and gets a fresh UUID. `issued_at_override` lets +/// the recovery path stamp the cert with the original transition +/// time so the upsert's `issued_at > issued_at` guard correctly +/// orders transitions regardless of write order. +#[allow(clippy::too_many_arguments)] +async fn build_ref_certificate( + state: &AppState, + repo_id: &str, + ref_name: &str, + old_sha: &str, + new_sha: &str, + pusher_did: &str, + cert_id_override: Option, + issued_at_override: Option, ) -> Result { let node_did = state.node_did.to_string(); - let issued_at = Utc::now().to_rfc3339(); + // P1 (reviewer-1 round 4): when the caller passes a transition- + // time `issued_at` (the recovery drain passes `row.created_at`), + // use it verbatim so the upsert's per-column guard + // `EXCLUDED.issued_at > ref_certificates.issued_at` correctly + // orders transitions regardless of write order. The live handler + // passes `None` and gets `Utc::now()` — for a live push the + // transition time and the wall-clock are the same. + let issued_at = issued_at_override.unwrap_or_else(|| Utc::now().to_rfc3339()); // Build the canonical signing payload. let payload = serde_json::json!({ @@ -40,8 +193,9 @@ pub async fn issue_ref_certificate( let signature = state.node_keypair.sign_b64(&payload_bytes); - let cert = RefCertificate { - id: Uuid::new_v4().to_string(), + let id = cert_id_override.unwrap_or_else(|| Uuid::new_v4().to_string()); + Ok(RefCertificate { + id, repo_id: repo_id.to_string(), ref_name: ref_name.to_string(), old_sha: old_sha.to_string(), @@ -50,9 +204,5 @@ pub async fn issue_ref_certificate( node_did, signature, issued_at, - }; - - // Persist and return the row as it exists in the database (on a - // conflict the existing row survives when it is newer). - state.db.insert_ref_certificate(&cert).await + }) } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 1fbf4376e..2149da9b4 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -702,6 +702,46 @@ pub struct Config { value_parser = clap::builder::RangedU64ValueParser::::new().range(0..=86_400) )] pub pin_repair_sweep_delay_secs: u64, + + /// #26 Split PR 1 step 4 — receive-pack queue retention window. + /// Terminal `complete` and `rejected_at_git` rows older than + /// this are eligible for the periodic purge. `quarantined` + /// rows are never purged on a timer. The v30 partial index + /// `idx_receive_pack_requests_completed_at` keeps the scan + /// cheap regardless of the value. + #[arg( + long, + env = "GITLAWB_QUEUE_RETENTION_DAYS", + default_value_t = 7, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=365) + )] + pub queue_retention_days: i64, + + /// #26 Split PR 1 step 4 — receive-pack queue purge batch size. + /// Each periodic purge pass deletes at most this many terminal + /// rows per batch. The drain and the purge share the same + /// `DRAIN_PER_PASS_LIMIT` budget; see the spawn function in + /// `main.rs` for the wiring. + #[arg( + long, + env = "GITLAWB_QUEUE_PURGE_BATCH", + default_value_t = 1000, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=100_000) + )] + pub queue_purge_batch: i64, + + /// #26 Split PR 1 step 5 — effects-executor retry bound. The + /// drain flips a request to `quarantined` after this many + /// `EffectsOutcome::Retry` returns, closing the + /// infinite-retry DoS window. Default 8 matches the spec; set + /// lower in tests. + #[arg( + long, + env = "GITLAWB_EFFECTS_MAX_ATTEMPTS", + default_value_t = 8, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1000) + )] + pub effects_max_attempts: i32, } impl Config { @@ -963,6 +1003,30 @@ mod tests { ); } + #[test] + fn queue_lifecycle_knobs_default_conservatively() { + let c = Config::parse_from(["gitlawb-node"]); + // 7-day retention matches the v30 partial index comment + // and the spec at .gravirei/plans/state-model-durable-post-receive.md. + assert_eq!(c.queue_retention_days, 7); + // 1000 rows per pass matches DRAIN_PER_PASS_LIMIT in durable_outbox. + assert_eq!(c.queue_purge_batch, 1000); + + assert_eq!( + Config::parse_from(["gitlawb-node", "--queue-retention-days", "30"]) + .queue_retention_days, + 30 + ); + assert!(Config::try_parse_from(["gitlawb-node", "--queue-retention-days", "0"]).is_err()); + assert!(Config::try_parse_from(["gitlawb-node", "--queue-retention-days", "366"]).is_err()); + + assert_eq!( + Config::parse_from(["gitlawb-node", "--queue-purge-batch", "500"]).queue_purge_batch, + 500 + ); + assert!(Config::try_parse_from(["gitlawb-node", "--queue-purge-batch", "0"]).is_err()); + } + #[test] fn ipfs_walk_per_source_defaults_and_rejects_out_of_range() { assert_eq!(Config::parse_from(["gitlawb-node"]).ipfs_walk_per_source, 4); diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index cc2cf0bd7..21e580fdd 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -1,6 +1,7 @@ use anyhow::{Context, Result}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use sqlx::{postgres::PgPoolOptions, PgPool, Row}; use std::time::Duration; use tracing::info; @@ -153,6 +154,232 @@ pub struct RefCertificate { pub issued_at: String, } +/// The lifecycle states of a row in `pending_ref_transitions`. Persisted as a +/// TEXT column with one of these string values; the constants are the canonical +/// spellings, and tests + the recovery drain all use them so a typo on one +/// side or the other cannot silently mismatch the other. +#[allow(dead_code)] // constants are used by tests + the next-slice handler +pub mod pending_state { + #[allow(dead_code)] + pub const PREPARED: &str = "prepared"; + #[allow(dead_code)] + pub const APPLIED: &str = "applied"; + #[allow(dead_code)] + pub const CANCELLED: &str = "cancelled"; + /// Receive-pack returned Err but the exit was non-zero / timed out, + /// so it is unknown whether some refs landed. The reconcile step + /// checks these rows against disk at startup the same way it + /// checks `prepared` rows, and promotes those whose target SHA + /// actually landed. + #[allow(dead_code)] + pub const UNCERTAIN: &str = "uncertain"; +} + +/// #26 Split PR 1 — durable intent row for a single (request, ref) transition. +/// +/// One row is written BEFORE `smart_http::receive_pack` runs, in state +/// `prepared`, carrying the verified pusher DID, the raw RFC 9421 signature +/// header that authorized the push, the request id, and the parsed ref +/// update. The handler then transitions the row to `applied` on Ok or +/// `cancelled` on Err. Startup recovery drains only `applied` rows. +/// +/// `request_id` is the per-handler UUID. It is the deterministic key for +/// the push event, the ref certificate, and the anchor job — those +/// artifacts derive their ids from `(request_id, ref_name)` (cert and push) +/// or `(repo_id, ref_name, old_sha, new_sha)` (anchor) so a recovery pass +/// that re-fires the same transition cannot create a second row of any +/// of them. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[allow(dead_code)] // wired by the handler refactor in the next slice +pub struct PendingRefTransition { + pub id: String, + pub request_id: String, + pub repo_id: String, + pub ref_name: String, + pub old_sha: String, + pub new_sha: String, + pub pusher_did: String, + pub node_did: String, + pub signature_header: String, + pub signature_input: String, + pub content_digest: String, + pub state: String, + pub created_at: String, + pub applied_at: Option, + pub cancelled_at: Option, + /// Zero-based position of this row in the live push's `ref_updates` + /// — the live handler assigns `0..N-1` as it walks the pkap-line + /// parsed refs in order. The push event identity and the cert + /// identity are both `(request_id, ordinal)`, so a recovery replay + /// re-derives the same artifact ids the live path produced without + /// depending on which ref happened to land first. Migration v30 + /// added this column; the live handler sets it from + /// `ref_updates.iter().enumerate()` so it is stable across live and + /// recovery. + pub ordinal: i32, + /// Snapshot of the git-side update kind at intent time: + /// `"create"`, `"update"`, `"delete"`, or `"branch-create"` / + /// `"tag-create"`. Recovery re-derives this from the per-ref + /// report if it is null, so the column is informational. Migration + /// v30 added it; older rows are `NULL`. + pub git_target_kind: Option, +} + +/// #26 Split PR 1 — anchor handoff row, owned by PR 1, consumed by PR 2. +/// +/// One row per `(repo_id, ref_name, old_sha, new_sha)` transition. The +/// recovery path inserts it on `applied` using `ON CONFLICT (id) DO +/// NOTHING` (id derived from the tuple) so re-running the drain is +/// idempotent. Split PR 2 reads the row, calls the bundler, and updates +/// `claimed_at` to take it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[allow(dead_code)] // wired by the handler refactor in the next slice +pub struct AnchorJob { + pub id: String, + pub repo_id: String, + pub ref_name: String, + pub old_sha: String, + pub new_sha: String, + pub pusher_did: String, + pub created_at: String, + pub claimed_at: Option, +} + +/// #26 Split PR 1 — request-level durability row. One per `git +/// receive-pack` call. Written in state `received` BEFORE +/// `receive_pack_raw` runs, so a node crash between intent and the +/// git return is recoverable. After git returns, the live handler +/// transitions the row to `outcomes_committed` (with `parsed_report` +/// and `accepted_ordinal` stamped) or `rejected_at_git`. The drain +/// (step 3) reads `effects_pending` rows and runs the per-ref effect +/// writes; today step 2 only reads the row to gate the push-event +/// identity on the request's `accepted_ordinal`. +/// +/// `request_bytes` is the raw HTTP body the handler received; the +/// drain could in principle re-run `git receive-pack` against it +/// after a crash, but the v30 model treats the parsed report as the +/// durable truth and the `request_bytes` column is informational. +/// `request_bytes_hash` is the SHA-256 digest of the body as raw +/// bytes (32 bytes), so a future replay can verify the row's content +/// matches what the handler saw. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[allow(dead_code)] // wired by the handler refactor in the next slice +pub struct ReceivePackRequest { + pub id: String, + pub repo_id: String, + pub pusher_did: String, + pub node_did: String, + pub request_bytes: Vec, + pub request_bytes_hash: Vec, + pub state: String, + pub git_exit_ok: Option, + pub parsed_report: Option, + pub accepted_ordinal: Option, + pub attempt_count: i32, + pub last_error: Option, + pub next_attempt_at: Option, + pub created_at: String, + pub completed_at: Option, +} + +/// #26 Split PR 1 — request-level state vocabulary. The `received` → +/// `outcomes_committed | rejected_at_git` transition happens in the +/// live handler (step 2). The `outcomes_committed → effects_pending +/// → complete` lifecycle lives in step 3's effect executor. Every +/// state-flip helper is a single SQL `UPDATE … WHERE state = `; +/// a state helper not gated on the `from` state is a bug because it +/// could clobber a row the drain is concurrently updating. +#[allow(dead_code)] // constants are used by tests + the next-slice handler +pub mod request_state { + /// The handler wrote the row but git has not yet returned. The + /// drain will not pick this row up. + #[allow(dead_code)] + pub const RECEIVED: &str = "received"; + /// Git returned, the report was parsed, and the request has + /// outcomes. The drain reads rows in this state (and its + /// retry variant `effects_pending`) and runs the per-ref + /// effect writes. Step 2 only writes this state; the + /// `effects_pending → complete` flip is step 3. + #[allow(dead_code)] + pub const OUTCOMES_COMMITTED: &str = "outcomes_committed"; + /// The drain attempted to run effects and failed; it left + /// `next_attempt_at` in the future. Step-3 territory. + #[allow(dead_code)] + pub const EFFECTS_PENDING: &str = "effects_pending"; + /// Drain succeeded. Step 3's terminal state for a successful + /// push. The request row is retained for the 7-day window + /// the v30 partial index on `completed_at` is built for. + #[allow(dead_code)] + pub const COMPLETE: &str = "complete"; + /// Git returned with a non-zero exit and no parseable report. + /// No effects were ever run; the request row is terminal. + /// The on-disk state of the children's refs is left to the + /// reconcile step (the children remain in `prepared`). + #[allow(dead_code)] + pub const REJECTED_AT_GIT: &str = "rejected_at_git"; + /// Operator-attended terminal state. The reconcile gates on + /// the git-side marker (see `durable_outbox::reconcile_prepared_page`) + /// and quarantines the request if the marker is missing or + /// hash-mismatched. The drain's `effects_max_attempts` bound + /// also flips retry-stuck requests here. No auto-recovery; an + /// operator inspects and reclassifies to `complete` or + /// `rejected_at_git` after manual inspection. Never purged + /// by the step-4 bounded retirement policy. + #[allow(dead_code)] + pub const QUARANTINED: &str = "quarantined"; +} + +/// SHA-256 hex of an arbitrary tuple, used as the deterministic id for the +/// artifacts that recovery inserts idempotently. Returns 64 lowercase hex +/// characters. The input is concatenated with `\x1f` (ASCII Unit Separator) +/// as the field separator so two distinct tuples can never collide by +/// accidental prefix overlap, e.g. `(a, bc)` and `(ab, c)` would otherwise +/// produce the same hash input. +#[allow(dead_code)] // called from tests + the next-slice handler refactor +pub fn deterministic_id(parts: &[&str]) -> String { + let mut hasher = Sha256::new(); + for part in parts { + hasher.update(b"\x1f"); + hasher.update(part.as_bytes()); + } + hasher.update(b"\x1e"); // end-of-record terminator; never appears in any field + let digest = hasher.finalize(); + hex::encode(digest) +} + +/// Deterministic id for a push event row. Derived from +/// `(request_id, ordinal)` so a recovery pass re-firing the same +/// transition produces the same id and the ON CONFLICT collapses to a +/// no-op rather than creating a second push event. Migration v30 +/// made the request's `accepted_ordinal` the carrier of the push +/// event identity, so this helper takes the ordinal the request row +/// stamps at `mark_request_outcomes_committed` time. +#[allow(dead_code)] // wired by the handler refactor in the next slice +pub fn push_event_id_for(request_id: &str, ordinal: i32) -> String { + deterministic_id(&["push_event", request_id, &ordinal.to_string()]) +} + +/// Deterministic id for a ref certificate row. Derived from +/// `(request_id, ordinal)` for the same idempotency reason as +/// `push_event_id_for`. The certificate's `id` column is the primary +/// key; the unique index on `(repo_id, ref_name)` still applies, so +/// the recovery path must additionally check for an existing cert +/// before inserting to avoid the upsert replacing a live-path cert. +#[allow(dead_code)] // wired by the handler refactor in the next slice +pub fn ref_cert_id_for(request_id: &str, ordinal: i32) -> String { + deterministic_id(&["ref_cert", request_id, &ordinal.to_string()]) +} + +/// Deterministic id for an anchor job. The anchor's uniqueness contract +/// is per-transition, not per-request, because two different pushes to +/// the same ref (different `request_id`) should still produce ONE +/// anchor per landed state. The key is the transition tuple +/// `(repo_id, ref_name, old_sha, new_sha)`. +#[allow(dead_code)] // wired by the handler refactor in the next slice +pub fn anchor_job_id_for(repo_id: &str, ref_name: &str, old_sha: &str, new_sha: &str) -> String { + deterministic_id(&["anchor_job", repo_id, ref_name, old_sha, new_sha]) +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PeerRecord { pub did: String, @@ -1123,6 +1350,253 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE pin_repair_sweep ADD COLUMN IF NOT EXISTS discovery_cursor_id TEXT NOT NULL DEFAULT ''", ], }, + Migration { + version: 27, + name: "pending_ref_transitions_durable_outbox", + stmts: &[ + // #26 Split PR 1 — durable post-receive lifecycle. + // + // The pre-outbox crash window the reviewer flagged: receive_pack can + // apply a ref to disk and return Ok, and a process exit, a dropped + // future, or a DB failure before the bookkeeping at + // crates/gitlawb-node/src/api/repos.rs:2361 (push event + cert + + // webhook) loses the recovery record. Startup drain enumerates only + // sources written from that bookkeeping, so it cannot reconstruct + // the missing work. The partial fallback that re-derives from a row + // present in the bookkeeping substitutes `did:key:recovered` and an + // empty attestation — not equivalent to the original authenticated + // push. + // + // The fix is to persist the authentic intent BEFORE the receive_pack + // call lands the ref. The row carries the verified pusher DID, the + // raw RFC 9421 signature header that authorized this push, the + // request id, and the parsed ref updates. The receive_pack call + // then transitions the row `prepared` → `applied` on Ok, or + // `prepared` → `cancelled` on Err. Startup recovery drains only + // `applied` rows, re-deriving the push event, the per-ref + // certificate (carrying the ORIGINAL pusher DID, not a placeholder), + // and the anchor handoff — exactly once per transition. + // + // `cancelled` rows are NEVER promoted. A failed or dropped + // receive_pack leaves the row in `prepared`; only the post-Ok code + // flips to `applied`, and only that state is drained. This is what + // closes the reviewer's second proof: a prepared intent that never + // lands cannot become a push event, a certificate, or an anchor. + // + // `request_id` is a per-handler UUID. It is the producer of the + // deterministic ids for the push event, the certificate, and the + // anchor job, so re-running recovery is idempotent on + // `(request_id, ref_name)` — the unique key. + // + // `signature_header` is the raw `Signature` request header value, + // the `keyid` is the pusher DID (already extracted to `pusher_did`). + // It is kept for audit, not re-verified on recovery: the + // `require_signature` middleware already verified it before the + // handler ran, and the route is gated by it. + r#"CREATE TABLE IF NOT EXISTS pending_ref_transitions ( + id TEXT NOT NULL PRIMARY KEY, + request_id TEXT NOT NULL, + repo_id TEXT NOT NULL, + ref_name TEXT NOT NULL, + old_sha TEXT NOT NULL, + new_sha TEXT NOT NULL, + pusher_did TEXT NOT NULL, + node_did TEXT NOT NULL, + signature_header TEXT NOT NULL, + signature_input TEXT NOT NULL, + content_digest TEXT NOT NULL, + state TEXT NOT NULL, + created_at TEXT NOT NULL, + applied_at TEXT, + cancelled_at TEXT + )"#, + // The drain order is by `applied_at ASC NULLS LAST, id ASC` so a + // crashed node that re-runs the drain processes transitions in the + // order they were applied. The `id` tiebreaker keeps the order + // stable when many transitions land in the same `applied_at` tick. + "CREATE INDEX IF NOT EXISTS idx_pending_ref_transitions_state_applied_at ON pending_ref_transitions (state, applied_at, id)", + "CREATE INDEX IF NOT EXISTS idx_pending_ref_transitions_request_ref ON pending_ref_transitions (request_id, ref_name)", + "CREATE INDEX IF NOT EXISTS idx_pending_ref_transitions_repo_ref ON pending_ref_transitions (repo_id, ref_name, old_sha, new_sha)", + // The anchor handoff for Split PR 2 to consume. Split PR 1 owns + // the durable queue: one row per (repo, ref, old, new) transition + // whose row in pending_ref_transitions is `applied`. ON CONFLICT + // DO NOTHING on the unique key makes the recovery re-derivation + // idempotent — a second drain pass cannot create a second anchor + // upload request. Split PR 2 owns the actual transport and the + // three-outcome probe; this PR only proves the handoff is + // exactly-once. + r#"CREATE TABLE IF NOT EXISTS anchor_jobs ( + id TEXT NOT NULL PRIMARY KEY, + repo_id TEXT NOT NULL, + ref_name TEXT NOT NULL, + old_sha TEXT NOT NULL, + new_sha TEXT NOT NULL, + pusher_did TEXT NOT NULL, + created_at TEXT NOT NULL, + claimed_at TEXT + )"#, + "CREATE UNIQUE INDEX IF NOT EXISTS idx_anchor_jobs_repo_ref_transition ON anchor_jobs (repo_id, ref_name, old_sha, new_sha)", + "CREATE INDEX IF NOT EXISTS idx_anchor_jobs_claimed_at ON anchor_jobs (claimed_at, id)", + ], + }, + Migration { + version: 28, + name: "pending_ref_transitions_add_first_ref_name", + stmts: &[ + // #26 Split PR 1 P2-B: the recovery drain must reproduce the + // live path's push event cardinality, which is "one push event + // per push, keyed on the first ref name". Without a persisted + // `first_ref_name` column, the drain would key each outbox + // row on its own `ref_name` and emit N push events for an + // N-ref push, over-counting `get_push_count` and inflating + // the trust score. + // + // The `NOT NULL DEFAULT ''` is required to add a NOT NULL + // column to a non-empty table in a single ALTER; a follow-up + // UPDATE backfills the value to `ref_name` for every + // historic row. Old rows in `applied` state that the drain + // processes will produce one push event per row (the + // pre-fix cardinality) — an accepted upgrade-window quirk + // with no historic state to regress. New rows written by + // the live handler always carry the request's actual + // `first_ref_name`. + "ALTER TABLE pending_ref_transitions ADD COLUMN IF NOT EXISTS first_ref_name TEXT NOT NULL DEFAULT ''", + "UPDATE pending_ref_transitions SET first_ref_name = ref_name WHERE first_ref_name = ''", + ], + }, + Migration { + version: 29, + name: "pending_ref_transitions_add_uncertain_state", + stmts: &[ + // No schema change: the `state` column is TEXT and the new + // `uncertain` value is written by the application layer. + // The comment-only migration documents the state-machine + // extension so the migration test's non-empty-stmts + // assertion is satisfied. + "COMMENT ON TABLE pending_ref_transitions IS 'v29: added uncertain state for receive-pack errors where some refs may have landed'", + ], + }, + Migration { + // #26 Split PR 1 round 6 — request-level data model. + // + // Reviewer finding: the request's push event was encoded into + // a mutable per-ref column (`first_ref_name`) whose correct + // value is knowable only after git, and the correction was + // not committed atomically with the per-ref outcomes. A + // crash between git updating a later ref and the rewrite + // left the durable rows naming the rejected ref, and + // `derive_one`'s `row.ref_name == row.first_ref_name` guard + // then meant no push event ever landed for the accepted + // child. + // + // The state-transition model (see + // .gravirei/plans/state-model-durable-post-receive.md) + // replaces `first_ref_name` with a request-level record + // that owns the push event and the trust score. The per-ref + // child becomes an ordinal child of that request. The + // push event id is keyed on + // `(request_id, accepted_ordinal)` — not on `ref_name` — + // so a mixed first-rejected/later-accepted push still + // produces exactly one push event under the request that + // did land. + // + // Version is 30 on this branch (the v29 migration is the + // highest here; the cert-compat branch had renumbered its + // own v28 to v37 independently). Re-check the floor on + // every push — the open PR list moves it. + version: 30, + name: "receive_pack_requests", + stmts: &[ + // The new request table. The push event and trust + // score are written in the same database transaction as + // the per-ref child outcomes, so a crash between git + // and effect-write rolls everything back together; the + // drain sees a coherent row in `outcomes_committed` + // (or its retry variant) and re-runs the same effect + // pipeline. + r#"CREATE TABLE IF NOT EXISTS receive_pack_requests ( + id TEXT NOT NULL PRIMARY KEY, + repo_id TEXT NOT NULL, + pusher_did TEXT NOT NULL, + node_did TEXT NOT NULL, + request_bytes BYTEA NOT NULL, + request_bytes_hash BYTEA NOT NULL, + state TEXT NOT NULL, + git_exit_ok BOOLEAN, + parsed_report JSONB, + accepted_ordinal INTEGER, + attempt_count INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + next_attempt_at TEXT, + created_at TEXT NOT NULL, + completed_at TEXT + )"#, + // The state-transition gate: the recovery drain + // selects rows in `outcomes_committed` (and its retry + // variant) and walks them by `(created_at, id)`. A + // composite index lets the drain do a single index scan + // without sorting. + "CREATE INDEX IF NOT EXISTS idx_receive_pack_requests_state_created ON receive_pack_requests (state, created_at, id)", + // The drain's retry predicate. `next_attempt_at IS NULL OR + // next_attempt_at < now()` is a frequent lookup; the + // partial index keeps the index small. + "CREATE INDEX IF NOT EXISTS idx_receive_pack_requests_state_next_attempt ON receive_pack_requests (state, next_attempt_at) WHERE state IN ('outcomes_committed', 'effects_pending')", + // The 7-day bounded-retirement predicate. The purge + // task deletes `complete` and `rejected_at_git` rows + // older than the retention interval. + "CREATE INDEX IF NOT EXISTS idx_receive_pack_requests_completed_at ON receive_pack_requests (completed_at) WHERE state IN ('complete', 'rejected_at_git')", + // The new ordinal column on the per-ref child. The + // drain and the effect executor both read this in + // `ORDER BY request_id, ordinal` order to reproduce + // the live path's ref-walk sequence. + "ALTER TABLE pending_ref_transitions ADD COLUMN IF NOT EXISTS ordinal INTEGER NOT NULL DEFAULT 0", + // The git-side marker's snapshot kind. Recovery + // re-derives this from the per-ref report if it is + // null, so the column is informational and the + // migration does not need to backfill it. + "ALTER TABLE pending_ref_transitions ADD COLUMN IF NOT EXISTS git_target_kind TEXT", + // Drop `first_ref_name`. The push event identity is + // now `(request_id, accepted_ordinal)` and lives on + // the request row, not on a child. The live handler + // never writes this column after this migration; the + // drain and the effect executor do not read it. The + // column is `IF EXISTS` so a fresh database that never + // ran v28 is unaffected. + // + // P3 (reviewer round 5): the recovery gate had been + // patching `first_ref_name` after git returned; the + // patch is the bug, the drop closes it. The model + // forbids the pattern (request-level event identity + // is encoded in mutable per-ref state) so removing + // the column is the structural fix, not a workaround. + "ALTER TABLE pending_ref_transitions DROP COLUMN IF EXISTS first_ref_name", + // Document the new relationship. The `comment on + // column` form leaves a discoverable note for anyone + // reading the schema in psql. + "COMMENT ON COLUMN pending_ref_transitions.ordinal IS 'v30: ordinal position in the parsed ref_updates list, 0-indexed. The drain and the effect executor read in ORDER BY request_id, ordinal to reproduce the live path'", + ], + }, + Migration { + // #26 Split PR 1 — step 5. The `quarantined` state is the + // operator-attended terminal state for requests whose + // git-side marker is missing or hash-mismatched (or whose + // attempt_count exceeds the configured bound). The schema + // does not need a CHECK constraint change because the + // `state` column is TEXT; this migration is a comment + + // index. + version: 31, + name: "receive_pack_requests_quarantined", + stmts: &[ + // Operator-attended state. Pinned in a comment so a + // reader of the schema in psql finds the convention. + "COMMENT ON TABLE receive_pack_requests IS 'v31: added quarantined state for marker-mismatch / reflog-ambiguity / max-attempts; operator reclassifies to complete or rejected_at_git'", + // Operator queries (e.g. `SELECT … WHERE state = + // 'quarantined' ORDER BY created_at`) need an index. A + // partial index on a low-cardinality state column is + // small and cheap. + "CREATE INDEX IF NOT EXISTS idx_receive_pack_requests_quarantined ON receive_pack_requests (created_at) WHERE state = 'quarantined'", + ], + }, ]; /// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). @@ -1760,6 +2234,7 @@ impl Db { Ok(()) } + #[allow(dead_code)] // legacy live-path entry; PR 3 owns the deprecation decision pub async fn record_push( &self, agent_did: &str, @@ -2333,6 +2808,7 @@ impl Db { /// late-landing older cert cannot regress a ref's persisted state. Returns /// the full row as it now exists in the database (the original row on a /// rejected upsert; the passed row on insert). + #[allow(dead_code)] // legacy live-path entry; PR 3 owns the deprecation decision pub async fn insert_ref_certificate(&self, cert: &RefCertificate) -> Result { let row = sqlx::query( "INSERT INTO ref_certificates @@ -2367,6 +2843,1145 @@ impl Db { Ok(row_to_cert(row)) } + // ── #26 Split PR 1: durable post-receive outbox ──────────────────────── + // + // The methods below own the producer/persistence/restore boundary the + // reviewer flagged: every externally visible ref transition has a row + // here before the receive_pack call, the row's state reflects the + // outcome (`applied` on Ok, `cancelled` on Err), and a startup drain + // re-derives the push event, ref certificate, and anchor handoff for + // any `applied` row. Recovery is idempotent because every derived + // artifact has a deterministic id (see `*_id_for` above) and the + // `INSERT ... ON CONFLICT (id) DO NOTHING` clause collapses a + // re-fired transition to a no-op. + // + // The handler is responsible for calling `insert_prepared` before the + // `smart_http::receive_pack` call and `mark_applied` / `mark_cancelled` + // after. The `drain_applied` method is called once at startup, after + // migrations and before serving. Wiring those into the handler is + // tracked as the next slice of work; this commit adds the durable + // boundary and the DB-level idempotency the handler will lean on. + + /// Insert one `prepared` row per ref update in the push, returning the + /// rows as persisted. Called from the receive-pack handler BEFORE + /// `smart_http::receive_pack` runs. + /// + /// `request_id` is the per-handler UUID; the same value must be used + /// for every ref update in a single push, and it becomes the + /// deterministic seed for the push event, ref cert, and anchor job + /// ids. `pusher_did` is the verified DID from the + /// `AuthenticatedDid` extension (the canonical identity the + /// `require_signature` middleware injected). `signature_header` and + /// `signature_input` are the raw RFC 9421 header values, persisted + /// for audit; they were already verified at handler entry. + /// + /// `ordinal` is the zero-based position of each ref in the pkap-line + /// stream; the live handler sets it from + /// `ref_updates.iter().enumerate()`. `git_target_kind` is a snapshot + /// of the update's git-side classification (`"create"`, `"update"`, + /// `"delete"`, …). The recovery re-derives the latter from the + /// per-ref report if the column is null, so the column is + /// informational and optional. + #[allow(dead_code, clippy::too_many_arguments)] // wired by the handler refactor in the next slice + pub async fn insert_pending_ref_transitions( + &self, + request_id: &str, + repo_id: &str, + node_did: &str, + pusher_did: &str, + ref_updates: &[crate::api::repos::RefUpdate], + signature_header: &str, + signature_input: &str, + content_digest: &str, + ) -> Result> { + let now = Utc::now().to_rfc3339(); + // P2 (reviewer-2 round 2): wrap the multi-row insert in a + // transaction. A mid-loop failure used to return the error + // and leave the rows already inserted as `prepared`, which + // the receive-pack handler then refused to call. The + // stranded `prepared` rows were eventually reaped by the + // startup reconcile, but the partial-success state was + // observable in the DB and could mask a partial push + // intent. The transaction rolls the prior inserts back + // when any single row fails, so the caller either sees a + // complete `prepared` set for the request or sees none of + // them and the handler can safely return 503. + let mut tx = self.pool.begin().await?; + let mut out = Vec::with_capacity(ref_updates.len()); + for (ordinal, update) in ref_updates.iter().enumerate() { + let ordinal_i32 = ordinal as i32; + let id = deterministic_id(&[ + "pending_ref_transition", + request_id, + repo_id, + &update.ref_name, + &update.old_sha, + &update.new_sha, + ]); + sqlx::query( + r#"INSERT INTO pending_ref_transitions + (id, request_id, repo_id, ref_name, old_sha, new_sha, pusher_did, node_did, + signature_header, signature_input, content_digest, state, created_at, + ordinal, git_target_kind) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)"#, + ) + .bind(&id) + .bind(request_id) + .bind(repo_id) + .bind(&update.ref_name) + .bind(&update.old_sha) + .bind(&update.new_sha) + .bind(pusher_did) + .bind(node_did) + .bind(signature_header) + .bind(signature_input) + .bind(content_digest) + .bind(pending_state::PREPARED) + .bind(&now) + .bind(ordinal_i32) + .bind(Option::::None) + .execute(&mut *tx) + .await?; + out.push(PendingRefTransition { + id, + request_id: request_id.to_string(), + repo_id: repo_id.to_string(), + ref_name: update.ref_name.clone(), + old_sha: update.old_sha.clone(), + new_sha: update.new_sha.clone(), + pusher_did: pusher_did.to_string(), + node_did: node_did.to_string(), + signature_header: signature_header.to_string(), + signature_input: signature_input.to_string(), + content_digest: content_digest.to_string(), + state: pending_state::PREPARED.to_string(), + created_at: now.clone(), + applied_at: None, + cancelled_at: None, + ordinal: ordinal_i32, + git_target_kind: None, + }); + } + tx.commit().await?; + Ok(out) + } + + // ── request-level surface (#26 Split PR 1 step 2) ──────────── + + /// Insert a `receive_pack_requests` row in state `received`. Step 2 + /// calls this from the handler's intent path BEFORE + /// `smart_http::receive_pack` runs; a node crash after this point + /// and before the live outcomes commit leaves the row in + /// `received` and its children in `prepared`, which the reconcile + /// step (already on this branch) handles via on-disk SHA + reflog + /// proof. + /// + /// The insert is single-row; the matching children are written by + /// the handler's existing `insert_pending_ref_transitions` call in + /// the SAME transaction boundary. Step 2 does not introduce a + /// "with-children" wrapper — the handler's call ordering is the + /// contract, and the tests pin the two writes as a pair. + pub async fn insert_receive_pack_request(&self, req: &ReceivePackRequest) -> Result<()> { + sqlx::query( + r#"INSERT INTO receive_pack_requests + (id, repo_id, pusher_did, node_did, request_bytes, request_bytes_hash, + state, git_exit_ok, parsed_report, accepted_ordinal, attempt_count, + last_error, next_attempt_at, created_at, completed_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)"#, + ) + .bind(&req.id) + .bind(&req.repo_id) + .bind(&req.pusher_did) + .bind(&req.node_did) + .bind(&req.request_bytes) + .bind(&req.request_bytes_hash) + .bind(&req.state) + .bind(req.git_exit_ok) + .bind(req.parsed_report.as_ref()) + .bind(req.accepted_ordinal) + .bind(req.attempt_count) + .bind(req.last_error.as_deref()) + .bind(req.next_attempt_at.as_deref()) + .bind(&req.created_at) + .bind(req.completed_at.as_deref()) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Read a single `receive_pack_requests` row by id. Used by + /// `durable_outbox::apply_request_effects` to load the + /// request's state, `accepted_ordinal`, and parsed report + /// before re-deriving per-ref artifacts. + pub async fn get_receive_pack_request( + &self, + request_id: &str, + ) -> Result> { + let row = sqlx::query( + r#"SELECT id, repo_id, pusher_did, node_did, request_bytes, request_bytes_hash, + state, git_exit_ok, parsed_report, accepted_ordinal, attempt_count, + last_error, next_attempt_at, created_at, completed_at + FROM receive_pack_requests WHERE id = $1"#, + ) + .bind(request_id) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(row_to_receive_pack_request)) + } + + /// `received → outcomes_committed`. The handler calls this once + /// per request, with the parsed report, the git exit, and the + /// ordinal of the first ref the report proves landed. The state + /// gate in the WHERE clause means a concurrent drain cannot + /// re-flip a row the handler is mid-update. + pub async fn mark_request_outcomes_committed( + &self, + request_id: &str, + git_exit_ok: bool, + parsed_report: &serde_json::Value, + accepted_ordinal: Option, + ) -> Result { + let res = sqlx::query( + r#"UPDATE receive_pack_requests + SET state = $2, git_exit_ok = $3, parsed_report = $4, + accepted_ordinal = $5 + WHERE id = $1 AND state = $6"#, + ) + .bind(request_id) + .bind(request_state::OUTCOMES_COMMITTED) + .bind(git_exit_ok) + .bind(parsed_report) + .bind(accepted_ordinal) + .bind(request_state::RECEIVED) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// `received → rejected_at_git`. Step 2 calls this when git + /// returned non-zero with no parseable report. The children + /// stay in `prepared` and the reconcile step decides their + /// fate via on-disk SHA + reflog proof. + pub async fn mark_request_rejected_at_git( + &self, + request_id: &str, + last_error: Option<&str>, + ) -> Result { + let res = sqlx::query( + r#"UPDATE receive_pack_requests + SET state = $2, git_exit_ok = FALSE, last_error = $3, + completed_at = $4 + WHERE id = $1 AND state = $5"#, + ) + .bind(request_id) + .bind(request_state::REJECTED_AT_GIT) + .bind(last_error) + .bind(Utc::now().to_rfc3339()) + .bind(request_state::RECEIVED) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// #26 Split PR 1 step 5 — flip any non-terminal state to + /// `quarantined`. The reconcile calls this when the marker + /// ref is missing or hash-mismatched; the drain's + /// `effects_max_attempts` bound calls this when a request + /// has been retry-stuck for too long. Operator-attended: the + /// drain never picks up `quarantined` rows. + /// + /// The state gate is intentionally permissive: any non-terminal + /// state can be quarantined. The caller decides which state + /// the row was in before the flip. + pub async fn mark_request_quarantined(&self, request_id: &str, reason: &str) -> Result { + let res = sqlx::query( + r#"UPDATE receive_pack_requests + SET state = $2, last_error = $3, completed_at = $4 + WHERE id = $1 + AND state IN ($5, $6, $7, $8)"#, + ) + .bind(request_id) + .bind(request_state::QUARANTINED) + .bind(reason) + .bind(Utc::now().to_rfc3339()) + .bind(request_state::RECEIVED) + .bind(request_state::OUTCOMES_COMMITTED) + .bind(request_state::EFFECTS_PENDING) + .bind(request_state::REJECTED_AT_GIT) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// #26 Split PR 1 step 5 — when a request moves to + /// `quarantined`, its `prepared` children are reclassified to + /// `cancelled` so the drain's residual scan doesn't keep + /// picking them up. The `cancelled_at` is stamped at the + /// parent's quarantine time so a future operator reclassifying + /// the parent can recover the timing. + pub async fn mark_children_rejected_for_quarantined_parent( + &self, + request_id: &str, + ) -> Result { + let now = Utc::now().to_rfc3339(); + let res = sqlx::query( + r#"UPDATE pending_ref_transitions + SET state = $2, cancelled_at = $3 + WHERE request_id = $1 AND state = $4"#, + ) + .bind(request_id) + .bind(pending_state::CANCELLED) + .bind(&now) + .bind(pending_state::PREPARED) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// #26 Split PR 1 step 5 — batch-load receive_pack_requests by + /// id. The reconcile calls this once per page to avoid N+1 + /// queries when the marker gate checks every row's parent + /// request. Returns a HashMap so the per-row check is a + /// O(1) lookup. + pub async fn get_receive_pack_requests_by_ids( + &self, + ids: &[String], + ) -> Result> { + if ids.is_empty() { + return Ok(Default::default()); + } + let rows = sqlx::query( + r#"SELECT id, repo_id, pusher_did, node_did, request_bytes, request_bytes_hash, + state, git_exit_ok, parsed_report, accepted_ordinal, attempt_count, + last_error, next_attempt_at, created_at, completed_at + FROM receive_pack_requests WHERE id = ANY($1)"#, + ) + .bind(ids) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(row_to_receive_pack_request) + .map(|r| (r.id.clone(), r)) + .collect()) + } + + /// `outcomes_committed → effects_pending`. Step 3's effect + /// executor calls this when the drain picked up a request and + /// scheduled a retry. + pub async fn mark_request_effects_pending( + &self, + request_id: &str, + next_attempt_at: &str, + last_error: &str, + ) -> Result { + let res = sqlx::query( + r#"UPDATE receive_pack_requests + SET state = $2, attempt_count = attempt_count + 1, + next_attempt_at = $3, last_error = $4 + WHERE id = $1 AND state = $5"#, + ) + .bind(request_id) + .bind(request_state::EFFECTS_PENDING) + .bind(next_attempt_at) + .bind(last_error) + .bind(request_state::OUTCOMES_COMMITTED) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// `effects_pending → complete`. Step 3 calls this after a + /// successful effects run. + pub async fn mark_request_complete(&self, request_id: &str) -> Result { + let res = sqlx::query( + r#"UPDATE receive_pack_requests + SET state = $2, completed_at = $3 + WHERE id = $1 AND state IN ($4, $5)"#, + ) + .bind(request_id) + .bind(request_state::COMPLETE) + .bind(Utc::now().to_rfc3339()) + .bind(request_state::OUTCOMES_COMMITTED) + .bind(request_state::EFFECTS_PENDING) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// Drain-side read. Returns every request whose state is + /// `outcomes_committed` or `effects_pending` and whose + /// `next_attempt_at` is null or in the past. + pub async fn list_receive_pack_requests_due( + &self, + limit: i64, + ) -> Result> { + let limit = limit.max(1); + let rows = sqlx::query( + r#"SELECT id, repo_id, pusher_did, node_did, request_bytes, request_bytes_hash, + state, git_exit_ok, parsed_report, accepted_ordinal, attempt_count, + last_error, next_attempt_at, created_at, completed_at + FROM receive_pack_requests + WHERE state IN ($1, $2) + AND (next_attempt_at IS NULL OR next_attempt_at < $3) + ORDER BY created_at ASC, id ASC + LIMIT $4"#, + ) + .bind(request_state::OUTCOMES_COMMITTED) + .bind(request_state::EFFECTS_PENDING) + .bind(Utc::now().to_rfc3339()) + .bind(limit) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter().map(row_to_receive_pack_request).collect()) + } + + /// Residual-backlog check for the per-request drain. Returns + /// the count of requests in `outcomes_committed` or + /// `effects_pending` with a due `next_attempt_at`. The drain's + /// `drain_receive_pack_requests_all` uses this after the + /// residual pass to decide whether to log a warning. + pub async fn count_receive_pack_requests_due(&self) -> Result { + let row: (i64,) = sqlx::query_as( + r#"SELECT COUNT(*)::BIGINT FROM receive_pack_requests + WHERE state IN ($1, $2) + AND (next_attempt_at IS NULL OR next_attempt_at < $3)"#, + ) + .bind(request_state::OUTCOMES_COMMITTED) + .bind(request_state::EFFECTS_PENDING) + .bind(Utc::now().to_rfc3339()) + .fetch_one(&self.pool) + .await?; + Ok(row.0) + } + + /// Backoff helper for the step-3 effect executor. Step 3 + /// introduces the helper but does not call it; a future + /// refinement (per-attempt exponential backoff) will land the + /// call site. Pinning the contract here means the helper cannot + /// drift away from what the next slice will use. + #[allow(dead_code)] // call site lands in a follow-up; the helper signature is pinned here + pub async fn update_request_attempt( + &self, + request_id: &str, + attempt_count: i32, + next_attempt_at: &str, + last_error: &str, + ) -> Result { + let res = sqlx::query( + r#"UPDATE receive_pack_requests + SET attempt_count = $2, next_attempt_at = $3, last_error = $4 + WHERE id = $1"#, + ) + .bind(request_id) + .bind(attempt_count) + .bind(next_attempt_at) + .bind(last_error) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// #26 Split PR 1 step 4 — bounded retirement. Deletes terminal + /// `receive_pack_requests` rows whose `completed_at` is older + /// than `older_than_iso`. Only `complete` and `rejected_at_git` + /// rows are eligible; `outcomes_committed` / `effects_pending` + /// are never purged (the drain is responsible for them), and + /// `received` rows are never purged (the handler is + /// responsible for them). + /// + /// The `idx_receive_pack_requests_completed_at` partial index + /// (built by v30) keeps this scan cheap. PostgreSQL does not + /// accept `LIMIT` directly inside a `DELETE`, so the limit is + /// applied via a subquery selecting the ids to delete. + pub async fn purge_completed_receive_pack_requests( + &self, + older_than_iso: &str, + limit: i64, + ) -> Result { + let limit = limit.max(1); + let res = sqlx::query( + r#"DELETE FROM receive_pack_requests + WHERE id IN ( + SELECT id FROM receive_pack_requests + WHERE state IN ($1, $2) + AND completed_at IS NOT NULL + AND completed_at < $3 + LIMIT $4 + )"#, + ) + .bind(request_state::COMPLETE) + .bind(request_state::REJECTED_AT_GIT) + .bind(older_than_iso) + .bind(limit) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// #26 Split PR 1 step 4 — bounded retirement. Deletes + /// `pending_ref_transitions` children whose parent request is + /// in `complete` or `rejected_at_git` AND whose + /// `applied_at` (for accepted children) or `cancelled_at` (for + /// rejected children) is older than `older_than_iso`. Children + /// in `prepared` / `uncertain` are NEVER purged — those are + /// the reconcile walk's responsibility. + /// + /// Callers MUST purge the parent requests first so this scan + /// has a clear contract. The `purge_request_queue` helper in + /// `durable_outbox.rs` enforces the order. + pub async fn purge_completed_pending_ref_transitions( + &self, + older_than_iso: &str, + limit: i64, + ) -> Result { + let limit = limit.max(1); + let res = sqlx::query( + r#"DELETE FROM pending_ref_transitions + WHERE id IN ( + SELECT id FROM pending_ref_transitions + WHERE state IN ($1, $2) + AND ((state = $1 AND applied_at IS NOT NULL AND applied_at < $3) + OR (state = $2 AND cancelled_at IS NOT NULL AND cancelled_at < $3)) + LIMIT $4 + )"#, + ) + .bind(pending_state::APPLIED) + .bind(pending_state::CANCELLED) + .bind(older_than_iso) + .bind(limit) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// Flip every `prepared` row attached to `request_id` to `applied`. + /// Called after `smart_http::receive_pack` returns Ok. A `prepared` + /// row that the handler never reaches this point for stays in + /// `prepared` and is dropped by the drain (the row is NEVER promoted + /// by anything other than this method), which is what closes the + /// reviewer's "a failed or cancelled receive-pack must not turn a + /// prepared intent into completed accounting or anchoring" invariant. + #[allow(dead_code)] // wired by the handler refactor in the next slice + pub async fn mark_pending_ref_transitions_applied(&self, request_id: &str) -> Result { + let now = Utc::now().to_rfc3339(); + let res = sqlx::query( + r#"UPDATE pending_ref_transitions + SET state = $1, applied_at = $2 + WHERE request_id = $3 AND state = $4"#, + ) + .bind(pending_state::APPLIED) + .bind(&now) + .bind(request_id) + .bind(pending_state::PREPARED) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// Flip every `prepared` row attached to `request_id` to `cancelled`. + /// Called when the receive_pack call returns Err or the handler + /// future is dropped. The drain does not promote `cancelled` rows. + #[allow(dead_code)] + pub async fn mark_pending_ref_transitions_cancelled(&self, request_id: &str) -> Result { + let now = Utc::now().to_rfc3339(); + let res = sqlx::query( + r#"UPDATE pending_ref_transitions + SET state = $1, cancelled_at = $2 + WHERE request_id = $3 AND state = $4"#, + ) + .bind(pending_state::CANCELLED) + .bind(&now) + .bind(request_id) + .bind(pending_state::PREPARED) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// Per-ref variant of [`mark_pending_ref_transitions_applied`]: + /// flip to `applied` only the rows whose `ref_name` is in + /// `ref_names`. Used by the live handler when the report-status + /// confirms per-ref `ok` results — refs the report rejected or + /// did not mention are left alone so the next call can flip them + /// to `cancelled` / `uncertain` independently. + #[allow(dead_code)] + pub async fn mark_pending_ref_transitions_applied_for_names( + &self, + request_id: &str, + ref_names: &[&str], + ) -> Result { + if ref_names.is_empty() { + return Ok(0); + } + let now = Utc::now().to_rfc3339(); + let res = sqlx::query( + r#"UPDATE pending_ref_transitions + SET state = $1, applied_at = $2 + WHERE request_id = $3 AND state = $4 AND ref_name = ANY($5)"#, + ) + .bind(pending_state::APPLIED) + .bind(&now) + .bind(request_id) + .bind(pending_state::PREPARED) + .bind(ref_names) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// Per-ref variant of [`mark_pending_ref_transitions_cancelled`]: + /// flip to `cancelled` only the rows whose `ref_name` is in + /// `ref_names`. Used by the live handler to mark specifically the + /// refs that the report-status listed as `ng` so their durable + /// effects are skipped. + #[allow(dead_code)] + pub async fn mark_pending_ref_transitions_cancelled_for_names( + &self, + request_id: &str, + ref_names: &[&str], + ) -> Result { + if ref_names.is_empty() { + return Ok(0); + } + let now = Utc::now().to_rfc3339(); + let res = sqlx::query( + r#"UPDATE pending_ref_transitions + SET state = $1, cancelled_at = $2 + WHERE request_id = $3 AND state = $4 AND ref_name = ANY($5)"#, + ) + .bind(pending_state::CANCELLED) + .bind(&now) + .bind(request_id) + .bind(pending_state::PREPARED) + .bind(ref_names) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// Per-ref variant of [`mark_pending_ref_transitions_uncertain`]: + /// flip to `uncertain` only the rows whose `ref_name` is in + /// `ref_names`. Used by the live handler for refs that are not + /// mentioned in the report-status output and need reconcile to + /// sort out which actually landed. + #[allow(dead_code)] + pub async fn mark_pending_ref_transitions_uncertain_for_names( + &self, + request_id: &str, + ref_names: &[&str], + ) -> Result { + if ref_names.is_empty() { + return Ok(0); + } + // P2 (reviewer-2 round 4): `cancelled_at` is reserved for rows + // that were *decided* not to land. An uncertain row is by + // definition undecided, so leave `cancelled_at` null and let + // any audit reason about it from `created_at`. + let res = sqlx::query( + r#"UPDATE pending_ref_transitions + SET state = $1 + WHERE request_id = $2 AND state IN ($3, $4) AND ref_name = ANY($5)"#, + ) + .bind(pending_state::UNCERTAIN) + .bind(request_id) + .bind(pending_state::PREPARED) + .bind(pending_state::APPLIED) + .bind(ref_names) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// Count the `applied` rows remaining in the table. Used by the + /// startup drain to decide whether the residual pass has work + /// left or whether the backlog was fully consumed. + #[allow(dead_code)] + pub async fn count_pending_ref_transitions_applied(&self) -> Result { + let row = + sqlx::query("SELECT COUNT(*) AS cnt FROM pending_ref_transitions WHERE state = $1") + .bind(pending_state::APPLIED) + .fetch_one(&self.pool) + .await?; + Ok(row.get::("cnt")) + } + + /// Return every `applied` row, oldest first. The startup drain calls + /// this once and processes each row by re-deriving the push event, + /// the per-ref cert, and the anchor handoff. + #[allow(dead_code)] // wired by the handler refactor in the next slice + pub async fn list_pending_ref_transitions_applied( + &self, + limit: i64, + ) -> Result> { + let limit = limit.max(1); + let rows = sqlx::query( + r#"SELECT id, request_id, repo_id, ref_name, old_sha, new_sha, pusher_did, node_did, + signature_header, signature_input, content_digest, state, created_at, + applied_at, cancelled_at, ordinal, git_target_kind + FROM pending_ref_transitions + WHERE state = $1 + ORDER BY applied_at ASC NULLS LAST, id ASC + LIMIT $2"#, + ) + .bind(pending_state::APPLIED) + .bind(limit) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(row_to_pending_ref_transition) + .collect()) + } + + /// Return every `prepared` row, oldest first. The startup + /// `reconcile_prepared_from_disk` step enumerates these, checks + /// each row's `new_sha` against the on-disk ref via + /// `git::store::list_refs`, and promotes the rows whose target + /// actually landed to `applied`. Rows that did NOT land (ref + /// rejected by receive_pack, or a `mark_applied` error stranded + /// the row in `prepared` with the ref still on the old SHA) stay + /// in `prepared`. + #[allow(dead_code)] // wired by the startup reconcile + pub async fn list_pending_ref_transitions_prepared( + &self, + limit: i64, + ) -> Result> { + let limit = limit.max(1); + let rows = sqlx::query( + r#"SELECT id, request_id, repo_id, ref_name, old_sha, new_sha, pusher_did, node_did, + signature_header, signature_input, content_digest, state, created_at, + applied_at, cancelled_at, ordinal, git_target_kind + FROM pending_ref_transitions + WHERE state = $1 + ORDER BY created_at ASC, id ASC + LIMIT $2"#, + ) + .bind(pending_state::PREPARED) + .bind(limit) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(row_to_pending_ref_transition) + .collect()) + } + + /// Return `prepared` and `uncertain` rows, oldest first. The + /// startup reconcile step checks both states against on-disk refs + /// and promotes those that actually landed to `applied`. A + /// `prepared` row that was interrupted after receive-pack returned + /// Ok, and an `uncertain` row from a receive-pack error, are + /// equally unrecoverable without this step: the drain's WHERE + /// clause does not see them. + #[allow(dead_code)] + pub async fn list_pending_ref_transitions_prepared_or_uncertain( + &self, + limit: i64, + ) -> Result> { + self.list_pending_ref_transitions_prepared_or_uncertain_after(None, limit) + .await + } + + /// The same page of `prepared` / `uncertain` rows, in the same + /// order, resuming strictly AFTER the `(created_at, id)` cursor. + /// + /// The multi-pass reconcile needs a cursor where the multi-pass + /// drain does not, and the asymmetry is the whole reason this + /// exists. The drain DELETES every row it finishes, so its next + /// `LIMIT n` page is always new work. The reconcile leaves every + /// row it cannot promote exactly where it was, so re-issuing the + /// cursor-less query hands it the same page over and over: a + /// single unprovable row at the head of the ordering pins page one + /// and the backlog behind it is never examined at all — which is + /// the very thing the multi-pass loop was added to fix. Rows that + /// wait for another restart keep ageing toward + /// `MAX_RECONCILE_AGE`, past which they lose automatic recovery + /// entirely. + /// + /// Advancing on `(created_at, id)` also stays correct while rows + /// leave the set underneath the walk: a promoted row is simply + /// absent from a later page, and it can never shift an unvisited + /// row into a page that was already read, the way an OFFSET would. + #[allow(dead_code)] + pub async fn list_pending_ref_transitions_prepared_or_uncertain_after( + &self, + after: Option<(&str, &str)>, + limit: i64, + ) -> Result> { + let limit = limit.max(1); + // The empty sentinel sorts before every RFC 3339 timestamp, so + // the first page needs no separate query. + let (after_created_at, after_id) = after.unwrap_or(("", "")); + let rows = sqlx::query( + r#"SELECT id, request_id, repo_id, ref_name, old_sha, new_sha, pusher_did, node_did, + signature_header, signature_input, content_digest, state, created_at, + applied_at, cancelled_at, ordinal, git_target_kind + FROM pending_ref_transitions + WHERE state IN ($1, $2) AND (created_at, id) > ($3, $4) + ORDER BY created_at ASC, id ASC + LIMIT $5"#, + ) + .bind(pending_state::PREPARED) + .bind(pending_state::UNCERTAIN) + .bind(after_created_at) + .bind(after_id) + .bind(limit) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(row_to_pending_ref_transition) + .collect()) + } + + /// Flip a set of `prepared` or `uncertain` rows to `applied`. Called by the + /// startup reconcile step after the on-disk SHA matches each row's + /// `new_sha`. The `state IN ('prepared', 'uncertain')` guard is + /// the second barrier against re-promoting a row that was cancelled + /// by another path while the reconcile was in flight; only rows + /// that were still in one of those states at the moment the UPDATE + /// runs are flipped. + #[allow(dead_code)] // wired by the startup reconcile + pub async fn mark_pending_ref_transitions_applied_for_rows( + &self, + ids: &[String], + ) -> Result { + if ids.is_empty() { + return Ok(0); + } + let now = Utc::now().to_rfc3339(); + let res = sqlx::query( + r#"UPDATE pending_ref_transitions + SET state = $1, applied_at = $2 + WHERE id = ANY($3) AND state IN ($4, $5)"#, + ) + .bind(pending_state::APPLIED) + .bind(&now) + .bind(ids) + .bind(pending_state::PREPARED) + .bind(pending_state::UNCERTAIN) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// Delete a row by id. Called by the recovery drain AFTER the push + /// event, the cert, and the anchor job have all landed. A subsequent + /// drain pass is a no-op for the same transition because the row is + /// gone and the deterministic artifact ids collide on `ON CONFLICT`. + #[allow(dead_code)] // wired by the handler refactor in the next slice + pub async fn delete_pending_ref_transition(&self, id: &str) -> Result { + let res = sqlx::query("DELETE FROM pending_ref_transitions WHERE id = $1") + .bind(id) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// Look up the deterministic `id` of a `pending_ref_transitions` + /// row by `(request_id, ref_name)`. Returns `Ok(None)` if no + /// such row exists (e.g. a ref that the report-status excluded + /// from the durable effects). The live handler uses this to + /// target per-ref cleanup after effects land so it can delete + /// only the rows whose required writes succeeded. + #[allow(dead_code)] + pub async fn lookup_pending_ref_transition_id( + &self, + request_id: &str, + ref_name: &str, + ) -> Result> { + let row = sqlx::query( + "SELECT id FROM pending_ref_transitions + WHERE request_id = $1 AND ref_name = $2", + ) + .bind(request_id) + .bind(ref_name) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.get::("id"))) + } + + /// Delete every `applied` or `uncertain` row for a `request_id`. + /// Called by the live handler AFTER the push event, cert, and anchor + /// job writes have all succeeded. This removes the outbox row once + /// its durable effects are complete, preventing replay on restart. + #[allow(dead_code)] + pub async fn delete_pending_ref_transitions_by_request_id( + &self, + request_id: &str, + ) -> Result { + let res = sqlx::query( + r#"DELETE FROM pending_ref_transitions + WHERE request_id = $1 AND state IN ($2, $3)"#, + ) + .bind(request_id) + .bind(pending_state::APPLIED) + .bind(pending_state::UNCERTAIN) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// Return every child of `request_id` in ordinal order. The step-3 + /// effect executor calls this after loading the request row to + /// re-derive the per-ref cert and anchor writes. The accepted + /// child is the one whose `ref_name` is in the parsed report's + /// ok set; the executor re-derives that set from + /// `req.parsed_report`, so this helper returns the full ordered + /// list and lets the caller filter. + pub async fn list_pending_ref_transitions_for_request( + &self, + request_id: &str, + ) -> Result> { + let rows = sqlx::query( + r#"SELECT id, request_id, repo_id, ref_name, old_sha, new_sha, pusher_did, node_did, + signature_header, signature_input, content_digest, state, created_at, + applied_at, cancelled_at, ordinal, git_target_kind + FROM pending_ref_transitions + WHERE request_id = $1 + ORDER BY ordinal ASC, id ASC"#, + ) + .bind(request_id) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(row_to_pending_ref_transition) + .collect()) + } + + /// Flip every `prepared` row attached to `request_id` to `uncertain`. + /// Called when receive-pack returns Err but the exit was non-zero or + /// timed out, meaning some refs may have landed before the failure. + /// The reconcile step checks these rows against disk at startup. + /// + /// P2 (reviewer-2 round 4): do NOT set `cancelled_at` on an + /// `uncertain` row — `cancelled_at` is reserved for transitions + /// that were *decided* not to land. An uncertain row is, by + /// definition, undecided; leaving the column null means any + /// future consumer filtering on `cancelled_at IS NOT NULL` sees + /// only the truly-cancelled rows. + #[allow(dead_code)] + pub async fn mark_pending_ref_transitions_uncertain(&self, request_id: &str) -> Result { + let res = sqlx::query( + r#"UPDATE pending_ref_transitions + SET state = $1 + WHERE request_id = $2 AND state = $3"#, + ) + .bind(pending_state::UNCERTAIN) + .bind(request_id) + .bind(pending_state::PREPARED) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// Flip every `uncertain` row for a `request_id` to `cancelled`. + /// Called after the reconcile step has confirmed none of the refs + /// landed on disk (all rows still have state `uncertain`). + #[allow(dead_code)] + pub async fn mark_uncertain_rows_cancelled(&self, request_id: &str) -> Result { + let now = Utc::now().to_rfc3339(); + let res = sqlx::query( + r#"UPDATE pending_ref_transitions + SET state = $1, cancelled_at = $2 + WHERE request_id = $3 AND state = $4"#, + ) + .bind(pending_state::CANCELLED) + .bind(&now) + .bind(request_id) + .bind(pending_state::UNCERTAIN) + .execute(&self.pool) + .await?; + Ok(res.rows_affected()) + } + + /// Test-only: insert a row directly in the given state. Used to + /// simulate the crash window ("row is `applied` but the handler + /// never reached the push event / cert / anchor code") without + /// running the full handler. Mirrors the production insert but + /// takes the state as an argument so a test can stage a row that + /// the drain will pick up. + #[cfg(test)] + pub async fn insert_pending_ref_transition_for_test( + &self, + row: &PendingRefTransition, + ) -> Result<()> { + let applied_at = row.applied_at.clone().unwrap_or_default(); + let cancelled_at = row.cancelled_at.clone().unwrap_or_default(); + let applied_at_opt: Option<&str> = if applied_at.is_empty() { + None + } else { + Some(&applied_at) + }; + let cancelled_at_opt: Option<&str> = if cancelled_at.is_empty() { + None + } else { + Some(&cancelled_at) + }; + sqlx::query( + r#"INSERT INTO pending_ref_transitions + (id, request_id, repo_id, ref_name, old_sha, new_sha, pusher_did, node_did, + signature_header, signature_input, content_digest, state, created_at, + applied_at, cancelled_at, ordinal, git_target_kind) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)"#, + ) + .bind(&row.id) + .bind(&row.request_id) + .bind(&row.repo_id) + .bind(&row.ref_name) + .bind(&row.old_sha) + .bind(&row.new_sha) + .bind(&row.pusher_did) + .bind(&row.node_did) + .bind(&row.signature_header) + .bind(&row.signature_input) + .bind(&row.content_digest) + .bind(&row.state) + .bind(&row.created_at) + .bind(applied_at_opt) + .bind(cancelled_at_opt) + .bind(row.ordinal) + .bind(row.git_target_kind.as_deref()) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Idempotent push event insert. Returns `true` if a NEW row was + /// created, `false` if the deterministic id collided with an + /// existing row (recovery re-fired the same transition). + #[allow(dead_code)] // wired by the handler refactor in the next slice + pub async fn record_push_with_id( + &self, + id: &str, + agent_did: &str, + repo_id: &str, + commit_hash: &str, + object_count: i64, + ) -> Result { + let res = sqlx::query( + r#"INSERT INTO push_events (id, agent_did, repo_id, commit_hash, object_count, pushed_at) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (id) DO NOTHING"#, + ) + .bind(id) + .bind(agent_did) + .bind(repo_id) + .bind(commit_hash) + .bind(object_count) + .bind(Utc::now().to_rfc3339()) + .execute(&self.pool) + .await?; + Ok(res.rows_affected() == 1) + } + + /// Idempotent ref certificate insert. Returns `Some` if a NEW cert + /// was created, `None` if the unique `(repo_id, ref_name)` index + /// already had a row (the live path got there first, or a previous + /// recovery pass did). + /// + /// The primary key is the deterministic `id`; the unique index on + /// `(repo_id, ref_name)` is what makes the recovery exactly-once, + /// because a second insert for the same `(repo_id, ref_name)` + /// returns `None` rather than overwriting the existing cert. + #[allow(dead_code)] // wired by the handler refactor in the next slice + pub async fn insert_ref_certificate_idempotent( + &self, + cert: &RefCertificate, + ) -> Result> { + let res = sqlx::query( + r#"INSERT INTO ref_certificates + (id, repo_id, ref_name, old_sha, new_sha, pusher_did, node_did, signature, issued_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (repo_id, ref_name) DO NOTHING + RETURNING id, repo_id, ref_name, old_sha, new_sha, pusher_did, node_did, signature, issued_at"#, + ) + .bind(&cert.id) + .bind(&cert.repo_id) + .bind(&cert.ref_name) + .bind(&cert.old_sha) + .bind(&cert.new_sha) + .bind(&cert.pusher_did) + .bind(&cert.node_did) + .bind(&cert.signature) + .bind(&cert.issued_at) + .fetch_optional(&self.pool) + .await?; + Ok(res.map(row_to_cert)) + } + + /// Idempotent anchor job insert. Returns `true` if a NEW row was + /// created, `false` if the `(repo_id, ref_name, old_sha, new_sha)` + /// unique index already had a row. PR 2's transport will read these + /// rows; the recovery drain writes them with `ON CONFLICT DO NOTHING` + /// so re-running the drain cannot create a second upload request. + #[allow(dead_code)] // wired by the handler refactor in the next slice + pub async fn insert_anchor_job_idempotent(&self, job: &AnchorJob) -> Result { + let res = sqlx::query( + r#"INSERT INTO anchor_jobs + (id, repo_id, ref_name, old_sha, new_sha, pusher_did, created_at, claimed_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (id) DO NOTHING"#, + ) + .bind(&job.id) + .bind(&job.repo_id) + .bind(&job.ref_name) + .bind(&job.old_sha) + .bind(&job.new_sha) + .bind(&job.pusher_did) + .bind(&job.created_at) + .bind(job.claimed_at.as_deref()) + .execute(&self.pool) + .await?; + Ok(res.rows_affected() == 1) + } + + /// Count anchor jobs for a transition, used by the test to assert + /// "at most one anchor upload" without depending on PR 2's transport. + #[allow(dead_code)] // wired by the handler refactor in the next slice + pub async fn count_anchor_jobs( + &self, + repo_id: &str, + ref_name: &str, + old_sha: &str, + new_sha: &str, + ) -> Result { + let row = sqlx::query( + "SELECT COUNT(*) AS cnt FROM anchor_jobs + WHERE repo_id = $1 AND ref_name = $2 AND old_sha = $3 AND new_sha = $4", + ) + .bind(repo_id) + .bind(ref_name) + .bind(old_sha) + .bind(new_sha) + .fetch_one(&self.pool) + .await?; + Ok(row.get::("cnt")) + } + + /// Count push events for a transition, used by the test to assert + /// "exactly one push event" after recovery. + #[allow(dead_code)] // wired by the handler refactor in the next slice + pub async fn count_push_events( + &self, + repo_id: &str, + commit_hash: &str, + agent_did: &str, + ) -> Result { + let row = sqlx::query( + "SELECT COUNT(*) AS cnt FROM push_events + WHERE repo_id = $1 AND commit_hash = $2 AND agent_did = $3", + ) + .bind(repo_id) + .bind(commit_hash) + .bind(agent_did) + .fetch_one(&self.pool) + .await?; + Ok(row.get::("cnt")) + } + pub async fn list_ref_certificates( &self, repo_id: &str, @@ -3930,21 +5545,64 @@ fn row_to_webhook(r: sqlx::postgres::PgRow) -> Webhook { events, created_by_did: r.get("created_by_did"), created_at: r.get("created_at"), - active: r.get::("active"), + active: r.get::("active"), + } +} + +fn row_to_cert(r: sqlx::postgres::PgRow) -> RefCertificate { + RefCertificate { + id: r.get("id"), + repo_id: r.get("repo_id"), + ref_name: r.get("ref_name"), + old_sha: r.get("old_sha"), + new_sha: r.get("new_sha"), + pusher_did: r.get("pusher_did"), + node_did: r.get("node_did"), + signature: r.get("signature"), + issued_at: r.get("issued_at"), + } +} + +#[allow(dead_code)] // wired by the handler refactor in the next slice +fn row_to_receive_pack_request(r: sqlx::postgres::PgRow) -> ReceivePackRequest { + ReceivePackRequest { + id: r.get("id"), + repo_id: r.get("repo_id"), + pusher_did: r.get("pusher_did"), + node_did: r.get("node_did"), + request_bytes: r.get("request_bytes"), + request_bytes_hash: r.get("request_bytes_hash"), + state: r.get("state"), + git_exit_ok: r.get("git_exit_ok"), + parsed_report: r.get("parsed_report"), + accepted_ordinal: r.get("accepted_ordinal"), + attempt_count: r.get("attempt_count"), + last_error: r.get("last_error"), + next_attempt_at: r.get("next_attempt_at"), + created_at: r.get("created_at"), + completed_at: r.get("completed_at"), } } -fn row_to_cert(r: sqlx::postgres::PgRow) -> RefCertificate { - RefCertificate { +fn row_to_pending_ref_transition(r: sqlx::postgres::PgRow) -> PendingRefTransition { + PendingRefTransition { id: r.get("id"), + request_id: r.get("request_id"), repo_id: r.get("repo_id"), ref_name: r.get("ref_name"), old_sha: r.get("old_sha"), new_sha: r.get("new_sha"), pusher_did: r.get("pusher_did"), node_did: r.get("node_did"), - signature: r.get("signature"), - issued_at: r.get("issued_at"), + signature_header: r.get("signature_header"), + signature_input: r.get("signature_input"), + content_digest: r.get("content_digest"), + state: r.get("state"), + created_at: r.get("created_at"), + applied_at: r.get("applied_at"), + cancelled_at: r.get("cancelled_at"), + ordinal: r.get("ordinal"), + git_target_kind: r.get("git_target_kind"), } } @@ -4818,14 +6476,14 @@ mod migration_tests { // then drop the owner_did column to simulate a pre-v10 schema. db.migrate().await.unwrap(); sqlx::query("ALTER TABLE received_ref_updates DROP COLUMN owner_did") - .execute(&db.pool) + .execute(db.pool()) .await .unwrap(); // Truncate schema_migrations and re-seed at v9 — simulate an existing // node that has run v1..v9 but not yet v10. sqlx::query("DELETE FROM schema_migrations") - .execute(&db.pool) + .execute(db.pool()) .await .unwrap(); for m in MIGRATIONS.iter().take_while(|m| m.version < 10) { @@ -4836,7 +6494,7 @@ mod migration_tests { .bind(m.version) .bind(m.name) .bind("2026-07-01T00:00:00Z") - .execute(&db.pool) + .execute(db.pool()) .await .unwrap(); } @@ -4861,12 +6519,12 @@ mod migration_tests { .bind::>(None) .bind("2026-07-01T12:00:01Z") .bind("12D3KooWPeer") - .execute(&db.pool) + .execute(db.pool()) .await .unwrap(); assert_eq!( sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM received_ref_updates") - .fetch_one(&db.pool) + .fetch_one(db.pool()) .await .unwrap(), 1, @@ -4882,7 +6540,7 @@ mod migration_tests { let owner: Option = sqlx::query_scalar("SELECT owner_did FROM received_ref_updates WHERE id = $1") .bind(&row_id) - .fetch_one(&db.pool) + .fetch_one(db.pool()) .await .unwrap(); assert_eq!(owner, None, "existing row's owner_did must be NULL"); @@ -4893,7 +6551,7 @@ mod migration_tests { FROM information_schema.columns WHERE table_name = 'received_ref_updates' AND column_name = 'owner_did'", ) - .fetch_one(&db.pool) + .fetch_one(db.pool()) .await .unwrap(); assert_eq!(col.0, "owner_did"); @@ -4903,7 +6561,7 @@ mod migration_tests { // (c) Version 11 is recorded as applied. let v11_count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM schema_migrations WHERE version = 11") - .fetch_one(&db.pool) + .fetch_one(db.pool()) .await .unwrap(); assert_eq!( @@ -4932,7 +6590,7 @@ mod migration_tests { async fn attempted_at_of(db: &super::Db, repo: &str) -> Option { sqlx::query_scalar("SELECT attempted_at FROM sync_queue WHERE repo = $1") .bind(repo) - .fetch_one(&db.pool) + .fetch_one(db.pool()) .await .unwrap() } @@ -4952,11 +6610,11 @@ mod migration_tests { // Roll back to v11: drop the column and forget the version. sqlx::query("ALTER TABLE sync_queue DROP COLUMN attempted_at") - .execute(&db.pool) + .execute(db.pool()) .await .unwrap(); sqlx::query("DELETE FROM schema_migrations WHERE version = 17") - .execute(&db.pool) + .execute(db.pool()) .await .unwrap(); @@ -4970,7 +6628,7 @@ mod migration_tests { FROM information_schema.columns WHERE table_name = 'sync_queue' AND column_name = 'attempted_at'", ) - .fetch_one(&db.pool) + .fetch_one(db.pool()) .await .unwrap(); assert_eq!(col.0, "text"); @@ -4978,7 +6636,7 @@ mod migration_tests { let recorded: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM schema_migrations WHERE version = 17") - .fetch_one(&db.pool) + .fetch_one(db.pool()) .await .unwrap(); assert_eq!(recorded.0, 1, "v17 must be recorded as applied"); @@ -5024,13 +6682,13 @@ mod migration_tests { sqlx::query("UPDATE sync_queue SET enqueued_at = $1 WHERE repo = $2") .bind("2026-07-29T00:00:00Z") .bind("z6Mkfoo/older") - .execute(&db.pool) + .execute(db.pool()) .await .unwrap(); sqlx::query("UPDATE sync_queue SET enqueued_at = $1 WHERE repo = $2") .bind("2026-07-29T00:00:01Z") .bind("z6Mkfoo/newer") - .execute(&db.pool) + .execute(db.pool()) .await .unwrap(); @@ -5055,7 +6713,7 @@ mod migration_tests { enqueue_one(&db, "z6Mkfoo/a").await; let before: String = sqlx::query_scalar("SELECT enqueued_at FROM sync_queue WHERE repo = 'z6Mkfoo/a'") - .fetch_one(&db.pool) + .fetch_one(db.pool()) .await .unwrap(); @@ -5063,7 +6721,7 @@ mod migration_tests { let after: String = sqlx::query_scalar("SELECT enqueued_at FROM sync_queue WHERE repo = 'z6Mkfoo/a'") - .fetch_one(&db.pool) + .fetch_one(db.pool()) .await .unwrap(); assert_eq!(before, after); @@ -6714,6 +8372,114 @@ mod ref_certificate_tests { ); } + /// P1-B: the live handler routes cert issuance through + /// `cert::issue_ref_certificate` (the upsert, NOT + /// `insert_ref_certificate_idempotent`'s DO NOTHING). This test + /// exercises the full `cert::issue_ref_certificate` call path + /// end-to-end through the `AppState`, asserting that: + /// + /// - a re-push to the same `(repo_id, ref_name)` updates + /// `old_sha` / `new_sha` / `pusher_did` / `issued_at` / + /// `signature` to the new transition's values, + /// - the deterministic `cert_id` (derived from + /// `ref_cert_id_for(request_id, ordinal)`) is preserved + /// across the re-push, and + /// - exactly one cert row exists for the ref after the + /// re-push. + /// + /// This pins the live-handler contract that the previous + /// `issue_ref_certificate_idempotent` call violated. The DB-level + /// `insert_ref_certificate_upserts_on_repo_ref` test pins the + /// underlying upsert SQL; this test pins the live-handler wrapper. + #[sqlx::test] + async fn issue_ref_certificate_upserts_on_repo_ref_via_live_path(pool: PgPool) { + use crate::cert; + use crate::db::ref_cert_id_for; + + let state = crate::test_support::test_state(pool.clone()).await; + let repo_id = uuid::Uuid::new_v4().to_string(); + state + .db + .create_repo(&RepoRecord { + id: repo_id.clone(), + name: "cert-upsert-live".into(), + owner_did: "did:key:zOWNER".into(), + description: None, + is_public: true, + default_branch: "main".into(), + created_at: Utc::now(), + updated_at: Utc::now(), + disk_path: "/tmp/cert-upsert-live".into(), + forked_from: None, + machine_id: None, + }) + .await + .unwrap(); + + // First push: 0000 -> 1111, pusher A. + let c1 = cert::issue_ref_certificate( + &state, + &repo_id, + "refs/heads/main", + "0000", + "1111", + "did:key:zFirstPusher", + &ref_cert_id_for("req-A", 0), + ) + .await + .unwrap(); + + // Sleep 1ms so the second push's `issued_at` is strictly + // greater than the first. `build_ref_certificate` stamps + // `issued_at = Utc::now()`, and the upsert's per-column + // guard `EXCLUDED.issued_at > ref_certificates.issued_at` + // only updates on strictly-newer timestamps. + tokio::time::sleep(std::time::Duration::from_millis(2)).await; + + // Second push: aaaa -> bbbb, pusher B, SAME deterministic + // cert id (same `request_id` and `ref_name`). + let c2 = cert::issue_ref_certificate( + &state, + &repo_id, + "refs/heads/main", + "aaaa", + "bbbb", + "did:key:zSecondPusher", + &ref_cert_id_for("req-A", 0), + ) + .await + .unwrap(); + + // The deterministic id is preserved across the re-push. + assert_eq!(c1.id, c2.id, "cert id is preserved across re-push"); + assert_eq!( + c1.id, + ref_cert_id_for("req-A", 0), + "cert id is the deterministic (request_id, ordinal) hash" + ); + + // The upsert updated every other field to the second push. + assert_eq!(c1.new_sha, "1111", "first push's new_sha"); + assert_eq!(c2.new_sha, "bbbb", "re-push updates new_sha"); + assert_eq!(c1.pusher_did, "did:key:zFirstPusher"); + assert_eq!(c2.pusher_did, "did:key:zSecondPusher"); + assert_ne!( + c1.issued_at, c2.issued_at, + "issued_at advances on a re-push" + ); + assert_ne!(c1.signature, c2.signature, "signature is re-signed"); + + // Exactly one row in the table for the ref. + let certs = state.db.list_ref_certificates(&repo_id, 10).await.unwrap(); + assert_eq!(certs.len(), 1, "exactly one cert row per ref"); + assert_eq!(certs[0].id, c1.id, "the original id survives"); + assert_eq!(certs[0].new_sha, "bbbb", "row reflects the latest push"); + assert_eq!( + certs[0].pusher_did, "did:key:zSecondPusher", + "row reflects the latest pusher" + ); + } + #[sqlx::test] async fn list_ref_certificates_clamps_negative_limit(pool: PgPool) { let db = db(pool).await; @@ -7922,7 +9688,7 @@ mod peer_authority_tests { .bind(legacy) .bind(HONEST_URL) .bind(chrono::Utc::now().to_rfc3339()) - .execute(&db.pool) + .execute(db.pool()) .await .expect("seeding a pre-gate row must succeed"); @@ -8552,3 +10318,668 @@ mod cid_candidate_order_tests { ); } } + +#[cfg(test)] +mod pending_ref_transition_tests { + //! #26 Split PR 1 — durable post-receive outbox at the DB layer. + //! + //! These tests exercise the producer / persistence / drain contracts + //! directly. The handler-level test (failure injection between + //! receive_pack and the bookkeeping) is a follow-up that lands with + //! the handler refactor in the next slice. Every test here uses + //! `Db::for_testing` + `run_migrations` to provision a clean schema, + //! so they are independent of any other test's seed state. + //! + //! Each test names the invariant it pins. Reverting the production + //! line under test turns the named assertion red. + + use super::{ + anchor_job_id_for, deterministic_id, pending_state, push_event_id_for, ref_cert_id_for, + AnchorJob, Db, PendingRefTransition, RepoRecord, + }; + use crate::api::repos::RefUpdate; + use chrono::Utc; + use sqlx::PgPool; + + async fn db(pool: PgPool) -> Db { + let db = Db::for_testing(pool); + db.run_migrations().await.unwrap(); + db + } + + fn ref_update(name: &str, old: &str, new: &str) -> RefUpdate { + RefUpdate { + ref_name: name.to_string(), + old_sha: old.to_string(), + new_sha: new.to_string(), + } + } + + /// The producer contract: every ref update in a push gets a `prepared` + /// row carrying the verified pusher, the signature header, and the + /// request id. `mark_applied` flips exactly those rows. + #[sqlx::test] + async fn insert_then_mark_applied_flips_state_for_every_ref(pool: PgPool) { + let db = db(pool).await; + let updates = vec![ + ref_update( + "refs/heads/main", + "a".repeat(40).as_str(), + "b".repeat(40).as_str(), + ), + ref_update( + "refs/heads/feature", + "c".repeat(40).as_str(), + "d".repeat(40).as_str(), + ), + ]; + let rows = db + .insert_pending_ref_transitions( + "req-1", + "repo-1", + "did:key:node", + "did:key:pusher", + &updates, + "Signature: sig=...", + "Signature-Input: ...", + "Content-Digest: ...", + ) + .await + .unwrap(); + assert_eq!(rows.len(), 2, "one row per ref update"); + assert!(rows.iter().all(|r| r.state == pending_state::PREPARED)); + + let flipped = db + .mark_pending_ref_transitions_applied("req-1") + .await + .unwrap(); + assert_eq!(flipped, 2, "every prepared row for the request flips"); + + let drained = db.list_pending_ref_transitions_applied(100).await.unwrap(); + assert_eq!(drained.len(), 2); + assert!(drained.iter().all(|r| r.state == pending_state::APPLIED)); + assert!(drained.iter().all(|r| r.pusher_did == "did:key:pusher")); + assert!( + drained + .iter() + .all(|r| r.signature_header == "Signature: sig=..."), + "the original signature header must survive the round trip — \ + recovery re-derives the cert and the anchor under the original identity" + ); + } + + /// A second `mark_applied` for the same request is a no-op — the row is + /// already in `applied` and the state predicate prevents re-flipping. + /// This is what makes a recovery re-pass safe. + #[sqlx::test] + async fn mark_applied_is_idempotent_on_repeat(pool: PgPool) { + let db = db(pool).await; + db.insert_pending_ref_transitions( + "req-2", + "repo-1", + "did:key:node", + "did:key:pusher", + &[ref_update( + "refs/heads/main", + "a".repeat(40).as_str(), + "b".repeat(40).as_str(), + )], + "Signature: sig=...", + "Signature-Input: ...", + "Content-Digest: ...", + ) + .await + .unwrap(); + + assert_eq!( + db.mark_pending_ref_transitions_applied("req-2") + .await + .unwrap(), + 1, + "first call flips the one row" + ); + assert_eq!( + db.mark_pending_ref_transitions_applied("req-2") + .await + .unwrap(), + 0, + "second call flips nothing — the row is already applied" + ); + } + + /// The reviewer's second proof: a `cancelled` row is never drained. + /// The drain's WHERE clause is on `state = 'applied'`, so a row that + /// never made it past receive_pack CANNOT become a push event, a + /// certificate, or an anchor handoff. + #[sqlx::test] + async fn cancelled_rows_are_not_returned_by_the_drain(pool: PgPool) { + let db = db(pool).await; + db.insert_pending_ref_transitions( + "req-3", + "repo-1", + "did:key:node", + "did:key:pusher", + &[ref_update( + "refs/heads/main", + "a".repeat(40).as_str(), + "b".repeat(40).as_str(), + )], + "Signature: sig=...", + "Signature-Input: ...", + "Content-Digest: ...", + ) + .await + .unwrap(); + db.mark_pending_ref_transitions_cancelled("req-3") + .await + .unwrap(); + + let drained = db.list_pending_ref_transitions_applied(100).await.unwrap(); + assert!( + drained.is_empty(), + "a cancelled receive-pack must never reach the drain — the row's \ + state is `cancelled`, not `applied`, and the drain is keyed on `applied`" + ); + } + + /// Same proof, but for the pre-flip state. A `prepared` row (handler + /// crashed between `insert_prepared` and `mark_applied` / never + /// reached either post-receive branch) is also never drained. The + /// recovery cannot promote a `prepared` row by itself — only the + /// handler's post-Ok code does, by calling `mark_applied`. + #[sqlx::test] + async fn prepared_rows_are_not_returned_by_the_drain(pool: PgPool) { + let db = db(pool).await; + db.insert_pending_ref_transitions( + "req-4", + "repo-1", + "did:key:node", + "did:key:pusher", + &[ref_update( + "refs/heads/main", + "a".repeat(40).as_str(), + "b".repeat(40).as_str(), + )], + "Signature: sig=...", + "Signature-Input: ...", + "Content-Digest: ...", + ) + .await + .unwrap(); + // No mark_applied / mark_cancelled call. The row stays `prepared`. + + let drained = db.list_pending_ref_transitions_applied(100).await.unwrap(); + assert!( + drained.is_empty(), + "a row the handler never reached the post-Ok branch for must not \ + be drained; only `mark_applied` flips a row, only the drain \ + picks up `applied` rows" + ); + } + + /// The reviewer's first proof (DB layer): a recovery re-pass on the + /// same `applied` row produces the same push event id, the same cert + /// id, and the same anchor job id, and the idempotent inserts all + /// collapse to no-ops. The drain deletes the row after the work + /// lands, so a third pass has nothing to do. + #[sqlx::test] + async fn drain_then_re_derive_is_idempotent(pool: PgPool) { + let db = db(pool).await; + let now = Utc::now().to_rfc3339(); + let row = PendingRefTransition { + id: super::deterministic_id(&[ + "pending_ref_transition", + "req-5", + "repo-1", + "refs/heads/main", + &"a".repeat(40), + &"b".repeat(40), + ]), + request_id: "req-5".to_string(), + repo_id: "repo-1".to_string(), + ref_name: "refs/heads/main".to_string(), + old_sha: "a".repeat(40), + new_sha: "b".repeat(40), + pusher_did: "did:key:pusher".to_string(), + node_did: "did:key:node".to_string(), + signature_header: "Signature: sig=...".to_string(), + signature_input: "Signature-Input: ...".to_string(), + content_digest: "Content-Digest: ...".to_string(), + state: pending_state::APPLIED.to_string(), + created_at: now.clone(), + applied_at: Some(now.clone()), + cancelled_at: None, + // Single-ref test fixture; the request's only child is + // ordinal 0. Multi-ref tests set the ordinal explicitly + // for each child row. + ordinal: 0, + git_target_kind: Some("update".to_string()), + }; + db.insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + + // First drain: picks up the row. Caller would now re-derive the + // artifacts; the row is then deleted. + let first = db.list_pending_ref_transitions_applied(100).await.unwrap(); + assert_eq!(first.len(), 1); + let push_id_1 = push_event_id_for(&row.request_id, row.ordinal); + let cert_id_1 = ref_cert_id_for(&row.request_id, row.ordinal); + let anchor_id_1 = + anchor_job_id_for(&row.repo_id, &row.ref_name, &row.old_sha, &row.new_sha); + + // Second drain: row is still there (we did not delete). Re-derive + // the same ids; the inserts collapse. + let second = db.list_pending_ref_transitions_applied(100).await.unwrap(); + assert_eq!(second.len(), 1, "the row is still in `applied`"); + let push_id_2 = push_event_id_for(&row.request_id, row.ordinal); + let cert_id_2 = ref_cert_id_for(&row.request_id, row.ordinal); + let anchor_id_2 = + anchor_job_id_for(&row.repo_id, &row.ref_name, &row.old_sha, &row.new_sha); + assert_eq!(push_id_1, push_id_2, "push id is deterministic"); + assert_eq!(cert_id_1, cert_id_2, "cert id is deterministic"); + assert_eq!(anchor_id_1, anchor_id_2, "anchor id is deterministic"); + + // Now exercise the idempotent inserts directly: a second + // `record_push_with_id` returns false, the cert insert returns + // None on the (repo_id, ref_name) unique, and the anchor insert + // returns false on the (repo_id, ref_name, old_sha, new_sha) + // unique. + assert!( + db.record_push_with_id(&push_id_1, &row.pusher_did, &row.repo_id, &row.new_sha, 0) + .await + .unwrap(), + "first push insert is created" + ); + assert!( + !db.record_push_with_id(&push_id_2, &row.pusher_did, &row.repo_id, &row.new_sha, 0) + .await + .unwrap(), + "second push insert with the same id collapses to a no-op" + ); + + // Anchor: one row, never two. + let job = AnchorJob { + id: anchor_id_1.clone(), + repo_id: row.repo_id.clone(), + ref_name: row.ref_name.clone(), + old_sha: row.old_sha.clone(), + new_sha: row.new_sha.clone(), + pusher_did: row.pusher_did.clone(), + created_at: now.clone(), + claimed_at: None, + }; + assert!(db.insert_anchor_job_idempotent(&job).await.unwrap()); + assert!( + !db.insert_anchor_job_idempotent(&job).await.unwrap(), + "a second anchor insert with the same id is a no-op" + ); + assert_eq!( + db.count_anchor_jobs(&row.repo_id, &row.ref_name, &row.old_sha, &row.new_sha) + .await + .unwrap(), + 1, + "exactly one anchor job per transition, no matter how many recovery passes" + ); + assert_eq!( + db.count_push_events(&row.repo_id, &row.new_sha, &row.pusher_did) + .await + .unwrap(), + 1, + "exactly one push event per (repo, commit, pusher)" + ); + + // After the work lands, the drain deletes the row. A third pass + // sees nothing. + db.delete_pending_ref_transition(&row.id).await.unwrap(); + let third = db.list_pending_ref_transitions_applied(100).await.unwrap(); + assert!(third.is_empty(), "the row is gone after recovery"); + } + + /// `mark_cancelled` is also idempotent. The state predicate is + /// `state = 'prepared'`, so a second call flips nothing. + #[sqlx::test] + async fn mark_cancelled_is_idempotent_on_repeat(pool: PgPool) { + let db = db(pool).await; + db.insert_pending_ref_transitions( + "req-6", + "repo-1", + "did:key:node", + "did:key:pusher", + &[ref_update( + "refs/heads/main", + "a".repeat(40).as_str(), + "b".repeat(40).as_str(), + )], + "Signature: sig=...", + "Signature-Input: ...", + "Content-Digest: ...", + ) + .await + .unwrap(); + assert_eq!( + db.mark_pending_ref_transitions_cancelled("req-6") + .await + .unwrap(), + 1 + ); + assert_eq!( + db.mark_pending_ref_transitions_cancelled("req-6") + .await + .unwrap(), + 0 + ); + } + + // ----- P1 round 4: per-ref variant tests ----- + // + // The new per-ref helpers are the foundation of the + // ref-by-ref outcome model. A mixed push where one ref was + // rejected and one was accepted must: + // 1. flip ONLY the accepted ref to `applied` + // 2. flip ONLY the rejected ref to `cancelled` + // 3. leave any ref the report did not mention as `prepared` + // The bulk helpers were the bug that issued certs for the + // rejected ref; the per-ref helpers are the fix. + + #[sqlx::test] + async fn per_ref_applied_only_flips_named_refs(pool: PgPool) { + let db = db(pool).await; + db.insert_pending_ref_transitions( + "req-per-ref-1", + "repo-1", + "did:key:node", + "did:key:pusher", + &[ + ref_update("refs/heads/main", &"a".repeat(40), &"b".repeat(40)), + ref_update("refs/heads/feature", &"c".repeat(40), &"d".repeat(40)), + ref_update("refs/tags/v1", &"e".repeat(40), &"f".repeat(40)), + ], + "Signature: sig=...", + "Signature-Input: ...", + "Content-Digest: ...", + ) + .await + .unwrap(); + + // Flip only `main` and `feature` (the OK refs); the + // tag stays `prepared` for the next call to handle. + let n = db + .mark_pending_ref_transitions_applied_for_names( + "req-per-ref-1", + &["refs/heads/main", "refs/heads/feature"], + ) + .await + .unwrap(); + assert_eq!(n, 2, "exactly the two named rows flip"); + + // The tag row is still `prepared`. + let applied = db.list_pending_ref_transitions_applied(100).await.unwrap(); + assert_eq!(applied.len(), 2, "two rows in applied"); + let names: std::collections::HashSet<&str> = + applied.iter().map(|r| r.ref_name.as_str()).collect(); + assert!(names.contains("refs/heads/main")); + assert!(names.contains("refs/heads/feature")); + assert!(!names.contains("refs/tags/v1")); + } + + #[sqlx::test] + async fn per_ref_cancelled_only_flips_named_refs(pool: PgPool) { + let db = db(pool).await; + db.insert_pending_ref_transitions( + "req-per-ref-2", + "repo-1", + "did:key:node", + "did:key:pusher", + &[ + ref_update("refs/heads/main", &"a".repeat(40), &"b".repeat(40)), + ref_update("refs/heads/feature", &"c".repeat(40), &"d".repeat(40)), + ], + "Signature: sig=...", + "Signature-Input: ...", + "Content-Digest: ...", + ) + .await + .unwrap(); + // The report rejected only `main`. + let n = db + .mark_pending_ref_transitions_cancelled_for_names("req-per-ref-2", &["refs/heads/main"]) + .await + .unwrap(); + assert_eq!(n, 1, "only the rejected ref flips"); + let still_prepared = db.list_pending_ref_transitions_prepared(100).await.unwrap(); + assert_eq!(still_prepared.len(), 1); + assert_eq!(still_prepared[0].ref_name, "refs/heads/feature"); + } + + #[sqlx::test] + async fn per_ref_uncertain_does_not_set_cancelled_at(pool: PgPool) { + // P2 (reviewer-2 round 4): `mark_uncertain` must NOT set + // `cancelled_at`. An `uncertain` row is undecided and + // should leave the column null so any future consumer + // filtering on `cancelled_at IS NOT NULL` only sees + // truly-cancelled rows. + let db = db(pool).await; + db.insert_pending_ref_transitions( + "req-uncertain-test", + "repo-1", + "did:key:node", + "did:key:pusher", + &[ref_update( + "refs/heads/main", + &"a".repeat(40), + &"b".repeat(40), + )], + "Signature: sig=...", + "Signature-Input: ...", + "Content-Digest: ...", + ) + .await + .unwrap(); + let n = db + .mark_pending_ref_transitions_uncertain("req-uncertain-test") + .await + .unwrap(); + assert_eq!(n, 1); + let rows = db + .list_pending_ref_transitions_prepared_or_uncertain(10) + .await + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].state, pending_state::UNCERTAIN); + assert!( + rows[0].cancelled_at.is_none(), + "uncertain row must leave cancelled_at null" + ); + } + + #[sqlx::test] + async fn lookup_pending_ref_transition_id_returns_named_ref(pool: PgPool) { + // P1 (reviewer-1 round 4): the per-ref cleanup loop needs + // to map (request_id, ref_name) → row_id. Verify the + // lookup returns the correct id for the ref it was + // inserted with, and `None` for an absent one. + let db = db(pool).await; + db.insert_pending_ref_transitions( + "req-lookup", + "repo-1", + "did:key:node", + "did:key:pusher", + &[ + ref_update("refs/heads/main", &"a".repeat(40), &"b".repeat(40)), + ref_update("refs/heads/feature", &"c".repeat(40), &"d".repeat(40)), + ], + "Signature: sig=...", + "Signature-Input: ...", + "Content-Digest: ...", + ) + .await + .unwrap(); + let main_id = db + .lookup_pending_ref_transition_id("req-lookup", "refs/heads/main") + .await + .unwrap(); + assert!(main_id.is_some(), "main row id is present"); + let absent = db + .lookup_pending_ref_transition_id("req-lookup", "refs/heads/never") + .await + .unwrap(); + assert!(absent.is_none(), "absent ref returns None"); + } + + #[sqlx::test] + async fn count_pending_ref_transitions_applied_reports_zero_after_drain(pool: PgPool) { + // P3 (reviewer-2 round 4): the residual-backlog warning + // key on REMAINING, not on EXAMINED. A backlog of exactly + // `per_pass_limit * (max_passes + 1)` rows that fully + // drains must report `remaining == 0` so the warning does + // not fire on a clean drain. + let db = db(pool).await; + assert_eq!(db.count_pending_ref_transitions_applied().await.unwrap(), 0); + } + + /// P2 (reviewer-2 round 2): the multi-row `insert_pending_ref_transitions` + /// must be atomic. A mid-loop failure (here simulated by pre-seeding a + /// row whose PK collides with the second ref's deterministic id) must + /// roll the first row back; otherwise the handler can return 503 after + /// some `prepared` rows are already on disk, leaving the request in + /// an inconsistent state for the startup reconcile to clean up. + #[sqlx::test] + async fn insert_pending_ref_transitions_rolls_back_on_mid_loop_failure(pool: sqlx::PgPool) { + let db = db(pool).await; + // Seed a repo so the FK (if any) is satisfied. + db.create_repo(&RepoRecord { + id: "repo-atomic".to_string(), + name: "atomic".to_string(), + owner_did: "did:key:zAtomic".to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: Utc::now(), + updated_at: Utc::now(), + disk_path: "/tmp/atomic".to_string(), + forked_from: None, + machine_id: None, + }) + .await + .unwrap(); + + // Pre-seed a row that collides with the SECOND ref update's + // deterministic id, so the loop's second INSERT fails on PK. + let second_ref = "refs/heads/feature-a"; + let second_old = "2".repeat(40); + let second_new = "3".repeat(40); + let collision_id = deterministic_id(&[ + "pending_ref_transition", + "req-atomic", + "repo-atomic", + second_ref, + &second_old, + &second_new, + ]); + // Direct insert bypassing the helper to land a `prepared` row + // with the colliding id. + sqlx::query( + r#"INSERT INTO pending_ref_transitions + (id, request_id, repo_id, ref_name, old_sha, new_sha, pusher_did, node_did, + signature_header, signature_input, content_digest, state, created_at, + ordinal, git_target_kind) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)"#, + ) + .bind(&collision_id) + .bind("req-pre-seed") + .bind("repo-atomic") + .bind(second_ref) + .bind(&second_old) + .bind(&second_new) + .bind("did:key:zPre") + .bind("did:key:zNode") + .bind("sig-pre") + .bind("sig-input-pre") + .bind("digest-pre") + .bind(pending_state::PREPARED) + .bind(Utc::now().to_rfc3339()) + .bind(1_i32) // second child of the seeded request + .bind(Option::::None) + .execute(db.pool()) + .await + .unwrap(); + + // Now call the production helper. The first ref (main) inserts + // fine; the second ref collides and the loop returns Err. + let res = db + .insert_pending_ref_transitions( + "req-atomic", + "repo-atomic", + "did:key:zNode", + "did:key:zPusher", + &[ + ref_update("refs/heads/main", &"1".repeat(40), &"2".repeat(40)), + ref_update(second_ref, &second_old, &second_new), + ], + "sig", + "sig-input", + "digest", + ) + .await; + assert!( + res.is_err(), + "the colliding insert must return Err (pre-condition for the rollback check)" + ); + + // The atomicity half: NO `req-atomic` row may exist. Without + // the transaction the first row would have been persisted + // before the second failed, and the startup reconcile would + // later see a stranded `prepared` row pointing at a push + // that never ran. + let stranded = sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM pending_ref_transitions WHERE request_id = $1", + ) + .bind("req-atomic") + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!( + stranded, 0, + "the transaction must roll back the first row when the second fails" + ); + // The pre-seeded row is unaffected. + let pres = sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM pending_ref_transitions WHERE id = $1", + ) + .bind(&collision_id) + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!(pres, 1, "the pre-seeded row is untouched"); + } + + /// The `deterministic_id` helper uses an ASCII Unit Separator between + /// fields so that two distinct tuples can never collide by accidental + /// prefix overlap. `(a, bc)` and `(ab, c)` would otherwise hash the + /// same input. A regression on the separator shows up here. + #[test] + fn deterministic_id_avoids_prefix_overlap_collisions() { + let a = super::deterministic_id(&["a", "bc"]); + let b = super::deterministic_id(&["ab", "c"]); + assert_ne!(a, b, "the field separator must distinguish ab+bc from a+bc"); + } + + /// The push event id is stable across calls. The recovery drain + /// derives it the same way twice and gets the same value, which is + /// the entire reason for using a hash instead of a UUID. + #[test] + fn push_event_id_for_is_stable() { + assert_eq!(push_event_id_for("req-x", 0), push_event_id_for("req-x", 0)); + assert_ne!( + push_event_id_for("req-x", 0), + push_event_id_for("req-y", 0), + "different request ids produce different push event ids" + ); + assert_ne!( + push_event_id_for("req-x", 0), + push_event_id_for("req-x", 1), + "different ordinals produce different push event ids" + ); + } +} diff --git a/crates/gitlawb-node/src/durable_outbox.rs b/crates/gitlawb-node/src/durable_outbox.rs new file mode 100644 index 000000000..af3905321 --- /dev/null +++ b/crates/gitlawb-node/src/durable_outbox.rs @@ -0,0 +1,3675 @@ +//! #26 Split PR 1 — durable post-receive outbox: the recovery drain. +//! +//! This module owns the STARTUP drain for `pending_ref_transitions`. It +//! iterates every row in state `applied`, re-derives the push event, the +//! per-ref certificate, and the anchor handoff using the ORIGINAL pusher +//! DID and signature header that was persisted BEFORE the receive-pack +//! call landed the ref, and then deletes the row. +//! +//! The drain is invoked once at startup, after migrations and before +//! serving, in [`crate::main`]. It is also the function the failure- +//! injection end-to-end test calls to simulate a "node restart" after +//! the crash window the reviewer flagged. +//! +//! Idempotency is delegated to the DB layer. The push event and anchor +//! job use `ON CONFLICT (id) DO NOTHING` keyed on the deterministic +//! `(request_id, ref_name)` / `(repo_id, ref_name, old_sha, new_sha)` +//! id. The ref certificate uses +//! `insert_ref_certificate_idempotent`, which checks the unique +//! `(repo_id, ref_name)` index and returns `None` if a live-path cert +//! already exists. Re-running the drain against the same row is +//! therefore a no-op for the artifact writes; the row deletion at the +//! end is also idempotent because a missing `id` simply affects zero +//! rows. + +use crate::cert; +use crate::db::PendingRefTransition; +use crate::state::AppState; +use chrono::{DateTime, Utc}; +use std::collections::HashMap; + +/// The git all-zeros object id — the create/delete sentinel in a ref update. +const ZERO_SHA: &str = "0000000000000000000000000000000000000000"; + +/// Promote `prepared` rows whose `new_sha` matches the on-disk ref to +/// `applied`, so the recovery drain (which only reads `state = +/// 'applied'`) picks them up on the next pass. The reconcile runs at +/// startup, BEFORE the drain. +/// +/// This is the second half of the P1-A fix. The first half is the +/// live handler's `mark_pending_ref_transitions_applied` call, which +/// can fail or be interrupted AFTER `receive_pack` returned `Ok`. A +/// `prepared` row whose target ref actually landed on disk has no +/// recovery path without this step: the drain's WHERE clause does not +/// see it, and a startup that boots and serves traffic would silently +/// lose the push event, the cert, and the anchor handoff for that +/// ref. +/// +/// Strict SHA equality is the load-bearing correctness check. A row +/// whose `new_sha` does NOT match the on-disk ref stays `prepared`; +/// the invariant "a failed receive-pack is never promoted to +/// completed accounting" is preserved by the equality check, not by +/// state alone. A `cancelled` row is also never promoted (the +/// `list_pending_ref_transitions_prepared` SELECT gates on +/// `state = 'prepared'`, and the UPDATE re-checks the state). +/// +/// The SHA check alone is not sufficient. A `prepared` row could +/// have a `new_sha` that currently matches the on-disk ref for a +/// reason OTHER than its own transition (e.g. a later push +/// re-introduced the same SHA on the same ref). To prevent the +/// recovery drain from writing artifacts for a transition the node +/// cannot prove actually happened, the reconcile ALSO requires the +/// row's `created_at` to be within [`MAX_RECONCILE_AGE`] of the +/// current time. Rows older than the window are left `prepared` for +/// human-attended recovery. This is the second correctness barrier, +/// and the reason a `prepared` row that happens to match a current +/// on-disk SHA does not silently turn into completed accounting. +/// +/// P1 (reviewer round 3, second half): the SHA check plus the age +/// window is still not landing PROOF. The reviewer's case is +/// `old = B, new = A` on a ref that was ALREADY sitting at A — which +/// is not an exotic coincidence but the ordinary shape of a REJECTED +/// push, because git refuses an update whose expected old value does +/// not match the ref. Both checks pass, the push never happened, and +/// the drain would sign a certificate for it. +/// +/// The live path answers this from git's `report-status` body; that +/// body is long gone by the time the reconcile runs, so the proof is +/// re-derived from the repository itself: the ref's REFLOG must carry +/// an entry whose ` ` pair is exactly this row's, stamped at +/// or after the row was written (see [`reflog_proves_landing`]). That +/// is git's own record of the ref MOVING the way the row claims, after +/// the intent was durable, which is precisely what a coincidental tip +/// cannot produce. +/// +/// Deletions are exempt from the reflog half: git removes a ref's +/// reflog when it removes the ref, so absence of the ref plus the age +/// window is all the evidence that can exist for one. +/// +/// No reflog means NO PROOF, and no proof means no promotion — the row +/// stays put and is logged for human-attended recovery. +/// [`crate::git::store::init_bare`] turns `core.logAllRefUpdates` on +/// for every repo this node creates (bare repos default it off), so +/// the gap is repos predating that. Deliberate trade: a stranded row +/// an operator can see beats accounting the node cannot substantiate. +#[allow(dead_code)] // single-page seam; startup boots through the multi-pass walk below +pub async fn reconcile_prepared_from_disk(state: AppState, limit: i64) -> anyhow::Result { + reconcile_prepared_page(state, None, limit) + .await + .map(|(promoted, _cursor)| promoted) +} + +/// One page of the reconcile, resuming after `after`. Returns +/// `(promoted, next_cursor)`, where `next_cursor` is the +/// `(created_at, id)` of the last row EXAMINED — promoted or not — and +/// `None` once a short page says the backlog is exhausted. +/// [`reconcile_prepared_from_disk_all`] walks with it. +/// +/// The cursor is what makes the multi-pass loop actually advance. A +/// pass consumes every row it looked at, including the ones it refused +/// to promote (a SHA that does not match, a row past +/// [`MAX_RECONCILE_AGE`], a landing with no reflog proof). Those rows +/// stay in `prepared` / `uncertain` by design, so a cursor-less next +/// pass would re-read the same page forever and never reach the +/// backlog behind them. +async fn reconcile_prepared_page( + state: AppState, + after: Option<(String, String)>, + limit: i64, +) -> anyhow::Result<(usize, Option<(String, String)>)> { + // P1 (reviewer-1/2 round 3): also reconcile `uncertain` rows, + // not just `prepared`. A receive-pack error that leaves rows + // `uncertain` has the same recovery need as an interrupted + // success path that leaves rows `prepared`: the drain's WHERE + // clause does not see either state, and without this step the + // push event, cert, and anchor handoff would be permanently lost + // for refs that DID land before the error. + let rows = state + .db + .list_pending_ref_transitions_prepared_or_uncertain_after( + after.as_ref().map(|(ts, id)| (ts.as_str(), id.as_str())), + limit, + ) + .await?; + if rows.is_empty() { + return Ok((0, None)); + } + // #26 Split PR 1 step 5 — load the parent request rows once + // per page so the marker gate (per-row, O(1) lookup) doesn't + // N+1 the DB. Distinct request ids; the HashMap omits requests + // that have been purged by the step-4 bounded retirement or + // are missing for any other reason. + let distinct_request_ids: Vec = rows + .iter() + .map(|r| r.request_id.clone()) + .collect::>() + .into_iter() + .collect(); + let requests_by_id: std::collections::HashMap = state + .db + .get_receive_pack_requests_by_ids(&distinct_request_ids) + .await?; + // Taken BEFORE any promotion, from the last row of the page as it + // was READ: the walk advances over examined rows, not over promoted + // ones. A short page means there is nothing behind it. + let next_cursor = if (rows.len() as i64) < limit.max(1) { + None + } else { + rows.last().map(|r| (r.created_at.clone(), r.id.clone())) + }; + + // Group rows by repo so we call `list_refs` once per repo, not + // once per row. + let mut by_repo: HashMap> = HashMap::new(); + for row in &rows { + by_repo.entry(row.repo_id.clone()).or_default().push(row); + } + + let mut to_promote: Vec = Vec::new(); + for (repo_id, repo_rows) in by_repo { + let repo = match state.db.get_repo_by_id(&repo_id).await? { + Some(r) => r, + None => { + tracing::warn!( + repo_id = %repo_id, + row_count = repo_rows.len(), + "reconcile: repo row missing; leaving prepared rows untouched" + ); + continue; + } + }; + let disk_path = std::path::Path::new(&repo.disk_path); + let refs = match crate::git::store::list_refs(disk_path) { + Ok(v) => v, + Err(e) => { + tracing::warn!( + err = %e, + repo_id = %repo_id, + row_count = repo_rows.len(), + "reconcile: list_refs failed; leaving prepared rows untouched" + ); + continue; + } + }; + let disk_refs: HashMap = refs.into_iter().collect(); + + for row in repo_rows { + // P2 (reviewer-1/2 round 3): handle deletions. A deletion's + // new_sha is ZERO_SHA and `git for-each-ref` omits deleted + // refs. The old equality check (disk_refs.get(ref) == row.new_sha) + // can never match a deletion because ZERO_SHA is never returned + // by `list_refs`. Instead, when new_sha is ZERO_SHA, treat a + // missing ref as a successful deletion match. + let is_deletion = row.new_sha == ZERO_SHA; + let matches = if is_deletion { + !disk_refs.contains_key(&row.ref_name) + } else { + disk_refs + .get(&row.ref_name) + .map(|sha| sha == &row.new_sha) + .unwrap_or(false) + }; + if !matches { + let on_disk = disk_refs + .get(&row.ref_name) + .cloned() + .unwrap_or_else(|| "".to_string()); + tracing::debug!( + request_id = %row.request_id, + repo_id = %row.repo_id, + ref_name = %row.ref_name, + row_new_sha = %row.new_sha, + on_disk_sha = %on_disk, + is_deletion = is_deletion, + "reconcile: row's new_sha does not match on-disk ref; staying prepared" + ); + continue; + } + // P1 (reviewer round 3): the reflog proof is required + // below via `reflog_proves_landing` for non-deletions. + // Deletions stay exempt — git removes a ref's reflog + // along with the ref, so a deleted ref's transition is + // proven by the absence-plus-age check, not by a reflog + // entry that cannot exist. (See `reflog_proves_landing` + // for the full invariant.) The earlier round-4 work in + // this branch added a separate `has_reflog_landing` + // helper, but it duplicated the gate without the deletion + // exemption, so it prevented landed deletions from + // being promoted — the wrong direction. Kevin's + // `reflog_proves_landing` is the canonical gate; the + // call site below applies it with the `!is_deletion` + // exemption, so the redundant block is removed here. + // SHA matched (or deletion confirmed by absent ref). Before + // promoting, confirm the row is recent enough to be the + // transition that produced the current on-disk state. + let row_created_at = DateTime::parse_from_rfc3339(&row.created_at) + .ok() + .map(|t| t.with_timezone(&Utc)); + let row_age = row_created_at + .map(|t| Utc::now().signed_duration_since(t)) + .unwrap_or_else(|| { + tracing::warn!( + row_id = %row.id, + request_id = %row.request_id, + created_at = %row.created_at, + "reconcile: unparseable created_at; staying prepared (human-attended recovery)" + ); + MAX_RECONCILE_AGE + chrono::Duration::seconds(1) + }); + if row_age > MAX_RECONCILE_AGE { + tracing::warn!( + row_id = %row.id, + request_id = %row.request_id, + repo_id = %row.repo_id, + ref_name = %row.ref_name, + row_new_sha = %row.new_sha, + row_age_secs = row_age.num_seconds(), + max_reconcile_age_secs = MAX_RECONCILE_AGE.num_seconds(), + "reconcile: row is older than the recovery window; staying prepared (human-attended recovery required)" + ); + continue; + } + // P1 (reviewer round 3): landing PROOF, not just a matching + // tip. The reflog must show this exact `old -> new` move, + // stamped after the row was written. Deletions are exempt — + // a deleted ref takes its reflog with it, so absence plus + // the age window above is the whole evidence set for one. + if !is_deletion + && !reflog_proves_landing( + disk_path, + &row.ref_name, + &row.old_sha, + &row.new_sha, + row_created_at, + ) + { + tracing::warn!( + row_id = %row.id, + request_id = %row.request_id, + repo_id = %row.repo_id, + ref_name = %row.ref_name, + row_old_sha = %row.old_sha, + row_new_sha = %row.new_sha, + "reconcile: the ref sits at the row's new_sha but no reflog entry proves THIS \ + transition landed (a coincidental tip, or a repo without \ + core.logAllRefUpdates); staying prepared (human-attended recovery)" + ); + continue; + } + // #26 Split PR 1 step 5 — the marker gate. Reads + // `refs/gitlawb/requests/` and compares its + // value to the request's `request_bytes_hash`. A + // missing or mismatched marker quarantines the + // request; the row stays `prepared` (operator-attended, + // not auto-promoted). + let request = match requests_by_id.get(&row.request_id) { + Some(r) => r, + None => { + // Parent missing (purged or never written). + // Skip; the row stays prepared. + continue; + } + }; + let marker_ref = format!("refs/gitlawb/requests/{}", row.request_id); + let marker_ok = match crate::git::store::read_ref(disk_path, &marker_ref) { + Ok(Some(value)) => match crate::git::store::marker_value_for( + disk_path, + &request.request_bytes_hash, + ) { + Ok(expected) => value == expected, + Err(e) => { + tracing::warn!( + err = %e, + request_id = %row.request_id, + "reconcile: marker_value_for failed; staying prepared" + ); + false + } + }, + Ok(None) => false, + Err(e) => { + tracing::warn!( + err = %e, + request_id = %row.request_id, + "reconcile: marker read failed; staying prepared" + ); + false + } + }; + if !marker_ok { + let reason = match crate::git::store::read_ref(disk_path, &marker_ref) { + Ok(Some(_)) => "marker hash mismatch", + _ => "missing marker ref", + }; + if let Err(e) = state + .db + .mark_request_quarantined(&row.request_id, reason) + .await + { + tracing::warn!( + err = %e, + request_id = %row.request_id, + "reconcile: mark_request_quarantined failed" + ); + continue; + } + let _ = state + .db + .mark_children_rejected_for_quarantined_parent(&row.request_id) + .await; + tracing::warn!( + request_id = %row.request_id, + ref_name = %row.ref_name, + "reconcile: marker gate failed; request quarantined" + ); + continue; + } + to_promote.push(row.id.clone()); + } + } + + let flipped = state + .db + .mark_pending_ref_transitions_applied_for_rows(&to_promote) + .await?; + if flipped > 0 { + tracing::info!( + flipped, + "reconciled prepared/uncertain -> applied via on-disk ref match" + ); + } + Ok((flipped as usize, next_cursor)) +} + +/// Does the ref's reflog prove that THIS row's transition landed? +/// +/// True only when `logs/` carries an entry whose ` ` +/// pair is exactly this row's, stamped at or after the row became +/// durable (allowing [`REFLOG_CLOCK_SKEW`], since git stamps whole +/// seconds while `created_at` carries sub-second precision). The +/// timestamp half is what separates a landing from a LATER push that +/// re-introduced the same pair: proof must postdate the intent it +/// proves. +/// +/// False whenever proof is UNAVAILABLE — no reflog file (a repo +/// predating `core.logAllRefUpdates` in +/// [`crate::git::store::init_bare`]), an unreadable one, or no +/// matching entry. Absence of evidence is not evidence, so the caller +/// leaves such rows where they are instead of deciding either way. +fn reflog_proves_landing( + disk_path: &std::path::Path, + ref_name: &str, + old_sha: &str, + new_sha: &str, + row_created_at: Option>, +) -> bool { + let entries = match crate::git::store::ref_reflog_entries(disk_path, ref_name) { + Ok(Some(entries)) => entries, + Ok(None) => return false, + Err(e) => { + tracing::warn!( + err = %e, + ref_name = %ref_name, + "reconcile: could not read the ref's reflog; treating the landing as unproven" + ); + return false; + } + }; + // No parseable `created_at` means no lower bound to check an entry + // against, and the age gate above has already refused such a row; + // refuse here too rather than accept an entry of any age. + let Some(created_at) = row_created_at else { + return false; + }; + let floor = created_at.timestamp() - REFLOG_CLOCK_SKEW.num_seconds(); + entries + .iter() + .any(|e| e.old_sha == old_sha && e.new_sha == new_sha && e.at >= floor) +} + +/// How far BEFORE a row's `created_at` a reflog entry may be stamped +/// and still count as proof of that row's landing. +/// +/// Git writes whole-second reflog timestamps while `created_at` is an +/// RFC 3339 instant with sub-second precision, so a ref that landed +/// 200ms after the intent was written can carry a reflog stamp one +/// second EARLIER than the row. The tolerance covers that truncation +/// and small clock jitter; it is deliberately far smaller than +/// [`MAX_RECONCILE_AGE`], so it cannot readmit an old entry left by a +/// previous push of the same pair. +pub const REFLOG_CLOCK_SKEW: chrono::Duration = chrono::Duration::seconds(60); + +/// P2 (reviewer-1/2 round 3): multi-pass reconcile for the prepared/ +/// uncertain backlog. Mirrors `drain_receive_pack_requests_all`: +/// runs a reconcile pass in a loop until either a pass examines fewer +/// rows than `per_pass_limit` (backlog exhausted) or `max_passes` +/// passes have completed. If rows remain after the last pass, a +/// residual-backlog warning is logged and those rows wait for the next +/// startup. +/// +/// The passes WALK, on the `(created_at, id)` cursor each page returns. +/// The drain can re-issue the same query every pass because it deletes +/// the rows it finishes, so its next page is always new work; the +/// reconcile deletes nothing and leaves every unpromotable row exactly +/// where it was, so re-issuing a cursor-less query re-read page one on +/// every pass. One unprovable row at the head of the ordering — a SHA +/// that never landed, a row past [`MAX_RECONCILE_AGE`], or (since the +/// reflog gate) a landing in a repo that keeps no reflog — was enough +/// to pin the whole loop there and leave the backlog behind it +/// unexamined, which is the finding this loop exists to close. Those +/// rows keep ageing toward `MAX_RECONCILE_AGE` while they wait, so +/// "next restart" can mean "never recovered". +pub async fn reconcile_prepared_from_disk_all( + state: AppState, + per_pass_limit: i64, + max_passes: usize, +) -> anyhow::Result { + let mut total = 0; + let mut cursor: Option<(String, String)> = None; + for _ in 0..max_passes { + let (n, next) = + reconcile_prepared_page(state.clone(), cursor.clone(), per_pass_limit).await?; + total += n; + // A short page is the backlog-exhausted signal, keyed on rows + // EXAMINED rather than rows promoted: a pass that could promote + // nothing has still consumed its page and must move on. + match next { + Some(c) => cursor = Some(c), + None => return Ok(total), + } + } + // One more pass to detect residual backlog. + let (residual, next) = reconcile_prepared_page(state.clone(), cursor, per_pass_limit).await?; + total += residual; + if next.is_some() { + tracing::warn!( + total, + max_passes, + per_pass_limit, + "reconcile backlog exceeds startup budget; residual rows will be picked up on next restart" + ); + } + Ok(total) +} + +/// Per-pass drain budget. Each call to `drain_receive_pack_requests` +/// processes at most this many requests. +pub const DRAIN_PER_PASS_LIMIT: i64 = 1000; + +/// Maximum age (relative to `Utc::now()`) at which a `prepared` row +/// is auto-promoted by [`reconcile_prepared_from_disk`]. Rows older +/// than this stay `prepared` and require human-attended recovery. +/// +/// The window bounds the blast radius of a stale-row promotion: the +/// only way the on-disk SHA matches a `prepared` row's `new_sha` for +/// an OLD row is if some OTHER push re-introduced the same SHA on +/// the same ref after the original transition failed. With a bounded +/// window, that mis-match only matters for `created_at` within the +/// window — recent enough that an operator can correlate the row +/// with the live handler's logs. Older rows are deliberately left +/// `prepared` so a human can audit them rather than have the node +/// silently write a push event / cert / anchor for a transition the +/// node has no way to prove actually happened. +pub const MAX_RECONCILE_AGE: chrono::Duration = chrono::Duration::seconds(24 * 60 * 60); + +/// Maximum number of passes the startup drain will run before logging +/// a residual-backlog warning. With `DRAIN_PER_PASS_LIMIT = 1000` and +/// `DRAIN_MAX_PASSES = 10`, the startup drain runs `max_passes` regular +/// passes (10 × 1000 = 10,000 rows) plus ONE residual pass that +/// detects overrun and surfaces the residual-backlog warning at +/// `drain_receive_pack_requests_all`'s tail. Total rows per boot +/// before the warning fires: 11,000. Rows beyond that remain +/// `applied` and are picked up on the next startup. P2-doc +/// (reviewer-2 round 2): the previous comment said "up to 10,000" but +/// the residual pass is the +1. +pub const DRAIN_MAX_PASSES: usize = 10; + +/// #26 Split PR 1 step 3 — per-request drain. Replaces the v29 +/// per-ref walk with a per-request walk: the unit of work is the +/// `receive_pack_requests` row, and [`apply_request_effects`] does +/// all the artifact writes per request in a single idempotent +/// pass. The drain reads `outcomes_committed` and `effects_pending` +/// requests whose `next_attempt_at` is due. +/// +/// P2-A: a `apply_request_effects` failure on one request is logged +/// but does NOT abort the rest of the batch — the request stays in +/// `outcomes_committed` (or `effects_pending`) for a later startup +/// to retry. Idempotent inserts make this safe. +pub async fn drain_receive_pack_requests( + state: AppState, + limit: i64, +) -> anyhow::Result<(usize, usize)> { + drain_receive_pack_requests_with(state, limit, |s, req_id| async move { + apply_request_effects(&s, &req_id).await + }) + .await +} + +/// Testable seam for the per-request drain. Production code calls +/// [`drain_receive_pack_requests`], which delegates here with the +/// real [`apply_request_effects`]. Tests inject a closure that +/// returns `Retry` for one request and `Done` for another to assert +/// the loop's state-flip behavior. +pub async fn drain_receive_pack_requests_with( + state: AppState, + limit: i64, + derive_fn: F, +) -> anyhow::Result<(usize, usize)> +where + F: Fn(AppState, String) -> Fut, + Fut: std::future::Future>, +{ + let reqs = state.db.list_receive_pack_requests_due(limit).await?; + let mut processed = 0; + let examined = reqs.len(); + for req in reqs { + let request_id = req.id.clone(); + match derive_fn(state.clone(), request_id.clone()).await { + Ok(EffectsOutcome::Done) => { + if let Err(e) = state.db.mark_request_complete(&request_id).await { + tracing::warn!( + err = %e, + request_id = %request_id, + "drain: mark_request_complete failed; will retry next startup" + ); + continue; + } + processed += 1; + } + Ok(EffectsOutcome::Nothing) => { + // The request had no `accepted_ordinal`; nothing to + // do. Move to `complete` so the drain skips it next + // pass. + if let Err(e) = state.db.mark_request_complete(&request_id).await { + tracing::warn!( + err = %e, + request_id = %request_id, + "drain: mark_request_complete (Nothing) failed" + ); + continue; + } + processed += 1; + } + Ok(EffectsOutcome::Retry { last_error }) => { + let next_attempt_at = + (chrono::Utc::now() + chrono::Duration::seconds(60)).to_rfc3339(); + // #26 Split PR 1 step 5 — the bound check. After + // this retry, the request's `attempt_count` will + // become `current + 1` (the helper increments). + // If that exceeds `effects_max_attempts`, the + // request goes to `quarantined` instead of + // `effects_pending` to close the infinite-retry + // DoS window. + let bound = state.config.effects_max_attempts; + let over_bound = match state.db.get_receive_pack_request(&request_id).await { + Ok(Some(r)) => r.attempt_count + 1 > bound, + Ok(None) => { + // Row missing — the next startup's purge + // will sweep up. Treat as over-bound so + // the drain moves on. + true + } + Err(e) => { + tracing::warn!( + err = %e, + request_id = %request_id, + "drain: bound-check get_receive_pack_request failed; proceeding with retry" + ); + false + } + }; + if over_bound { + if let Err(e) = state + .db + .mark_request_quarantined(&request_id, &last_error) + .await + { + tracing::warn!( + err = %e, + request_id = %request_id, + "drain: mark_request_quarantined failed; will retry next startup" + ); + } else { + let _ = state + .db + .mark_children_rejected_for_quarantined_parent(&request_id) + .await; + } + } else if let Err(e) = state + .db + .mark_request_effects_pending(&request_id, &next_attempt_at, &last_error) + .await + { + tracing::warn!( + err = %e, + request_id = %request_id, + "drain: mark_request_effects_pending failed; will retry next startup" + ); + } + } + Err(e) => { + tracing::error!( + err = %e, + request_id = %request_id, + "drain: apply_request_effects returned Err; request left for next startup" + ); + } + } + } + Ok((processed, examined)) +} + +/// Drain an unbounded per-request backlog across multiple passes. +/// The residual warning is keyed on the request-table count. +pub async fn drain_receive_pack_requests_all( + state: AppState, + per_pass_limit: i64, + max_passes: usize, +) -> anyhow::Result { + let mut total = 0; + for _ in 0..max_passes { + let (processed, examined) = + drain_receive_pack_requests(state.clone(), per_pass_limit).await?; + total += processed; + if (examined as i64) < per_pass_limit { + return Ok(total); + } + } + let (residual_processed, _residual_examined) = + drain_receive_pack_requests(state.clone(), per_pass_limit).await?; + total += residual_processed; + let remaining_after_residual = state.db.count_receive_pack_requests_due().await?; + if remaining_after_residual > 0 { + tracing::warn!( + total, + max_passes, + per_pass_limit, + remaining_after_residual, + "drain: per-request backlog exceeds startup budget; residual requests will be picked up on next restart" + ); + } + Ok(total) +} + +/// #26 Split PR 1 step 4 — bounded retirement. Purges terminal +/// `receive_pack_requests` rows and their per-ref children that are +/// older than `retention_days`. Runs as a periodic task from +/// `main.rs` (one per cluster per day is the spec's target rate). +/// +/// Deletion order matters: purge the parent requests first, then +/// the orphaned children. The parent delete is bounded by the +/// partial index `idx_receive_pack_requests_completed_at` (built +/// by v30); the children delete is bounded by the same predicate +/// on `applied_at` / `cancelled_at`. +/// +/// `quarantined` rows are NEVER purged by this path — the spec +/// reserves those for operator inspection. Step 5 introduces the +/// `quarantined` state; this PR's purge is intentionally restricted +/// to `complete` and `rejected_at_git`. +/// +/// Returns `(requests_deleted, children_deleted)`. The caller logs +/// the totals; a non-zero `requests_deleted` is the success signal, +/// and a non-zero `children_deleted` after a `requests_deleted` of +/// zero is a hint that the children were orphaned by a previous +/// purge that crashed mid-run. +pub async fn purge_request_queue( + db: &crate::db::Db, + retention_days: i64, + per_pass_limit: i64, +) -> anyhow::Result<(u64, u64)> { + let older_than = chrono::Utc::now() - chrono::Duration::days(retention_days); + let older_than_iso = older_than.to_rfc3339(); + + let requests_deleted = db + .purge_completed_receive_pack_requests(&older_than_iso, per_pass_limit) + .await?; + let children_deleted = db + .purge_completed_pending_ref_transitions(&older_than_iso, per_pass_limit) + .await?; + + if requests_deleted > 0 || children_deleted > 0 { + tracing::info!( + retention_days, + older_than = %older_than_iso, + requests_deleted, + children_deleted, + "queue lifecycle: purged terminal request rows" + ); + } + Ok((requests_deleted, children_deleted)) +} + +/// Outcome of a single `apply_request_effects` call. The caller (live +/// handler or drain) decides what to do with the request row based on +/// this. +/// +/// `Done` — all four artifacts (push event, per-ref certs, per-ref +/// anchor jobs, trust-score bump) landed. The request is moved to +/// `complete`. +/// +/// `Nothing` — the request had no `accepted_ordinal` (no ref proved +/// landed, or the parsed report was empty). No effects were +/// attempted. The request is moved to `complete` (or +/// `rejected_at_git` if the parsed report shows an explicit failure; +/// the live handler does that flag separately). +/// +/// `Retry { last_error }` — one or more per-ref effects failed +/// transiently. The request is moved to `effects_pending` with +/// `next_attempt_at` in the future. The drain will retry on the next +/// startup. +#[derive(Debug)] +pub enum EffectsOutcome { + Done, + Nothing, + Retry { last_error: String }, +} + +/// #26 Split PR 1 step 3 — the shared effect executor. The live +/// handler and the recovery drain both call this function, so the +/// per-ref effects fan-out is in exactly one place. The function is +/// idempotent: every artifact write uses `ON CONFLICT` semantics +/// (deterministic id, `record_push_with_id` / `insert_anchor_job_idempotent` +/// / `insert_ref_certificate` upsert), so a recovery replay against +/// the same request produces the same artifacts the live path did. +/// +/// Crash-safety window: if a crash lands between "git returned" and +/// "all four artifacts written", the request row is in +/// `outcomes_committed` with no effects recorded. The drain picks it +/// up and re-runs the same effect pipeline, and the idempotent +/// inserts collapse to no-ops for the artifacts that did land. +/// +/// If a crash lands between "all artifacts written" and "request +/// moved to `complete`", the same drain pass completes the state +/// transition. The artifacts are already in place; the +/// `mark_request_complete` call is a single SQL UPDATE. +pub async fn apply_request_effects( + state: &AppState, + request_id: &str, +) -> anyhow::Result { + // 1. Load the request row. + let req = state + .db + .get_receive_pack_request(request_id) + .await? + .ok_or_else(|| anyhow::anyhow!("request row missing for {request_id}"))?; + + // 2. State gate: only `outcomes_committed` and `effects_pending` are + // eligible. Terminal states (`complete`, `rejected_at_git`) are + // skipped. + if !matches!( + req.state.as_str(), + crate::db::request_state::OUTCOMES_COMMITTED | crate::db::request_state::EFFECTS_PENDING + ) { + return Ok(EffectsOutcome::Nothing); + } + + // 3. No accepted ordinal means no ref proved landed. The request is + // eligible for `complete` (or `rejected_at_git` if the parsed + // report shows an explicit failure, but that flag is set by the + // handler's four-branch flip, not here). + let accepted_ordinal = match req.accepted_ordinal { + Some(o) => o, + None => return Ok(EffectsOutcome::Nothing), + }; + + // 4. Load the request's children. Certs and anchor jobs run for + // every child whose `ref_name` is in the parsed report's ok + // set; the request row's `parsed_report` is the durable + // record of that set. + let children = state + .db + .list_pending_ref_transitions_for_request(request_id) + .await?; + let ok_ref_names: std::collections::HashSet = req + .parsed_report + .as_ref() + .and_then(|v| v.get("ref_results")) + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|r| { + let ok = r.get("ok").and_then(|o| o.as_bool()).unwrap_or(false); + let name = r + .get("ref_name") + .and_then(|n| n.as_str()) + .map(|s| s.to_string()); + if ok { + name + } else { + None + } + }) + .collect() + }) + .unwrap_or_default(); + let accepted_children: Vec<&PendingRefTransition> = children + .iter() + .filter(|c| ok_ref_names.contains(&c.ref_name)) + .collect(); + + // 5. Look up the repo for cert/webhook payload construction. If + // the row is missing (deleted under us), bail with Retry so + // the drain re-runs later when the cache is warm again. + let repo = match state.db.get_repo_by_id(&req.repo_id).await? { + Some(r) => r, + None => { + return Ok(EffectsOutcome::Retry { + last_error: format!("repo {} not found", req.repo_id), + }); + } + }; + + // 6. Push event — written once, for the request. The live and + // recovery paths produce the same id because both key on + // `(request_id, accepted_ordinal)`. + let push_event_id = crate::db::push_event_id_for(&req.id, accepted_ordinal); + let accepted_ref = children.iter().find(|c| c.ordinal == accepted_ordinal); + let commit_hash = accepted_ref + .map(|c| c.new_sha.clone()) + .unwrap_or_else(|| chrono::Utc::now().timestamp().to_string()); + if let Err(e) = state + .db + .record_push_with_id( + &push_event_id, + &req.pusher_did, + &req.repo_id, + &commit_hash, + 0, + ) + .await + { + tracing::warn!( + err = %e, + request_id = %request_id, + "apply_request_effects: push event insert failed; request left for drain retry" + ); + return Ok(EffectsOutcome::Retry { + last_error: format!("push event: {e}"), + }); + } + + // 7. Trust score bump — best-effort, like the inline handler. A + // failure here does NOT retry the request; the bump is + // informational and the next push will catch up. + if let Ok(push_count) = state.db.get_push_count(&req.pusher_did).await { + // 0.05 base (from registration) + 0.05 per push, capped at 1.0 + let new_score = (push_count as f64 * 0.05 + 0.05).min(1.0); + let _ = state + .db + .update_trust_score(&req.pusher_did, new_score) + .await; + } + + // 8. Per-ref certs and anchor jobs. Each accepted child gets one + // of each. Failures are accumulated; the first one is + // returned as the Retry reason. + let mut first_error: Option = None; + for child in &accepted_children { + let cert_id = crate::db::ref_cert_id_for(&req.id, child.ordinal); + if let Err(e) = cert::issue_ref_certificate_with_issued_at( + state, + &req.repo_id, + &child.ref_name, + &child.old_sha, + &child.new_sha, + &req.pusher_did, + &cert_id, + Some(child.created_at.clone()), + ) + .await + { + tracing::warn!( + err = %e, + request_id = %request_id, + ref_name = %child.ref_name, + "apply_request_effects: cert insert failed; child left for drain retry" + ); + first_error.get_or_insert_with(|| format!("cert {}: {e}", child.ref_name)); + continue; + } + + let anchor_id = crate::db::anchor_job_id_for( + &req.repo_id, + &child.ref_name, + &child.old_sha, + &child.new_sha, + ); + let job = crate::db::AnchorJob { + id: anchor_id, + repo_id: req.repo_id.clone(), + ref_name: child.ref_name.clone(), + old_sha: child.old_sha.clone(), + new_sha: child.new_sha.clone(), + pusher_did: req.pusher_did.clone(), + created_at: chrono::Utc::now().to_rfc3339(), + claimed_at: None, + }; + if let Err(e) = state.db.insert_anchor_job_idempotent(&job).await { + tracing::warn!( + err = %e, + request_id = %request_id, + ref_name = %child.ref_name, + "apply_request_effects: anchor insert failed; child left for drain retry" + ); + first_error.get_or_insert_with(|| format!("anchor {}: {e}", child.ref_name)); + } + } + + if let Some(err) = first_error { + return Ok(EffectsOutcome::Retry { last_error: err }); + } + + // 9. All artifacts landed — clean up the children and let the + // caller move the request to `complete`. + if let Err(e) = state + .db + .delete_pending_ref_transitions_by_request_id(request_id) + .await + { + tracing::warn!( + err = %e, + request_id = %request_id, + "apply_request_effects: child cleanup failed; idempotent retry will pick them up on next pass" + ); + // Don't fail the request — the artifacts are in place and a + // future pass is harmless. + } + + // 10. Webhooks — best-effort, per landed ref. Same shape as the + // inline handler's webhook block. + if !ok_ref_names.is_empty() { + let base_url = state + .config + .public_url + .as_deref() + .unwrap_or("http://127.0.0.1:7545") + .trim_end_matches('/'); + let owner_short = crate::db::normalize_owner_key(&repo.owner_did); + let clone_url = format!("{}/{}/{}.git", base_url, owner_short, repo.name); + for child in &accepted_children { + let payload = serde_json::json!({ + "ref": child.ref_name, + "before": child.old_sha, + "after": child.new_sha, + "created": child.old_sha == "0000000000000000000000000000000000000000", + "forced": false, + "pusher": { + "did": req.pusher_did, + }, + "repository": { + "id": repo.id, + "name": repo.name, + "owner_did": repo.owner_did, + "clone_url": clone_url, + }, + }); + crate::webhooks::fire_event( + state.db.clone(), + state.http_client.clone(), + &repo.id, + "push", + payload, + ); + } + } + + Ok(EffectsOutcome::Done) +} + +#[cfg(test)] +mod drain_tests { + //! End-to-end failure-injection test the reviewer demanded: + //! + //! "Inject failure after Git applies the ref but before the first + //! transition/job write, restart the node, and show that the + //! original transition produces exactly one push event, one + //! certificate carrying the original pusher/proof, and at most + //! one anchor upload." + //! + //! The crash window is simulated by inserting a + //! `pending_ref_transitions` row directly in `applied` state + //! (bypassing the handler). The drain then re-derives the three + //! artifacts using the persisted authentic pusher DID and the + //! raw RFC 9421 signature header. Assertions check the invariants + //! the reviewer named: exactly one push event row, exactly one + //! cert row carrying the original pusher, exactly one anchor job + //! row. A second drain pass is a no-op. + //! + //! Each assertion names the invariant it pins. Reverting the + //! production line under test turns the named assertion red. + + use super::*; + use crate::db::pending_state; + use crate::db::request_state; + use crate::db::Db; + use crate::db::PendingRefTransition; + use chrono::Utc; + use std::path::Path; + + async fn _db(pool: sqlx::PgPool) -> Db { + let db = Db::for_testing(pool); + db.run_migrations().await.unwrap(); + db + } + + fn make_row(repo_id: &str, ref_name: &str, old: &str, new: &str) -> PendingRefTransition { + let now = Utc::now().to_rfc3339(); + PendingRefTransition { + id: crate::db::deterministic_id(&[ + "pending_ref_transition", + "req-1", + repo_id, + ref_name, + old, + new, + ]), + request_id: "req-1".to_string(), + repo_id: repo_id.to_string(), + ref_name: ref_name.to_string(), + old_sha: old.to_string(), + new_sha: new.to_string(), + pusher_did: "did:key:z6pusher".to_string(), + node_did: "did:key:z6node".to_string(), + signature_header: "Signature: sig=\"abc...\"".to_string(), + signature_input: "Signature-Input: sig=(\"@authority\");...".to_string(), + content_digest: "Content-Digest: sha-256=:...:".to_string(), + state: pending_state::APPLIED.to_string(), + created_at: now.clone(), + applied_at: Some(now), + cancelled_at: None, + // The existing tests are single-ref pushes, so the + // request's only child is ordinal 0. The new multi-ref + // test sets this explicitly per child. + ordinal: 0, + git_target_kind: Some("update".to_string()), + } + } + + /// Stage a `receive_pack_requests` row in `outcomes_committed` + /// alongside the per-ref children that landed under it. The + /// `parsed_report` is the durable record the effect executor + /// reads to decide which children are `ok`. Each child is + /// inserted via `insert_pending_ref_transition_for_test`, so + /// the deterministic PKs match what the production handler + /// would write. The repo row is also seeded so `apply_request_effects`'s + /// `get_repo_by_id` lookup succeeds (the live handler always + /// has the repo in cache before the effect executor is called). + async fn stage_request_with_children( + db: &Db, + request_id: &str, + repo_id: &str, + accepted_ordinal: Option, + children: &[PendingRefTransition], + parsed_report: serde_json::Value, + ) { + stage_request_with_pusher( + db, + request_id, + repo_id, + "did:key:z6pusher", + accepted_ordinal, + children, + parsed_report, + ) + .await; + } + + /// Like [`stage_request_with_children`] but lets the caller pick + /// the request row's `pusher_did`. Used by the cert-refresh + /// tests where the recovery's pusher DID must NOT match the + /// helper's default. + async fn stage_request_with_pusher( + db: &Db, + request_id: &str, + repo_id: &str, + pusher_did: &str, + accepted_ordinal: Option, + children: &[PendingRefTransition], + parsed_report: serde_json::Value, + ) { + // Seed a minimal repo row so the effect executor's + // `get_repo_by_id` lookup succeeds. `ON CONFLICT DO NOTHING` + // means tests that already seeded a repo (e.g. cert-refresh + // tests that need a specific `owner_did`) are unaffected. + sqlx::query( + r#"INSERT INTO repos (id, name, owner_did, description, is_public, default_branch, + created_at, updated_at, disk_path, forked_from, machine_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + ON CONFLICT (id) DO NOTHING"#, + ) + .bind(repo_id) + .bind(repo_id) + .bind(pusher_did) + .bind(Option::::None) + .bind(true) + .bind("main") + .bind(chrono::Utc::now().to_rfc3339()) + .bind(chrono::Utc::now().to_rfc3339()) + .bind(format!("/tmp/{repo_id}")) + .bind(Option::::None) + .bind(Option::::None) + .execute(db.pool()) + .await + .expect("seed repo row"); + + sqlx::query( + r#"INSERT INTO receive_pack_requests + (id, repo_id, pusher_did, node_did, request_bytes, request_bytes_hash, + state, git_exit_ok, parsed_report, accepted_ordinal, attempt_count, + last_error, next_attempt_at, created_at, completed_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)"#, + ) + .bind(request_id) + .bind(repo_id) + .bind(pusher_did) + .bind("did:key:z6node") + .bind(Vec::::new()) + .bind([0u8; 32].to_vec()) + .bind(crate::db::request_state::OUTCOMES_COMMITTED) + .bind(Some(true)) + .bind(&parsed_report) + .bind(accepted_ordinal) + .bind(0_i32) + .bind(Option::::None) + .bind(Option::::None) + .bind(Utc::now().to_rfc3339()) + .bind(Option::::None) + .execute(db.pool()) + .await + .unwrap(); + + for child in children { + db.insert_pending_ref_transition_for_test(child) + .await + .unwrap(); + } + } + + /// Build the `parsed_report` JSON the drain reads. The + /// `apply_request_effects` effect-executor uses the `ok` field + /// per `ref_name` to decide which children get certs and + /// anchors; the `accepted_ordinal` field on the request row + /// picks the row whose `new_sha` carries the push event. + fn parsed_report_ok(refs: &[(&str, bool)]) -> serde_json::Value { + serde_json::json!({ + "unpack_ok": true, + "ref_results": refs.iter().map(|(name, ok)| serde_json::json!({ + "ref_name": name, + "ok": ok, + })).collect::>(), + }) + } + + /// #26 Split PR 1 step 5 — write the per-request marker ref via + /// `git update-ref`. The marker's value is the 40-char SHA-1 hex + /// of a blob whose bytes are the first 20 bytes of the request's + /// `request_bytes_hash` (32-byte SHA-256). `git update-ref` + /// rejects arbitrary 64-char hex and only accepts 40-char SHA-1 + /// that resolves to an existing object; `marker_value_for` does + /// the `hash-object -w` half so the value is content-addressed. + /// The reconcile's `read_ref` reads it back and compares hex + /// strings via the same helper. + /// + /// The live handler in `api/repos.rs` follows the same scheme. + /// + /// Tests that intentionally exercise the missing-marker path skip + /// this helper. + async fn stage_marker(repo_path: &Path, request_id: &str, request_bytes_hash: &[u8]) { + let marker_ref = format!("refs/gitlawb/requests/{request_id}"); + let marker_value = crate::git::store::marker_value_for(repo_path, request_bytes_hash) + .expect("marker_value_for"); + let out = tokio::process::Command::new("git") + .args(["update-ref", &marker_ref, &marker_value]) + .arg("--no-deref") + .current_dir(repo_path) + .output() + .await + .expect("git update-ref"); + assert!( + out.status.success(), + "git update-ref for marker failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + + /// #26 Split PR 1 step 5 — the marker gate's positive path + /// requires both a parent `receive_pack_requests` row AND a + /// matching marker ref on disk. Insert the parent row in + /// `received` state with the given hash (so the reconcile's + /// `get_receive_pack_requests_by_ids` lookup hits and the gate + /// has something to verify). Tests call `stage_marker` after + /// this to write the matching ref. + async fn seed_parent_request( + db: &Db, + request_id: &str, + repo_id: &str, + request_bytes_hash: Vec, + ) { + sqlx::query( + r#"INSERT INTO receive_pack_requests + (id, repo_id, pusher_did, node_did, request_bytes, request_bytes_hash, + state, git_exit_ok, parsed_report, accepted_ordinal, attempt_count, + last_error, next_attempt_at, created_at, completed_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)"#, + ) + .bind(request_id) + .bind(repo_id) + .bind("did:key:z6pusher") + .bind("did:key:z6node") + .bind(Vec::::new()) + .bind(&request_bytes_hash) + .bind(crate::db::request_state::RECEIVED) + .bind(Option::::None) + .bind(Option::::None) + .bind(Option::::None) + .bind(0_i32) + .bind(Option::::None) + .bind(Option::::None) + .bind(Utc::now().to_rfc3339()) + .bind(Option::::None) + .execute(db.pool()) + .await + .expect("seed parent receive_pack_requests row"); + } + + /// The reviewer's proof at the durable-outbox layer. Stage a + /// `receive_pack_requests` row in `outcomes_committed` with one + /// landed child (the crash window — receive_pack returned Ok and + /// git accepted, only the effects fan-out didn't run), drain, and + /// assert exactly one push event, one cert with the original + /// pusher, one anchor job, and the request moved to `complete`. + #[sqlx::test] + async fn drain_re_derives_all_three_artifacts_for_an_applied_row(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + let repo_id = "repo-failure-injection"; + let ref_name = "refs/heads/main"; + let old = "a".repeat(40); + let new = "b".repeat(40); + let row = make_row(repo_id, ref_name, &old, &new); + let request_id = row.request_id.clone(); + let parsed_report = parsed_report_ok(&[(ref_name, true)]); + stage_request_with_children( + &state.db, + &request_id, + repo_id, + Some(row.ordinal), + std::slice::from_ref(&row), + parsed_report, + ) + .await; + + let (n, examined) = drain_receive_pack_requests(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n, 1, "exactly one request re-derived"); + assert_eq!(examined, 1, "the loop examined the single request"); + + // Push event: exactly one row, keyed on the deterministic id. + let _push_id = crate::db::push_event_id_for(&row.request_id, row.ordinal); + let push_count = state + .db + .count_push_events(&row.repo_id, &row.new_sha, &row.pusher_did) + .await + .unwrap(); + assert_eq!( + push_count, 1, + "exactly one push event, keyed on the original pusher" + ); + + // Cert: exactly one row, carrying the original pusher DID. + let certs = state + .db + .list_ref_certificates(&row.repo_id, 10) + .await + .unwrap(); + assert_eq!(certs.len(), 1, "exactly one ref certificate"); + assert_eq!( + certs[0].pusher_did, row.pusher_did, + "cert carries the original pusher DID, not a placeholder" + ); + assert_eq!( + certs[0].id, + crate::db::ref_cert_id_for(&row.request_id, row.ordinal), + "cert id is deterministic" + ); + assert_eq!(certs[0].new_sha, row.new_sha, "cert carries the new_sha"); + assert_eq!(certs[0].old_sha, row.old_sha, "cert carries the old_sha"); + + // Anchor job: exactly one row. + let anchor_count = state + .db + .count_anchor_jobs(&row.repo_id, &row.ref_name, &row.old_sha, &row.new_sha) + .await + .unwrap(); + assert_eq!(anchor_count, 1, "exactly one anchor job per transition"); + + // The request row moved to `complete`. + let after = state + .db + .get_receive_pack_request(&request_id) + .await + .unwrap(); + assert_eq!( + after.expect("request row exists").state, + crate::db::request_state::COMPLETE, + "drain moves the request to complete" + ); + // Children are cleaned up. + let still_applied = state + .db + .list_pending_ref_transitions_applied(100) + .await + .unwrap(); + assert!( + still_applied.is_empty(), + "drain deletes the children after the work lands" + ); + + // A second drain pass is a no-op. + let (n2, examined2) = drain_receive_pack_requests(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n2, 0, "a second drain pass has nothing to do"); + assert_eq!(examined2, 0, "no requests to examine on a second pass"); + } + + /// The reviewer's second proof, end-to-end. A request that git + /// rejected (no `accepted_ordinal`) never produces a push event, + /// cert, or anchor. The drain still picks up the request + /// (because it's in `outcomes_committed` — the live handler + /// always lands here after git returns), `apply_request_effects` + /// returns `Nothing` because there is no accepted ref, and the + /// drain moves the request to `complete` without writing any + /// artifacts. + #[sqlx::test] + async fn rejected_at_git_request_produces_no_artifacts(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + // Stage a request in `outcomes_committed` with NO + // `accepted_ordinal` (git rejected all refs). The drain + // picks it up, `apply_request_effects` short-circuits at the + // `accepted_ordinal.is_none()` gate, and the drain calls + // `mark_request_complete` for `Nothing`. + let request_id = "req-rejected"; + let repo_id = "repo-rejected"; + let parsed_report = serde_json::json!({ + "unpack_ok": false, + "ref_results": [{ + "ref_name": "refs/heads/main", + "ok": false, + "message": "deny non-fast-forward", + }], + }); + stage_request_with_children(&state.db, request_id, repo_id, None, &[], parsed_report).await; + + let (n, examined) = drain_receive_pack_requests(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n, 1, "drain processed the no-effect request"); + assert_eq!(examined, 1, "the loop examined the request"); + + // The request is now `complete` — Nothing outcome moves it. + let after = state.db.get_receive_pack_request(request_id).await.unwrap(); + assert_eq!( + after.expect("request row exists").state, + crate::db::request_state::COMPLETE, + "Nothing outcome moves the request to complete" + ); + + // No push event, no cert, no anchor. + let push_count = state.db.get_push_count("did:key:z6pusher").await.unwrap(); + assert_eq!( + push_count, 0, + "no push event for a request with no accepted ref" + ); + let certs = state.db.list_ref_certificates(repo_id, 10).await.unwrap(); + assert!(certs.is_empty(), "no certs for a no-effect request"); + let still_applied = state + .db + .list_pending_ref_transitions_applied(100) + .await + .unwrap(); + assert!( + still_applied.is_empty(), + "no children exist for this no-effect request" + ); + } + + /// A request the handler has not yet finished (state = + /// `received`, git has not yet returned) is invisible to the + /// per-request drain. The drain only reads `outcomes_committed` + /// and `effects_pending`, so a `received` row stays where the + /// handler left it and no effects are attempted. + #[sqlx::test] + async fn received_request_produces_no_artifacts(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + // Stage the request row directly in `received` (the state + // the handler writes before git returns). + let request_id = "req-received"; + let repo_id = "repo-received"; + sqlx::query( + r#"INSERT INTO receive_pack_requests + (id, repo_id, pusher_did, node_did, request_bytes, request_bytes_hash, + state, git_exit_ok, parsed_report, accepted_ordinal, attempt_count, + last_error, next_attempt_at, created_at, completed_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)"#, + ) + .bind(request_id) + .bind(repo_id) + .bind("did:key:z6pusher") + .bind("did:key:z6node") + .bind(Vec::::new()) + .bind([0u8; 32].to_vec()) + .bind(crate::db::request_state::RECEIVED) + .bind(Option::::None) + .bind(Option::::None) + .bind(Option::::None) + .bind(0_i32) + .bind(Option::::None) + .bind(Option::::None) + .bind(Utc::now().to_rfc3339()) + .bind(Option::::None) + .execute(state.db.pool()) + .await + .unwrap(); + + // The drain must not pick up a `received` row. + let (n, examined) = drain_receive_pack_requests(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n, 0, "the drain must not touch a `received` request"); + assert_eq!(examined, 0, "the drain's WHERE excludes `received`"); + + // The request is unchanged. + let after = state.db.get_receive_pack_request(request_id).await.unwrap(); + assert_eq!( + after.expect("request row exists").state, + crate::db::request_state::RECEIVED, + "received requests are left to the handler" + ); + + let push_count = state.db.get_push_count("did:key:z6pusher").await.unwrap(); + assert_eq!(push_count, 0, "no push event for an unstarted request"); + } + + // ----- P1-A reconcile tests ----- + // + // These tests cover the startup-time `reconcile_prepared_from_disk` + // step: a `prepared` row whose target ref actually landed on disk + // is promoted to `applied`; a row whose target did NOT land (or + // whose ref is missing) stays `prepared`; a `cancelled` row is + // never promoted; and a second call is a no-op. + // + // The on-disk state is a real bare git repo (so `list_refs` can + // read it) seeded with a synthetic commit via the plumbing + // commands `mktree` (empty tree) + `commit-tree` (root commit) + + // `update-ref` (point a ref at the commit). + + /// Build a real commit on a bare git repo's `ref_name`. Returns + /// the new commit SHA. Used by the reconcile tests to seed a + /// known SHA on disk so `list_refs` can read it back. + fn seed_ref_on_bare(bare_path: &std::path::Path, ref_name: &str) -> String { + use std::process::Command; + // Empty tree. + let tree = String::from_utf8( + Command::new("git") + .args(["mktree"]) + .current_dir(bare_path) + .stdin(std::process::Stdio::null()) + .output() + .expect("git mktree") + .stdout, + ) + .expect("mktree stdout utf8") + .trim() + .to_string(); + // Root commit on the empty tree. The env vars override any + // missing global config in CI. + let commit = String::from_utf8( + Command::new("git") + .args(["commit-tree", &tree, "-m", "test root"]) + .env("GIT_AUTHOR_NAME", "test") + .env("GIT_AUTHOR_EMAIL", "test@example.com") + .env("GIT_COMMITTER_NAME", "test") + .env("GIT_COMMITTER_EMAIL", "test@example.com") + .current_dir(bare_path) + .stdin(std::process::Stdio::null()) + .output() + .expect("git commit-tree") + .stdout, + ) + .expect("commit-tree stdout utf8") + .trim() + .to_string(); + // Point the ref at the commit. `update-ref` writes into the + // bare repo's refs/ tree. + Command::new("git") + .args(["update-ref", ref_name, &commit]) + .current_dir(bare_path) + .stdin(std::process::Stdio::null()) + .output() + .expect("git update-ref"); + commit + } + + /// Seed a `RepoRecord` row pointing at `disk_path` and return + /// the repo id. Mirrors what `repos::create_repo` does in + /// production, but without the rest of the create-repo + /// bookkeeping the test does not exercise. + async fn seed_repo_row(state: &crate::state::AppState, disk_path: &str) -> String { + use crate::db::RepoRecord; + use chrono::Utc; + let repo_id = uuid::Uuid::new_v4().to_string(); + state + .db + .create_repo(&RepoRecord { + id: repo_id.clone(), + name: "reconcile-test".into(), + owner_did: "did:key:z6owner".into(), + description: None, + is_public: true, + default_branch: "main".into(), + created_at: Utc::now(), + updated_at: Utc::now(), + disk_path: disk_path.to_string(), + forked_from: None, + machine_id: None, + }) + .await + .expect("create_repo"); + repo_id + } + + #[sqlx::test] + async fn reconcile_promotes_prepared_row_when_on_disk_sha_matches(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + // Create a bare repo on disk with refs/heads/main = X. + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + let on_disk_sha = seed_ref_on_bare(&bare, "refs/heads/main"); + + // Persist a `prepared` row whose `new_sha` matches the on-disk + // SHA. This simulates the crash window the reviewer flagged: + // receive_pack returned Ok and the ref landed, but the + // handler never reached `mark_pending_ref_transitions_applied`. + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + let mut row = make_row(&repo_id, "refs/heads/main", &"0".repeat(40), &on_disk_sha); + row.state = pending_state::PREPARED.to_string(); + row.applied_at = None; + // #26 Split PR 1 step 5 — the reconcile's marker gate requires + // both a parent `receive_pack_requests` row AND a matching + // marker ref on disk. Seed the parent (so the gate has + // something to verify) and write the matching marker. + seed_parent_request(&state.db, &row.request_id, &repo_id, vec![0xab; 32]).await; + stage_marker(&bare, &row.request_id, &[0xab; 32]).await; + state + .db + .insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + + // The drain must NOT see the row before reconcile (it only + // reads `applied`). + let pre_drain = state + .db + .list_pending_ref_transitions_applied(100) + .await + .unwrap(); + assert!(pre_drain.is_empty(), "drain cannot see a prepared row"); + + // Reconcile: the row's new_sha matches the on-disk ref, so it + // should be promoted. + let n = reconcile_prepared_from_disk(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n, 1, "exactly one row promoted to applied"); + + // The row is now in `applied` and the drain can see it. + let after_drain = state + .db + .list_pending_ref_transitions_applied(100) + .await + .unwrap(); + assert_eq!(after_drain.len(), 1, "row is now visible to the drain"); + assert_eq!(after_drain[0].id, row.id, "the same row is promoted"); + assert_eq!(after_drain[0].state, pending_state::APPLIED); + assert!( + after_drain[0].applied_at.is_some(), + "applied_at is set on promotion" + ); + + // The prepared list is now empty. + let still_prepared = state + .db + .list_pending_ref_transitions_prepared(100) + .await + .unwrap(); + assert!( + still_prepared.is_empty(), + "no prepared rows remain after a successful reconcile" + ); + } + + #[sqlx::test] + async fn reconcile_leaves_prepared_row_when_on_disk_sha_differs(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + // On-disk SHA is `aaaa...` (the live push landed at this SHA). + // The row's `new_sha` is `bbbb...` — the SHA the row CLAIMS + // the push went to, but the actual on-disk state disagrees. + // This models a row stranded by a `mark_applied` failure on a + // push whose target was rolled back, or any case where the + // recorded `new_sha` does not match reality. + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + let on_disk_sha = seed_ref_on_bare(&bare, "refs/heads/main"); + assert_ne!(on_disk_sha, "b".repeat(40), "test sanity: SHAs differ"); + + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + let mut row = make_row( + &repo_id, + "refs/heads/main", + &"0".repeat(40), + &"b".repeat(40), + ); + row.state = pending_state::PREPARED.to_string(); + row.applied_at = None; + state + .db + .insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + + // Reconcile: the SHA does not match, so NOTHING is promoted. + let n = reconcile_prepared_from_disk(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n, 0, "no row promoted when SHAs do not match"); + + // The row is still `prepared` and the drain cannot see it. + let after_drain = state + .db + .list_pending_ref_transitions_applied(100) + .await + .unwrap(); + assert!( + after_drain.is_empty(), + "drain must not see a row whose on-disk SHA does not match" + ); + let still_prepared = state + .db + .list_pending_ref_transitions_prepared(100) + .await + .unwrap(); + assert_eq!(still_prepared.len(), 1, "row stays prepared"); + assert_eq!(still_prepared[0].id, row.id); + assert!( + still_prepared[0].applied_at.is_none(), + "applied_at is NOT set on a non-promotion" + ); + } + + #[sqlx::test] + async fn reconcile_leaves_cancelled_row_untouched(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + // On-disk ref matches the row's `new_sha` — but the row is + // `cancelled`, so the reconcile must NEVER promote it. The + // reviewer's invariant: a failed receive-pack is never + // promoted to completed accounting. + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + let on_disk_sha = seed_ref_on_bare(&bare, "refs/heads/main"); + + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + let mut row = make_row(&repo_id, "refs/heads/main", &"0".repeat(40), &on_disk_sha); + row.state = pending_state::CANCELLED.to_string(); + row.applied_at = None; + state + .db + .insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + + let n = reconcile_prepared_from_disk(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n, 0, "cancelled rows are never promoted"); + + // The row is still cancelled and the drain still cannot see it. + let after_drain = state + .db + .list_pending_ref_transitions_applied(100) + .await + .unwrap(); + assert!(after_drain.is_empty()); + let still_prepared = state + .db + .list_pending_ref_transitions_prepared(100) + .await + .unwrap(); + assert!( + still_prepared.is_empty(), + "cancelled rows do not show up in the prepared list either" + ); + } + + #[sqlx::test] + async fn reconcile_is_idempotent(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + let on_disk_sha = seed_ref_on_bare(&bare, "refs/heads/main"); + + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + let mut row = make_row(&repo_id, "refs/heads/main", &"0".repeat(40), &on_disk_sha); + row.state = pending_state::PREPARED.to_string(); + row.applied_at = None; + // #26 Split PR 1 step 5 — seed parent + marker for the gate. + seed_parent_request(&state.db, &row.request_id, &repo_id, vec![0xcd; 32]).await; + stage_marker(&bare, &row.request_id, &[0xcd; 32]).await; + state + .db + .insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + + // First call: 1 row promoted. + let n1 = reconcile_prepared_from_disk(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n1, 1); + // Second call: 0 rows — the row is no longer `prepared`, so + // the list query returns empty. + let n2 = reconcile_prepared_from_disk(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n2, 0, "a second reconcile is a no-op"); + + // Final state: applied, with applied_at set. + let applied = state + .db + .list_pending_ref_transitions_applied(100) + .await + .unwrap(); + assert_eq!(applied.len(), 1); + assert_eq!(applied[0].state, pending_state::APPLIED); + assert!(applied[0].applied_at.is_some()); + } + + /// A `prepared` row whose `new_sha` happens to match the current + /// on-disk ref value for a reason OTHER than its own transition + /// (e.g. a later push re-introduced the same SHA on the same + /// ref) must NOT be promoted just because the SHAs match. The + /// `MAX_RECONCILE_AGE` window is the second correctness barrier: + /// rows older than the window stay `prepared` for + /// human-attended recovery. + /// + /// This test seeds a `prepared` row whose `new_sha` DOES match + /// the on-disk ref, but whose `created_at` is 25 hours in the + /// past (one hour past `MAX_RECONCILE_AGE = 24h`). The reconcile + /// must NOT promote it. + #[sqlx::test] + async fn reconcile_does_not_promote_stale_prepared_row(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + // On-disk ref with a known SHA. + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + let on_disk_sha = seed_ref_on_bare(&bare, "refs/heads/main"); + + // Build a `prepared` row whose SHA matches the on-disk ref, + // but whose `created_at` is older than `MAX_RECONCILE_AGE`. + // This models a row that was stranded by an ancient + // `mark_applied` failure and then re-introduced the same + // SHA via a later push. + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + let mut row = make_row(&repo_id, "refs/heads/main", &"0".repeat(40), &on_disk_sha); + row.state = pending_state::PREPARED.to_string(); + row.applied_at = None; + // 25 hours ago — outside the 24h window. + row.created_at = (chrono::Utc::now() - chrono::Duration::hours(25)).to_rfc3339(); + // `make_row` derives `id` from the deterministic hash using + // the `created_at` it generated at construction time. Now + // that we've overwritten `created_at`, the row's `id` no + // longer matches what `insert_pending_ref_transitions` + // would have produced in production, but the test only + // checks the reconcile's behavior, not the id's contents, + // so the stale id is harmless here. + state + .db + .insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + + // The SHA matches, but the row is older than the window: + // reconcile must NOT promote it. + let n = reconcile_prepared_from_disk(state.clone(), 100) + .await + .unwrap(); + assert_eq!( + n, 0, + "stale row stays prepared (SHA matched but age exceeded the recovery window)" + ); + + // The row is still `prepared` and the drain cannot see it. + let after_drain = state + .db + .list_pending_ref_transitions_applied(100) + .await + .unwrap(); + assert!( + after_drain.is_empty(), + "drain must not see a row outside the recovery window" + ); + let still_prepared = state + .db + .list_pending_ref_transitions_prepared(100) + .await + .unwrap(); + assert_eq!(still_prepared.len(), 1, "row stays prepared"); + assert_eq!(still_prepared[0].id, row.id); + assert!( + still_prepared[0].applied_at.is_none(), + "applied_at is NOT set on a stale row" + ); + } + + /// A `prepared` row that is fresh (within `MAX_RECONCILE_AGE`) + /// and SHA-matches the on-disk ref MUST still be promoted. This + /// is the existing happy-path contract; the test pins it so a + /// future change to the age check does not silently break the + /// recovery path for legitimate stranded rows. + #[sqlx::test] + async fn reconcile_promotes_fresh_prepared_row_with_matching_sha(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + let on_disk_sha = seed_ref_on_bare(&bare, "refs/heads/main"); + + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + // `make_row` defaults `created_at` to `Utc::now()`, which is + // well within `MAX_RECONCILE_AGE`. The SHA matches. This + // row should be promoted. + let mut row = make_row(&repo_id, "refs/heads/main", &"0".repeat(40), &on_disk_sha); + row.state = pending_state::PREPARED.to_string(); + row.applied_at = None; + // #26 Split PR 1 step 5 — seed parent + marker for the gate. + seed_parent_request(&state.db, &row.request_id, &repo_id, vec![0x11; 32]).await; + stage_marker(&bare, &row.request_id, &[0x11; 32]).await; + state + .db + .insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + + let n = reconcile_prepared_from_disk(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n, 1, "fresh row with matching SHA is promoted"); + } + + // ----- P1 (reviewer round 3, second half): reflog landing proof ----- + // + // The SHA match plus the age window says "the ref is where the row + // wanted it". It does NOT say the row's push is what put it there. + // These tests pin the difference, which is what + // `reflog_proves_landing` decides. + + /// THE reviewer's case. A row claims `B -> A` while the ref has been + /// sitting at A all along — the ordinary shape of a REJECTED push, + /// since git refuses an update whose expected old value is stale. + /// The SHA matches and the row is fresh, so only the reflog refuses + /// it; without that refusal the drain writes a push event, a signed + /// certificate and an anchor for a transition that never happened. + /// + /// MUTATION (RED): drop the `reflog_proves_landing` gate and this + /// promotes 1. + #[sqlx::test] + async fn reconcile_refuses_a_coincidental_tip_with_no_reflog_proof(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + // The ref reached A by an unrelated update: its reflog says + // `0{40} -> A`, never `B -> A`. + let on_disk_sha = seed_ref_on_bare(&bare, "refs/heads/main"); + + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + let mut row = make_row(&repo_id, "refs/heads/main", &"b".repeat(40), &on_disk_sha); + row.state = pending_state::PREPARED.to_string(); + row.applied_at = None; + state + .db + .insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + + let n = reconcile_prepared_from_disk(state.clone(), 100) + .await + .unwrap(); + assert_eq!( + n, 0, + "the current SHA is not landing proof: a row whose claimed old_sha never \ + appears in the ref's reflog must stay put" + ); + let still_pending = state + .db + .list_pending_ref_transitions_prepared_or_uncertain(100) + .await + .unwrap(); + assert_eq!(still_pending.len(), 1, "the row is left where it was"); + let applied = state + .db + .list_pending_ref_transitions_applied(100) + .await + .unwrap(); + assert!(applied.is_empty(), "the drain must never see it"); + } + + /// The positive control for the test above: the SAME on-disk SHA, + /// but a row whose transition the reflog actually records (the + /// `0{40} -> A` entry that created the ref). Proof present, so the + /// row promotes — the strict gate must not break real recovery. + #[sqlx::test] + async fn reconcile_promotes_a_row_the_reflog_proves(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + let on_disk_sha = seed_ref_on_bare(&bare, "refs/heads/main"); + + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + let mut row = make_row(&repo_id, "refs/heads/main", &"0".repeat(40), &on_disk_sha); + row.state = pending_state::PREPARED.to_string(); + row.applied_at = None; + // #26 Split PR 1 step 5 — seed parent + marker for the gate. + seed_parent_request(&state.db, &row.request_id, &repo_id, vec![0x22; 32]).await; + stage_marker(&bare, &row.request_id, &[0x22; 32]).await; + state + .db + .insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + + let n = reconcile_prepared_from_disk(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n, 1, "a transition the reflog records is promoted"); + } + + /// A repo that keeps no reflog (created before `init_bare` enabled + /// `core.logAllRefUpdates`) can produce no proof, and no proof means + /// no promotion — never a fallback to the SHA-only guess. + #[sqlx::test] + async fn reconcile_leaves_a_row_prepared_when_the_repo_keeps_no_reflog(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + let on_disk_sha = seed_ref_on_bare(&bare, "refs/heads/main"); + // Model a legacy repo: throw the reflogs away after the fact. + std::fs::remove_dir_all(bare.join("logs")).expect("remove logs"); + + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + let mut row = make_row(&repo_id, "refs/heads/main", &"0".repeat(40), &on_disk_sha); + row.state = pending_state::PREPARED.to_string(); + row.applied_at = None; + state + .db + .insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + + let n = reconcile_prepared_from_disk(state.clone(), 100) + .await + .unwrap(); + assert_eq!( + n, 0, + "absence of evidence is not evidence: without a reflog the row waits for \ + human-attended recovery" + ); + } + + /// A reflog entry that PREDATES the row cannot be proof of that + /// row's landing: it is the signature of an earlier push that moved + /// the same pair. The SHA matches and the row is fresh, so only the + /// timestamp floor refuses it. + #[sqlx::test] + async fn reconcile_refuses_a_reflog_entry_older_than_the_row(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + let on_disk_sha = seed_ref_on_bare(&bare, "refs/heads/main"); + + // Rewrite the entry's timestamp to an hour back — far outside + // REFLOG_CLOCK_SKEW, which only covers git's whole-second + // truncation. + let log_path = bare.join("logs/refs/heads/main"); + let raw = std::fs::read_to_string(&log_path).expect("reflog exists"); + let old_ts = (chrono::Utc::now() - chrono::Duration::hours(1)).timestamp(); + let rewritten: String = raw + .lines() + .map(|line| { + let (header, msg) = line.split_once('\t').unwrap_or((line, "")); + let mut tokens: Vec = + header.split_whitespace().map(|s| s.to_string()).collect(); + let n = tokens.len(); + tokens[n - 2] = old_ts.to_string(); + format!("{}\t{}\n", tokens.join(" "), msg) + }) + .collect(); + std::fs::write(&log_path, rewritten).expect("rewrite reflog"); + + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + let mut row = make_row(&repo_id, "refs/heads/main", &"0".repeat(40), &on_disk_sha); + row.state = pending_state::PREPARED.to_string(); + row.applied_at = None; + state + .db + .insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + + let n = reconcile_prepared_from_disk(state.clone(), 100) + .await + .unwrap(); + assert_eq!( + n, 0, + "proof must postdate the intent it proves, or a row inherits an older \ + push's reflog entry" + ); + } + + /// The reflog gate must NOT break the deletion recovery above it. A + /// deleted ref takes its reflog with it, so a landed + /// `git push :branch` can never produce reflog proof; absence of the + /// ref plus the age window is the whole evidence set for one, and + /// the gate exempts deletions for exactly that reason. + /// + /// MUTATION (RED): drop the `!is_deletion` guard on the reflog check + /// and a landed deletion stops being recoverable again. + #[sqlx::test] + async fn reconcile_still_promotes_a_landed_deletion_which_can_have_no_reflog( + pool: sqlx::PgPool, + ) { + use std::process::Command; + let state = crate::test_support::test_state(pool).await; + + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + let doomed_sha = seed_ref_on_bare(&bare, "refs/heads/doomed"); + // The push landed: the branch is gone, and so is its reflog. + let out = Command::new("git") + .args(["update-ref", "-d", "refs/heads/doomed"]) + .current_dir(&bare) + .stdin(std::process::Stdio::null()) + .output() + .expect("git update-ref -d"); + assert!( + out.status.success(), + "update-ref -d failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + let mut row = make_row(&repo_id, "refs/heads/doomed", &doomed_sha, ZERO_SHA); + row.state = pending_state::PREPARED.to_string(); + row.applied_at = None; + // #26 Split PR 1 step 5 — seed parent + marker for the gate. + seed_parent_request(&state.db, &row.request_id, &repo_id, vec![0x33; 32]).await; + stage_marker(&bare, &row.request_id, &[0x33; 32]).await; + state + .db + .insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + + let n = reconcile_prepared_from_disk(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n, 1, "a landed branch delete is still recovered"); + let applied = state + .db + .list_pending_ref_transitions_applied(100) + .await + .unwrap(); + assert_eq!(applied.len(), 1, "the deletion row reaches the drain"); + assert_eq!(applied[0].id, row.id); + } + + // ----- P2 (reviewer round 3): the multi-pass reconcile must WALK ----- + + /// The backlog past the first page is reconciled in the SAME + /// startup, not one page per restart. With a per-pass limit of ONE, + /// a single pass can promote at most one row, so anything above one + /// proves the loop advanced. + /// + /// MUTATION (RED): call the single-page `reconcile_prepared_from_disk` + /// and only the first row is promoted. + #[sqlx::test] + async fn reconcile_all_walks_the_backlog_past_the_first_page(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + + // Three landed refs, each with reflog proof of its own creation. + let ref_names = ["refs/heads/one", "refs/heads/two", "refs/heads/three"]; + for (i, ref_name) in ref_names.iter().enumerate() { + let sha = seed_ref_on_bare(&bare, ref_name); + let mut row = make_row(&repo_id, ref_name, &"0".repeat(40), &sha); + row.request_id = format!("req-backlog-{i}"); + row.state = pending_state::PREPARED.to_string(); + row.applied_at = None; + row.id = crate::db::deterministic_id(&[ + "pending_ref_transition", + &row.request_id, + &row.repo_id, + &row.ref_name, + &row.old_sha, + &row.new_sha, + ]); + // #26 Split PR 1 step 5 — seed parent + marker for the gate. + seed_parent_request(&state.db, &row.request_id, &repo_id, vec![0x44; 32]).await; + stage_marker(&bare, &row.request_id, &[0x44; 32]).await; + state + .db + .insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + } + + let promoted = reconcile_prepared_from_disk_all(state.clone(), 1, DRAIN_MAX_PASSES) + .await + .unwrap(); + assert_eq!( + promoted, + ref_names.len(), + "every row is reconciled in ONE startup, not one page per restart" + ); + let still_pending = state + .db + .list_pending_ref_transitions_prepared_or_uncertain(100) + .await + .unwrap(); + assert!( + still_pending.is_empty(), + "no backlog is left stranded past the first page" + ); + } + + /// The walk must step OVER rows it cannot promote. Those rows stay + /// `prepared` by design, so a pass that re-queried from the start + /// would hand itself the same page forever and never reach the + /// provable rows behind them. + /// + /// The blocker here is the class the reflog gate introduces: a ref + /// that really is on disk at the row's `new_sha`, in a repo that + /// keeps no reflog for it — permanently unprovable, so it jams page + /// one on every startup for as long as it exists, not just once. + /// + /// MUTATION (RED): ignore the cursor when selecting the next page + /// and the provable row behind the blocker is never promoted. + #[sqlx::test] + async fn reconcile_all_advances_past_an_unprovable_row(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + + // Row 1 (oldest, so it sorts first): the ref IS on disk at the + // row's new_sha, but its reflog is gone — the SHA matches, the + // age passes, and the landing is still unproven. + let legacy_sha = seed_ref_on_bare(&bare, "refs/heads/legacy"); + std::fs::remove_file(bare.join("logs/refs/heads/legacy")).expect("drop the ref's reflog"); + let mut blocker = make_row(&repo_id, "refs/heads/legacy", &"0".repeat(40), &legacy_sha); + blocker.request_id = "req-blocker".to_string(); + blocker.state = pending_state::PREPARED.to_string(); + blocker.applied_at = None; + blocker.created_at = (chrono::Utc::now() - chrono::Duration::minutes(5)).to_rfc3339(); + blocker.id = crate::db::deterministic_id(&["pending_ref_transition", "req-blocker"]); + state + .db + .insert_pending_ref_transition_for_test(&blocker) + .await + .unwrap(); + + // Row 2 (newer): a provable landing sitting behind it. + let sha = seed_ref_on_bare(&bare, "refs/heads/landed"); + let mut good = make_row(&repo_id, "refs/heads/landed", &"0".repeat(40), &sha); + good.request_id = "req-good".to_string(); + good.state = pending_state::PREPARED.to_string(); + good.applied_at = None; + good.id = crate::db::deterministic_id(&["pending_ref_transition", "req-good"]); + // #26 Split PR 1 step 5 — seed parent + marker for the gate. + seed_parent_request(&state.db, &good.request_id, &repo_id, vec![0x55; 32]).await; + stage_marker(&bare, &good.request_id, &[0x55; 32]).await; + state + .db + .insert_pending_ref_transition_for_test(&good) + .await + .unwrap(); + + let promoted = reconcile_prepared_from_disk_all(state.clone(), 1, DRAIN_MAX_PASSES) + .await + .unwrap(); + assert_eq!( + promoted, 1, + "the provable row behind a permanently unprovable one is still reached" + ); + let still_pending = state + .db + .list_pending_ref_transitions_prepared_or_uncertain(100) + .await + .unwrap(); + assert_eq!(still_pending.len(), 1, "the unprovable row is left alone"); + assert_eq!(still_pending[0].id, blocker.id); + } + + // ----- P2-A drain resilience tests ----- + // + // These tests cover the "drain must not abort on first failure" + // and "drain must not cap at 1000 requests per startup" findings. + // Backlog processing uses the production + // `drain_receive_pack_requests_all` (the `DRAIN_PER_PASS_LIMIT` / + // `DRAIN_MAX_PASSES` constants from this module) so the test + // exercises the same wrapper the startup calls. Failure isolation + // uses `drain_receive_pack_requests_with` to inject a closure + // that returns `Retry` for one request and `Done` for the next. + + #[sqlx::test] + async fn drain_processes_backlog_larger_than_one_pass(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + // Seed 1500 distinct request rows. Each request is its own + // `receive_pack_requests.id`; per-ref PKs hash the request + // id, and the certs / anchor jobs hash the request id too. + const N: usize = 1500; + for i in 0..N { + let mut row = make_row( + "repo-backlog", + "refs/heads/main", + &"0".repeat(40), + &format!("{:040x}", i as u64), + ); + row.request_id = format!("req-{i}"); + row.id = crate::db::deterministic_id(&[ + "pending_ref_transition", + &row.request_id, + &row.repo_id, + &row.ref_name, + &row.old_sha, + &row.new_sha, + ]); + let parsed_report = parsed_report_ok(&[("refs/heads/main", true)]); + stage_request_with_children( + &state.db, + &row.request_id, + "repo-backlog", + Some(row.ordinal), + std::slice::from_ref(&row), + parsed_report, + ) + .await; + } + + // Drain with the production limits. Two passes of 1000 each + // cover all 1500 requests; the third pass would be empty and + // exits the loop early on the `n < per_pass_limit` check. + let total = + drain_receive_pack_requests_all(state.clone(), DRAIN_PER_PASS_LIMIT, DRAIN_MAX_PASSES) + .await + .unwrap(); + assert_eq!(total, N, "drain processed the full backlog"); + + // No `outcomes_committed` requests remain. + let after = state.db.count_receive_pack_requests_due().await.unwrap(); + assert_eq!( + after, 0, + "every request row was processed and moved to complete" + ); + // No per-ref children remain either. + let still = state + .db + .list_pending_ref_transitions_applied(10_000) + .await + .unwrap(); + assert!(still.is_empty(), "every child was cleaned up"); + } + + #[sqlx::test] + async fn drain_continues_past_a_failing_row(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + // Seed two requests (A first so the `ORDER BY created_at ASC, + // id ASC` query hits A before B). Each request owns a single + // child row at ordinal 0. + let mut row_a = make_row( + "repo-fail-then-pass", + "refs/heads/main", + &"0".repeat(40), + &"a".repeat(40), + ); + row_a.request_id = "req-A".to_string(); + row_a.id = crate::db::deterministic_id(&[ + "pending_ref_transition", + &row_a.request_id, + &row_a.repo_id, + &row_a.ref_name, + &row_a.old_sha, + &row_a.new_sha, + ]); + let mut row_b = make_row( + "repo-fail-then-pass", + "refs/heads/main", + &"0".repeat(40), + &"b".repeat(40), + ); + row_b.request_id = "req-B".to_string(); + row_b.id = crate::db::deterministic_id(&[ + "pending_ref_transition", + &row_b.request_id, + &row_b.repo_id, + &row_b.ref_name, + &row_b.old_sha, + &row_b.new_sha, + ]); + let parsed_report = parsed_report_ok(&[("refs/heads/main", true)]); + stage_request_with_children( + &state.db, + "req-A", + "repo-fail-then-pass", + Some(0), + std::slice::from_ref(&row_a), + parsed_report.clone(), + ) + .await; + stage_request_with_children( + &state.db, + "req-B", + "repo-fail-then-pass", + Some(0), + std::slice::from_ref(&row_b), + parsed_report, + ) + .await; + + // Inject a closure that returns Retry for request A and + // delegates to the real `apply_request_effects` for B. + // Request A is moved to `effects_pending` for a future + // retry; request B is fully processed. + let state_for_closure = state.clone(); + let (processed, examined) = + drain_receive_pack_requests_with(state.clone(), 100, |_s, req_id| { + let target = String::from("req-A"); + let st = state_for_closure.clone(); + async move { + if req_id == target { + Ok(EffectsOutcome::Retry { + last_error: "injected derive failure".to_string(), + }) + } else { + apply_request_effects(&st, &req_id).await + } + } + }) + .await + .unwrap(); + assert_eq!(processed, 1, "only request B is fully processed"); + assert_eq!( + examined, 2, + "the loop examined both requests; processed/derivation is independent of pagination" + ); + + // Request A is in `effects_pending` (Retry moved it there). + let a_req = state + .db + .get_receive_pack_request("req-A") + .await + .unwrap() + .expect("req-A exists"); + assert_eq!( + a_req.state, + crate::db::request_state::EFFECTS_PENDING, + "Retry outcome moves A to effects_pending" + ); + // Request B is in `complete`. + let b_req = state + .db + .get_receive_pack_request("req-B") + .await + .unwrap() + .expect("req-B exists"); + assert_eq!( + b_req.state, + crate::db::request_state::COMPLETE, + "Done outcome moves B to complete" + ); + + // Request A's artifacts were not created (the closure + // returned Retry before any insert ran). + let a_push = state + .db + .count_push_events(&row_a.repo_id, &row_a.new_sha, &row_a.pusher_did) + .await + .unwrap(); + assert_eq!(a_push, 0, "request A's push event was not created"); + let a_anchors = state + .db + .count_anchor_jobs( + &row_a.repo_id, + &row_a.ref_name, + &row_a.old_sha, + &row_a.new_sha, + ) + .await + .unwrap(); + assert_eq!(a_anchors, 0, "request A's anchor job was not created"); + let a_cert_id = crate::db::ref_cert_id_for(&row_a.request_id, row_a.ordinal); + let a_cert = state.db.get_ref_certificate(&a_cert_id).await.unwrap(); + assert!( + a_cert.is_none(), + "request A's cert id must not exist (got {:?})", + a_cert.map(|c| c.id) + ); + + // Request B's artifacts WERE created. + let b_push = state + .db + .count_push_events(&row_b.repo_id, &row_b.new_sha, &row_b.pusher_did) + .await + .unwrap(); + assert_eq!(b_push, 1, "request B's push event was created"); + let b_anchors = state + .db + .count_anchor_jobs( + &row_b.repo_id, + &row_b.ref_name, + &row_b.old_sha, + &row_b.new_sha, + ) + .await + .unwrap(); + assert_eq!(b_anchors, 1, "request B's anchor job was created"); + let b_cert_id = crate::db::ref_cert_id_for(&row_b.request_id, row_b.ordinal); + let b_cert = state.db.get_ref_certificate(&b_cert_id).await.unwrap(); + assert!(b_cert.is_some(), "request B's cert was created"); + } + + // ----- P2-B multi-ref push event cardinality test ----- + // + // The live handler and the recovery drain must produce the same + // push event id for a multi-ref push. Under the v30 model the id + // is keyed on `(request_id, accepted_ordinal)`: the request row + // records which child landed first, and only that child writes + // the push event. Other children skip the event write but still + // produce their own certs and anchor jobs. + // + // Certs stay per-ref (one per `(repo, ref)` transition); anchor + // jobs stay per-transition (one per `(repo, ref, old, new)` tuple). + // The push event is the only artifact that is request-scoped. + + #[sqlx::test] + async fn multi_ref_push_produces_exactly_one_event_across_live_and_recovery( + pool: sqlx::PgPool, + ) { + let state = crate::test_support::test_state(pool).await; + + // Three children for the SAME `request_id`, distinct + // `ref_name`s, distinct ordinals 0/1/2. The `new_sha` is + // shared across all three because this models a push that + // advanced a tip commit onto three refs at once (the + // ordinary shape of `git push --all`). + let shared_new_sha = "c".repeat(40); + let ref_names = [ + "refs/heads/main", + "refs/heads/feature-a", + "refs/heads/feature-b", + ]; + let mut children = Vec::new(); + for (i, ref_name) in ref_names.iter().enumerate() { + let mut row = make_row("repo-multi", ref_name, &"0".repeat(40), &shared_new_sha); + row.request_id = "req-multi".to_string(); + row.ordinal = i as i32; + // Vary `old_sha` per row so the anchor job PKs (which + // hash `old_sha`) don't collide. + row.old_sha = format!("{:040x}", (i + 1) as u64); + row.id = crate::db::deterministic_id(&[ + "pending_ref_transition", + &row.request_id, + &row.repo_id, + &row.ref_name, + &row.old_sha, + &row.new_sha, + ]); + children.push(row); + } + // Stage the request with `accepted_ordinal = Some(0)` so the + // first child is the one whose `new_sha` becomes the push + // event's commit_hash. All three refs are in the parsed + // report's ok set. + let parsed_report = parsed_report_ok(&[ + ("refs/heads/main", true), + ("refs/heads/feature-a", true), + ("refs/heads/feature-b", true), + ]); + stage_request_with_children( + &state.db, + "req-multi", + "repo-multi", + Some(0), + &children, + parsed_report, + ) + .await; + + // Drain the request. + let (n, examined) = drain_receive_pack_requests(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n, 1, "the request was processed"); + assert_eq!(examined, 1, "the loop examined the single request"); + + // Exactly one push event row, keyed on the deterministic + // (request_id, accepted_ordinal) id. + let push_count = state.db.get_push_count("did:key:z6pusher").await.unwrap(); + assert_eq!( + push_count, 1, + "exactly one push event for a multi-ref push (trust-score predicate)" + ); + let events = state + .db + .count_push_events("repo-multi", &shared_new_sha, "did:key:z6pusher") + .await + .unwrap(); + assert_eq!(events, 1, "exactly one push_events row"); + + // The deterministic id is the one the live path would have + // written. + let expected_id = crate::db::push_event_id_for("req-multi", 0); + assert_eq!( + expected_id, + crate::db::push_event_id_for("req-multi", 0), + "push_event_id_for is deterministic" + ); + + // Certs: one per ref (the cert contract is per-ref, NOT + // collapsed by ordinal). Three children → three certs. + let certs = state + .db + .list_ref_certificates("repo-multi", 10) + .await + .unwrap(); + assert_eq!( + certs.len(), + 3, + "one cert per ref transition (not collapsed by accepted_ordinal)" + ); + + // Anchor jobs: one per `(repo, ref, old, new)` transition. + // Three children → three anchor jobs. + for (i, ref_name) in ref_names.iter().enumerate() { + let n = state + .db + .count_anchor_jobs( + "repo-multi", + ref_name, + &format!("{:040x}", (i + 1) as u64), + &shared_new_sha, + ) + .await + .unwrap(); + assert_eq!(n, 1, "one anchor job per transition"); + } + } + + // ----- P2 (reviewer-1 round 2): distinct new_shas across refs ----- + // + // The previous multi-ref test shared one `new_sha` across all + // refs; that masked the wrong-hash bug. This test gives every ref + // a distinct `new_sha` and asserts the persisted `commit_hash` is + // the FIRST ref's `new_sha`. The live handler derives + // `accepted_ordinal` from `ref_updates.first()`'s position and + // uses that ordinal's new_sha for the push event. Without the + // `row.ordinal == request.accepted_ordinal` gate the drain would + // `record_push_with_id` for whichever row the `ORDER BY + // applied_at, id` query returned first, leaving the wrong hash + // for any other drain order. + #[sqlx::test] + async fn multi_ref_recovery_uses_first_refs_new_sha_for_push_event(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + // Three refs, each with a distinct `new_sha` modelling a + // multi-branch push where each ref advanced to a different + // tip. The first ref's new_sha is the one the live handler + // would have used. + let first_new_sha = "a".repeat(40); + let second_new_sha = "b".repeat(40); + let third_new_sha = "c".repeat(40); + let ref_names = [ + "refs/heads/main", + "refs/heads/feature-a", + "refs/heads/feature-b", + ]; + let new_shas = [&first_new_sha, &second_new_sha, &third_new_sha]; + let mut children = Vec::new(); + for (i, (ref_name, new_sha)) in ref_names.iter().zip(new_shas.iter()).enumerate() { + let mut row = make_row("repo-multi-distinct", ref_name, &"0".repeat(40), new_sha); + row.request_id = "req-multi-distinct".to_string(); + row.ordinal = i as i32; + // Vary `old_sha` per row so the anchor job PKs don't + // collide and so the certs distinguish the three + // transitions. + row.old_sha = format!("{:040x}", (i + 1) as u64); + row.id = crate::db::deterministic_id(&[ + "pending_ref_transition", + &row.request_id, + &row.repo_id, + &row.ref_name, + &row.old_sha, + &row.new_sha, + ]); + children.push(row); + } + let parsed_report = parsed_report_ok(&[ + ("refs/heads/main", true), + ("refs/heads/feature-a", true), + ("refs/heads/feature-b", true), + ]); + stage_request_with_children( + &state.db, + "req-multi-distinct", + "repo-multi-distinct", + Some(0), + &children, + parsed_report, + ) + .await; + + // Drain the request. + let (n, examined) = drain_receive_pack_requests(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n, 1, "the request was processed"); + assert_eq!(examined, 1, "the loop examined the single request"); + + // Exactly one push event row, keyed on the deterministic + // (request_id, accepted_ordinal) id. The persisted + // `commit_hash` is the FIRST ref's `new_sha` — the same + // value the live path would have written. + let push_count = state.db.get_push_count("did:key:z6pusher").await.unwrap(); + assert_eq!(push_count, 1, "exactly one push event row"); + let first_event = state + .db + .count_push_events("repo-multi-distinct", &first_new_sha, "did:key:z6pusher") + .await + .unwrap(); + assert_eq!( + first_event, 1, + "the persisted commit_hash is the FIRST ref's new_sha" + ); + // The non-first new_shas MUST NOT have a push event + // pointing at them — that would be the wrong-hash bug. + for other in [&second_new_sha, &third_new_sha] { + let n = state + .db + .count_push_events("repo-multi-distinct", other, "did:key:z6pusher") + .await + .unwrap(); + assert_eq!( + n, 0, + "no push event for the non-first ref's new_sha ({other})" + ); + } + + // Certs stay per-ref (three refs → three certs). + let certs = state + .db + .list_ref_certificates("repo-multi-distinct", 10) + .await + .unwrap(); + assert_eq!(certs.len(), 3, "one cert per ref transition"); + } + + // ----- P2-D (reviewer-2 round 2): all-fail batch does not early-exit ----- + // + // The previous loop's exit condition was `(n as i64) < per_pass_limit` + // where `n` was rows *fully processed* (derive + delete). A pass + // where every `apply_request_effects` returns Retry logs each + // failure but increments `processed = 0`; the outer loop sees + // `0 < per_pass_limit` and returns. Remaining requests are never + // attempted that boot. The fix returns `(processed, examined)` + // and keys the exit on `examined`. This test seeds + // `per_pass_limit` requests with a closure that retries every + // one, then asserts the drain ran every request (processed=0, + // examined=per_pass_limit) so the outer loop continues to the + // next pass. + #[sqlx::test] + async fn drain_does_not_exit_early_when_every_row_fails(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + const N: usize = 5; + let parsed_report = parsed_report_ok(&[("refs/heads/main", true)]); + for i in 0..N { + let mut row = make_row( + "repo-all-fail", + "refs/heads/main", + &"0".repeat(40), + &format!("{:040x}", i as u64), + ); + row.request_id = format!("req-all-fail-{i}"); + row.id = crate::db::deterministic_id(&[ + "pending_ref_transition", + &row.request_id, + &row.repo_id, + &row.ref_name, + &row.old_sha, + &row.new_sha, + ]); + stage_request_with_children( + &state.db, + &row.request_id, + "repo-all-fail", + Some(row.ordinal), + std::slice::from_ref(&row), + parsed_report.clone(), + ) + .await; + } + + let (processed, examined) = + drain_receive_pack_requests_with(state.clone(), N as i64, |_s, _req_id| async move { + Ok(EffectsOutcome::Retry { + last_error: "injected: every request fails".to_string(), + }) + }) + .await + .unwrap(); + assert_eq!(processed, 0, "no request was fully processed"); + assert_eq!( + examined, N, + "the loop examined every request even though every derive failed" + ); + + // Every request is in `effects_pending` for a future retry. + let due = state.db.count_receive_pack_requests_due().await.unwrap(); + // The Retry path sets `next_attempt_at` 60s in the future, + // so the due count is 0 — but the requests still exist. + assert_eq!(due, 0, "Retry schedules the requests 60s out"); + // And there are N total outcomes_committed/effects_pending. + let total: i64 = sqlx::query_scalar( + r#"SELECT COUNT(*)::BIGINT FROM receive_pack_requests + WHERE state IN ($1, $2)"#, + ) + .bind(crate::db::request_state::OUTCOMES_COMMITTED) + .bind(crate::db::request_state::EFFECTS_PENDING) + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert_eq!( + total, N as i64, + "all N requests are still pending for the next startup" + ); + } + + // ----- P1 (reviewer-1 round 2): recovery refreshes a stale cert ----- + // + // The crash window the reviewer named: the live cert was issued + // before the push actually landed on disk (e.g. cert was emitted + // at t1, the apply succeeded at t2, the live upsert never re-ran + // because the handler errored after the cert write). A second + // startup runs the recovery drain, which must update the cert's + // `old_sha` / `new_sha` / `pusher_did` / `signature` / + // `issued_at` to the new transition. The `id` (deterministic + // from `(request_id, ref_name)`) is preserved — the upsert only + // touches the SHAs/did/signature/ts columns. Without the upsert + // the cert stays at the old transition and consumers reading + // `ref_certificates.new_sha` see a value that does not match + // the ref on disk. + + #[sqlx::test] + async fn recovery_refreshes_stale_cert_to_landed_transition(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + // Seed a repo so the cert FK is satisfied. + let owner_did = "did:key:zCertOwner"; + let rec = crate::db::RepoRecord { + id: "repo-cert-refresh".to_string(), + name: "cert-refresh".to_string(), + owner_did: owner_did.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + disk_path: "/tmp/cert-refresh".to_string(), + forked_from: None, + machine_id: None, + }; + state.db.create_repo(&rec).await.unwrap(); + + // Insert a STALE cert directly: old SHA → some "stale new" SHA + // at t1, with a different pusher DID. This models the live + // cert issued before the push landed. The cert id is + // deterministic on `(request_id, ordinal)`; the recovery row + // is a single child at ordinal 0. + let stale_cert_id = crate::db::ref_cert_id_for("req-stale", 0); + let stale_old = "0".repeat(40); + let stale_new = "1".repeat(40); + let stale_pusher = "did:key:zStalePusher"; + let stale_issued = (chrono::Utc::now() - chrono::Duration::seconds(60)).to_rfc3339(); + state + .db + .insert_ref_certificate_idempotent(&crate::db::RefCertificate { + id: stale_cert_id.clone(), + repo_id: rec.id.clone(), + ref_name: "refs/heads/main".to_string(), + old_sha: stale_old.clone(), + new_sha: stale_new.clone(), + pusher_did: stale_pusher.to_string(), + node_did: state.node_did.to_string(), + signature: "stale-signature".to_string(), + issued_at: stale_issued.clone(), + }) + .await + .unwrap(); + + // Seed the durable child with the LANDED transition (what + // the push actually applied to disk): a different old_sha + // and new_sha, the genuine pusher DID. The drain must + // refresh the stale cert to this transition. + let landed_old = "2".repeat(40); + let landed_new = "3".repeat(40); + let landed_pusher = "did:key:zLandedPusher"; + let mut row = make_row(&rec.id, "refs/heads/main", &landed_old, &landed_new); + row.request_id = "req-stale".to_string(); + row.pusher_did = landed_pusher.to_string(); + row.ordinal = 0; + row.id = crate::db::deterministic_id(&[ + "pending_ref_transition", + &row.request_id, + &row.repo_id, + &row.ref_name, + &row.old_sha, + &row.new_sha, + ]); + let parsed_report = parsed_report_ok(&[("refs/heads/main", true)]); + stage_request_with_pusher( + &state.db, + "req-stale", + &rec.id, + landed_pusher, + Some(0), + std::slice::from_ref(&row), + parsed_report, + ) + .await; + + // Drain. The recovery upsert must overwrite the stale cert + // with the landed transition's SHAs / pusher / signature. + let (processed, examined) = drain_receive_pack_requests(state.clone(), 100) + .await + .unwrap(); + assert_eq!(processed, 1, "the request was drained"); + assert_eq!(examined, 1, "the loop examined the single request"); + + let certs = state.db.list_ref_certificates(&rec.id, 10).await.unwrap(); + assert_eq!(certs.len(), 1, "exactly one cert row, the same id"); + let cert = &certs[0]; + assert_eq!(cert.id, stale_cert_id, "deterministic id preserved"); + assert_eq!( + cert.old_sha, landed_old, + "old_sha refreshed to the landed transition" + ); + assert_eq!( + cert.new_sha, landed_new, + "new_sha refreshed to the landed transition (was the bug)" + ); + assert_eq!( + cert.pusher_did, landed_pusher, + "pusher refreshed to the actual landed pusher" + ); + assert_ne!( + cert.signature, "stale-signature", + "signature was re-signed with the landed transition" + ); + // `issued_at` is a free-form string; just assert the row is + // populated. The monotonic `issued_at > stale_issued` is what + // the upsert's CASE WHEN checks. + assert!(!cert.issued_at.is_empty(), "issued_at populated"); + } + + // ----- P1 round 4: A → B → restart replay test ----- + // + // The reviewer's invariant: a recovery replay of A's row after a + // later live cert B has been written must NOT overwrite B's + // fields. Without `issued_at_override`, the recovery's + // `Utc::now()` is later than B's live `Utc::now()` (because the + // replay happens after B's live write), and the + // `EXCLUDED.issued_at > ref_certificates.issued_at` upsert guard + // would let A's stale transition clobber B's fresh cert. + // + // The fix stamps the recovery cert's `issued_at` with the row's + // `created_at`, which carries the original transition time and + // is earlier than B's `Utc::now()`. This test pins that the + // replay does not outrank B. + #[sqlx::test] + async fn replay_of_stale_row_does_not_overwrite_live_cert_b(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let owner_did = "did:key:z6Mkreplay"; + let rec = crate::db::RepoRecord { + id: "repo-replay".to_string(), + name: "replay".to_string(), + owner_did: owner_did.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + disk_path: "/tmp/replay".to_string(), + forked_from: None, + machine_id: None, + }; + state.db.create_repo(&rec).await.unwrap(); + + // A: original push, request row still in `outcomes_committed`. + let a_old = "0".repeat(40); + let a_new = "1".repeat(40); + let a_pusher = "did:key:zA"; + let a_request = "req-A"; + let mut a_row = make_row(&rec.id, "refs/heads/main", &a_old, &a_new); + a_row.request_id = a_request.to_string(); + a_row.pusher_did = a_pusher.to_string(); + a_row.ordinal = 0; + a_row.id = crate::db::deterministic_id(&[ + "pending_ref_transition", + &a_row.request_id, + &a_row.repo_id, + &a_row.ref_name, + &a_row.old_sha, + &a_row.new_sha, + ]); + // Backdate A's created_at by 5 minutes so the replay's + // stamped `issued_at` is provably older than B's live one. + a_row.created_at = (chrono::Utc::now() - chrono::Duration::minutes(5)).to_rfc3339(); + let parsed_report = parsed_report_ok(&[("refs/heads/main", true)]); + stage_request_with_pusher( + &state.db, + a_request, + &rec.id, + a_pusher, + Some(0), + std::slice::from_ref(&a_row), + parsed_report, + ) + .await; + + // A's cert was written live (or never — we test the case + // where the row was left pending and the cert was NOT yet + // written, then B's live push arrives first and writes its + // cert, then A's drain replays). + // + // Simulate: the live cert B has been written by a later + // push. + let b_old = a_new.clone(); + let b_new = "2".repeat(40); + let b_pusher = "did:key:zB"; + // B is a stand-in for "a later live push already wrote its + // cert". The cert id is arbitrary — what matters is the + // row collides with A's recovery on the + // `(repo_id, ref_name)` unique index. Use B's + // request-scoped id at ordinal 0 so the id is a real + // `(request_id, ordinal)` shape. + let b_cert_id = crate::db::ref_cert_id_for("req-B", 0); + state + .db + .insert_ref_certificate(&crate::db::RefCertificate { + id: b_cert_id.clone(), + repo_id: rec.id.clone(), + ref_name: "refs/heads/main".to_string(), + old_sha: b_old.clone(), + new_sha: b_new.clone(), + pusher_did: b_pusher.to_string(), + node_did: state.node_did.to_string(), + signature: "b-live-signature".to_string(), + issued_at: chrono::Utc::now().to_rfc3339(), + }) + .await + .unwrap(); + + // Drain A's replay. The upsert sees A's `issued_at` (A's + // created_at = now-5min) is OLDER than B's cert (now), so + // the per-column CASE WHEN guards must NOT update B's + // fields. + let (processed, examined) = drain_receive_pack_requests(state.clone(), 100) + .await + .unwrap(); + assert_eq!(processed, 1, "A's request was drained"); + assert_eq!(examined, 1, "the loop examined A's request"); + + let certs = state.db.list_ref_certificates(&rec.id, 10).await.unwrap(); + assert_eq!(certs.len(), 1, "exactly one cert row remains"); + let cert = &certs[0]; + assert_eq!( + cert.old_sha, b_old, + "old_sha stays at B's; A's replay (now-5min) must not outrank B's (now)" + ); + assert_eq!( + cert.new_sha, b_new, + "new_sha stays at B's; A's replay must not outrank B's" + ); + assert_eq!( + cert.pusher_did, b_pusher, + "pusher stays at B's; A's replay must not outrank B's" + ); + assert_eq!( + cert.signature, "b-live-signature", + "signature stays at B's live signature; A's replay must not outrank B's" + ); + } + + // #26 Split PR 1 step 4 — bounded retirement. The periodic + // purge task deletes terminal `complete` / `rejected_at_git` + // rows and their children older than the retention window. + // The tests below pin the contract: + // + // 1. Only `complete` / `rejected_at_git` rows are eligible. + // 2. Only rows with `completed_at < now - retention` are eligible. + // 3. Children are purged after their parent request. + // 4. `quarantined` (not yet a state) and non-terminal states are NEVER purged. + // 5. Idempotent: a second purge with no new eligible rows returns (0, 0). + + /// Helper: insert a request row with the given state and `completed_at`. + /// Returns the request id. + async fn stage_request_for_purge( + pool: &sqlx::PgPool, + request_id: &str, + state: &str, + completed_at: Option<&str>, + ) { + let now = chrono::Utc::now().to_rfc3339(); + let created_at = now.clone(); + let bytes = b"purge-test".to_vec(); + sqlx::query( + r#"INSERT INTO receive_pack_requests + (id, repo_id, pusher_did, node_did, request_bytes, request_bytes_hash, + state, git_exit_ok, parsed_report, accepted_ordinal, attempt_count, + last_error, next_attempt_at, created_at, completed_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)"#, + ) + .bind(request_id) + .bind("purge-test-repo") + .bind("did:key:zPurgeTester") + .bind("did:key:zPurgeNode") + .bind(&bytes) + .bind(vec![0u8; 32]) + .bind(state) + .bind(Some(true)) + .bind(None::) + .bind(Some(0_i32)) + .bind(0_i32) + .bind(None::) + .bind(None::) + .bind(&created_at) + .bind(completed_at) + .execute(pool) + .await + .expect("insert request for purge test"); + } + + #[sqlx::test] + async fn purge_deletes_only_old_complete_and_rejected_at_git(pool: sqlx::PgPool) { + let db = _db(pool.clone()).await; + // 8 days ago, well past the 7-day retention. + let old = (chrono::Utc::now() - chrono::Duration::days(8)).to_rfc3339(); + // 1 day ago, inside the window. + let fresh = (chrono::Utc::now() - chrono::Duration::days(1)).to_rfc3339(); + let ids_old: Vec = vec![ + "r-old-complete".into(), + "r-old-rejected".into(), + "r-fresh-complete".into(), + "r-fresh-rejected".into(), + "r-old-received".into(), + "r-old-outcomes".into(), + "r-old-effects-pending".into(), + "r-old-no-completion".into(), + ]; + stage_request_for_purge(&pool, "r-old-complete", request_state::COMPLETE, Some(&old)).await; + stage_request_for_purge( + &pool, + "r-old-rejected", + request_state::REJECTED_AT_GIT, + Some(&old), + ) + .await; + stage_request_for_purge( + &pool, + "r-fresh-complete", + request_state::COMPLETE, + Some(&fresh), + ) + .await; + stage_request_for_purge( + &pool, + "r-fresh-rejected", + request_state::REJECTED_AT_GIT, + Some(&fresh), + ) + .await; + // Non-terminal states: never purged even when old. + stage_request_for_purge(&pool, "r-old-received", request_state::RECEIVED, Some(&old)).await; + stage_request_for_purge( + &pool, + "r-old-outcomes", + request_state::OUTCOMES_COMMITTED, + Some(&old), + ) + .await; + stage_request_for_purge( + &pool, + "r-old-effects-pending", + request_state::EFFECTS_PENDING, + Some(&old), + ) + .await; + // A request with no completed_at: never purged (NULL is excluded by the WHERE). + stage_request_for_purge(&pool, "r-old-no-completion", request_state::COMPLETE, None).await; + + let (reqs, _children) = purge_request_queue(&db, 7, 100).await.unwrap(); + assert_eq!(reqs, 2, "exactly the two old terminal rows"); + + // Verify which ids survived by re-reading each one directly. + for id in &ids_old { + let after = db.get_receive_pack_request(id).await.unwrap(); + let expected_deleted = matches!(id.as_str(), "r-old-complete" | "r-old-rejected"); + if expected_deleted { + assert!(after.is_none(), "{id} should have been purged"); + } else { + assert!(after.is_some(), "{id} should have been retained"); + } + } + } + + #[sqlx::test] + async fn purge_idempotent_returns_zero_on_second_call(pool: sqlx::PgPool) { + let db = _db(pool.clone()).await; + let old = (chrono::Utc::now() - chrono::Duration::days(8)).to_rfc3339(); + stage_request_for_purge(&pool, "r-once", request_state::COMPLETE, Some(&old)).await; + + let (a, _) = purge_request_queue(&db, 7, 100).await.unwrap(); + assert_eq!(a, 1); + let (b, _) = purge_request_queue(&db, 7, 100).await.unwrap(); + assert_eq!(b, 0, "second pass has nothing left to delete"); + } + + #[sqlx::test] + async fn purge_retention_window_pins_at_7_days(pool: sqlx::PgPool) { + // The spec calls for a 7-day window. The CLI's `1..=365` range + // guarantees a non-zero window, so we don't test retention = 0 + // here — that path is not exposed to operators. This test pins + // the invariant: a row with completed_at = now is INSIDE the + // 7-day window and is NOT purged. + let db = _db(pool.clone()).await; + let now_iso = chrono::Utc::now().to_rfc3339(); + stage_request_for_purge(&pool, "r-now", request_state::COMPLETE, Some(&now_iso)).await; + + let (n, _) = purge_request_queue(&db, 7, 100).await.unwrap(); + assert_eq!( + n, 0, + "row with completed_at = now is inside the 7-day window" + ); + + let after = db.get_receive_pack_request("r-now").await.unwrap(); + assert!(after.is_some(), "r-now must survive the 7-day window"); + } + + // ----- #26 Split PR 1 step 5 — failure-matrix tests ----- + // + // The mark gate (`reconcile_prepared_page`'s third barrier) + // quarantines a request whose on-disk marker is missing or + // hash-mismatched, and the drain's `effects_max_attempts` + // bound quarantines a request that retries past the bound. + // These tests pin each cell of that matrix. + + /// The marker is absent (the live handler never wrote it, or a + /// cleanup ran): reconcile quarantines the request and cancels + /// the child. The `reconcile_prepared_from_disk` return value + /// is the count of PROMOTED rows, so an absent marker means + /// the row is not promoted (the gate quarantined the parent + /// before the child could reach `applied`). + #[sqlx::test] + async fn cell_marker_missing_quarantines_request(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + let on_disk_sha = seed_ref_on_bare(&bare, "refs/heads/main"); + + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + // Seed the parent receive_pack_requests row WITHOUT calling + // stage_marker — that's the "missing" half of this cell. + seed_parent_request(&state.db, "req-marker-missing", &repo_id, vec![0xa1; 32]).await; + let mut row = make_row(&repo_id, "refs/heads/main", &"0".repeat(40), &on_disk_sha); + row.request_id = "req-marker-missing".to_string(); + row.state = pending_state::PREPARED.to_string(); + row.applied_at = None; + state + .db + .insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + + let n = reconcile_prepared_from_disk(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n, 0, "no row promoted when the marker is missing"); + + // Parent quarantined. + let parent = state + .db + .get_receive_pack_request("req-marker-missing") + .await + .unwrap() + .expect("parent row exists"); + assert_eq!( + parent.state, + request_state::QUARANTINED, + "missing marker quarantines the request" + ); + + // Child cancelled. + let child = state + .db + .list_pending_ref_transitions_for_request("req-marker-missing") + .await + .unwrap(); + assert_eq!(child.len(), 1, "the child exists"); + assert_eq!( + child[0].state, + pending_state::CANCELLED, + "missing marker cancels the child" + ); + assert!(child[0].cancelled_at.is_some(), "cancelled_at is stamped"); + } + + /// The marker is present but the value mismatches the parent's + /// `request_bytes_hash`. Reconcile quarantines the request and + /// stamps `last_error` with the mismatch reason. + #[sqlx::test] + async fn cell_marker_hash_mismatch_quarantines_request(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + let on_disk_sha = seed_ref_on_bare(&bare, "refs/heads/main"); + + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + seed_parent_request(&state.db, "req-marker-mismatch", &repo_id, vec![0xa2; 32]).await; + // Stage a marker with a WRONG hex — all zeros — that does + // not match the parent's hash. The reconcile's read_ref + // comparison will see the mismatch and quarantine. + stage_marker(&bare, "req-marker-mismatch", &[0x00; 32]).await; + + let mut row = make_row(&repo_id, "refs/heads/main", &"0".repeat(40), &on_disk_sha); + row.request_id = "req-marker-mismatch".to_string(); + row.state = pending_state::PREPARED.to_string(); + row.applied_at = None; + state + .db + .insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + + let n = reconcile_prepared_from_disk(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n, 0, "no row promoted when the marker mismatches"); + + let parent = state + .db + .get_receive_pack_request("req-marker-mismatch") + .await + .unwrap() + .expect("parent row exists"); + assert_eq!( + parent.state, + request_state::QUARANTINED, + "mismatched marker quarantines the request" + ); + assert_eq!( + parent.last_error.as_deref(), + Some("marker hash mismatch"), + "last_error names the mismatch reason" + ); + } + + /// Happy path: marker is present and the value matches the + /// parent's `request_bytes_hash`. The row promotes to + /// `applied`; the parent stays in its current state (the + /// handler flips it later, after `outcomes_committed` writes). + #[sqlx::test] + async fn cell_marker_present_promotes_request(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + let on_disk_sha = seed_ref_on_bare(&bare, "refs/heads/main"); + + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + seed_parent_request(&state.db, "req-marker-ok", &repo_id, vec![0xa3; 32]).await; + stage_marker(&bare, "req-marker-ok", &[0xa3; 32]).await; + + let mut row = make_row(&repo_id, "refs/heads/main", &"0".repeat(40), &on_disk_sha); + row.request_id = "req-marker-ok".to_string(); + row.state = pending_state::PREPARED.to_string(); + row.applied_at = None; + state + .db + .insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + + let n = reconcile_prepared_from_disk(state.clone(), 100) + .await + .unwrap(); + assert_eq!(n, 1, "marker ok: row is promoted to applied"); + + let applied = state + .db + .list_pending_ref_transitions_applied(100) + .await + .unwrap(); + assert_eq!(applied.len(), 1, "the row is in applied"); + assert_eq!(applied[0].id, row.id); + + // The parent is NOT touched by the reconcile (it stays in + // `received` for the live handler to flip later). + let parent = state + .db + .get_receive_pack_request("req-marker-ok") + .await + .unwrap() + .expect("parent row exists"); + assert_eq!( + parent.state, + request_state::RECEIVED, + "reconcile leaves the parent in its current state" + ); + } + + /// Drain's `EffectsOutcome::Retry` arm flips to `quarantined` + /// once `attempt_count + 1 > effects_max_attempts`. With bound + /// = 2 and `attempt_count` = 2, the next retry puts the row + /// over the bound. + #[sqlx::test] + async fn cell_retry_stuck_request_goes_to_quarantined(pool: sqlx::PgPool) { + // Lower the bound so the test exercises the over-bound path. + // `test_state_with` builds the AppState with a clone of the + // config so the test can pin the bound rather than rely on the + // default. + let state = crate::test_support::test_state_with(pool, |cfg| { + cfg.effects_max_attempts = 2; + }) + .await; + + // Stage a request in `effects_pending` with attempt_count = 2. + // The drain will pick it up via list_receive_pack_requests_due, + // run the closure (returning Retry), then check the bound. + let request_id = "req-retry-stuck"; + let repo_id = "repo-retry-stuck"; + sqlx::query( + r#"INSERT INTO receive_pack_requests + (id, repo_id, pusher_did, node_did, request_bytes, request_bytes_hash, + state, git_exit_ok, parsed_report, accepted_ordinal, attempt_count, + last_error, next_attempt_at, created_at, completed_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)"#, + ) + .bind(request_id) + .bind(repo_id) + .bind("did:key:z6pusher") + .bind("did:key:z6node") + .bind(Vec::::new()) + .bind(vec![0u8; 32]) + .bind(request_state::EFFECTS_PENDING) + .bind(Some(true)) + .bind(Some( + serde_json::json!({"unpack_ok": true, "ref_results": []}), + )) + .bind(Some(0_i32)) + .bind(2_i32) + .bind(Option::::None) + .bind(Option::::None) + .bind(Utc::now().to_rfc3339()) + .bind(Option::::None) + .execute(state.db.pool()) + .await + .unwrap(); + + // Add a child row (the drain's quarantine path also flips the + // child to `cancelled`). + let mut child_row = make_row(repo_id, "refs/heads/main", &"0".repeat(40), &"a".repeat(40)); + child_row.request_id = request_id.to_string(); + child_row.state = pending_state::PREPARED.to_string(); + child_row.applied_at = None; + child_row.id = crate::db::deterministic_id(&[ + "pending_ref_transition", + request_id, + repo_id, + &child_row.ref_name, + &child_row.old_sha, + &child_row.new_sha, + ]); + state + .db + .insert_pending_ref_transition_for_test(&child_row) + .await + .unwrap(); + + // The drain closure returns Retry unconditionally. Bound = 2, + // attempt_count = 2 → 2 + 1 = 3 > 2 → quarantined. + let state_for_closure = state.clone(); + let (processed, examined) = + drain_receive_pack_requests_with(state.clone(), 100, move |_s, req_id| { + let st = state_for_closure.clone(); + async move { + if req_id == request_id { + Ok(EffectsOutcome::Retry { + last_error: "injected retry-stuck".to_string(), + }) + } else { + apply_request_effects(&st, &req_id).await + } + } + }) + .await + .unwrap(); + assert_eq!(processed, 0, "Retry over-bound does not count as Done"); + assert_eq!(examined, 1, "the loop examined the request"); + + let after = state + .db + .get_receive_pack_request(request_id) + .await + .unwrap() + .expect("request row exists"); + assert_eq!( + after.state, + request_state::QUARANTINED, + "over-bound Retry quarantines the request" + ); + assert_eq!( + after.last_error.as_deref(), + Some("injected retry-stuck"), + "last_error carries the Retry reason" + ); + + let child = state + .db + .list_pending_ref_transitions_for_request(request_id) + .await + .unwrap(); + assert_eq!(child.len(), 1); + assert_eq!( + child[0].state, + pending_state::CANCELLED, + "quarantined parent cancels the child" + ); + } + + /// Under-bound Retry stays in `effects_pending`. With bound = 2 + /// and `attempt_count` = 1, the next retry puts the row at + /// `2 + 1 = 3`? No — the helper increments AFTER its + /// `attempt_count + 1 > bound` check. The check sees + /// `1 + 1 = 2 > 2 == false`, so the request stays in + /// `effects_pending` with attempt_count = 2. + #[sqlx::test] + async fn cell_retry_under_bound_stays_in_effects_pending(pool: sqlx::PgPool) { + let state = crate::test_support::test_state_with(pool, |cfg| { + cfg.effects_max_attempts = 2; + }) + .await; + + let request_id = "req-retry-under"; + let repo_id = "repo-retry-under"; + // The drain's `mark_request_effects_pending` only flips from + // `outcomes_committed`, so the test starts in that state and + // picks `attempt_count = 1`. The under-bound Retry keeps the + // row in `effects_pending` and increments `attempt_count` to 2. + sqlx::query( + r#"INSERT INTO receive_pack_requests + (id, repo_id, pusher_did, node_did, request_bytes, request_bytes_hash, + state, git_exit_ok, parsed_report, accepted_ordinal, attempt_count, + last_error, next_attempt_at, created_at, completed_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)"#, + ) + .bind(request_id) + .bind(repo_id) + .bind("did:key:z6pusher") + .bind("did:key:z6node") + .bind(Vec::::new()) + .bind(vec![0u8; 32]) + .bind(request_state::OUTCOMES_COMMITTED) + .bind(Some(true)) + .bind(Some( + serde_json::json!({"unpack_ok": true, "ref_results": []}), + )) + .bind(Some(0_i32)) + .bind(1_i32) + .bind(Option::::None) + .bind(Option::::None) + .bind(Utc::now().to_rfc3339()) + .bind(Option::::None) + .execute(state.db.pool()) + .await + .unwrap(); + + let state_for_closure = state.clone(); + let (processed, examined) = + drain_receive_pack_requests_with(state.clone(), 100, move |_s, req_id| { + let st = state_for_closure.clone(); + async move { + if req_id == request_id { + Ok(EffectsOutcome::Retry { + last_error: "under-bound".to_string(), + }) + } else { + apply_request_effects(&st, &req_id).await + } + } + }) + .await + .unwrap(); + assert_eq!(processed, 0, "Retry under-bound does not count as Done"); + assert_eq!(examined, 1, "the loop examined the request"); + + let after = state + .db + .get_receive_pack_request(request_id) + .await + .unwrap() + .expect("request row exists"); + assert_eq!( + after.state, + request_state::EFFECTS_PENDING, + "under-bound Retry stays in effects_pending" + ); + assert_eq!( + after.attempt_count, 2, + "attempt_count incremented by the under-bound Retry" + ); + } + + /// A child whose parent request has been PURGED (e.g. the + /// step-4 bounded retirement swept it) cannot be quarantined + /// because the parent is no longer in the table. The gate + /// logs a warning and the child stays `prepared`. + #[sqlx::test] + async fn cell_purged_request_orphans_children(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + + let tmp = tempfile::tempdir().expect("tempdir"); + let bare = tmp.path().join("repo.git"); + crate::git::store::init_bare(&bare).expect("init_bare"); + let on_disk_sha = seed_ref_on_bare(&bare, "refs/heads/main"); + + let repo_id = seed_repo_row(&state, bare.to_str().unwrap()).await; + + // Insert the CHILD only — no parent receive_pack_requests + // row, modelling the "parent purged" case. + let mut row = make_row(&repo_id, "refs/heads/main", &"0".repeat(40), &on_disk_sha); + row.request_id = "req-purged-parent".to_string(); + row.state = pending_state::PREPARED.to_string(); + row.applied_at = None; + state + .db + .insert_pending_ref_transition_for_test(&row) + .await + .unwrap(); + + let n = reconcile_prepared_from_disk(state.clone(), 100) + .await + .unwrap(); + // The parent-missing path skips — the row stays prepared + // because the reconcile's "if matches { ... } continue" + // short-circuits BEFORE promotion when the parent is gone. + assert_eq!( + n, 0, + "child stays prepared when its parent is purged (no parent to check)" + ); + let still_prepared = state + .db + .list_pending_ref_transitions_prepared(100) + .await + .unwrap(); + assert_eq!(still_prepared.len(), 1, "the child stays prepared"); + assert_eq!( + still_prepared[0].state, + pending_state::PREPARED, + "no parent → no quarantine; the child waits for human-attended recovery" + ); + } +} diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 67a85d695..5a528efb9 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -220,6 +220,7 @@ pub fn response_served_pack(output: &[u8]) -> bool { /// /// Accepts a push. The caller MUST verify HTTP Signature auth before /// calling this function. +#[allow(dead_code)] // used by tests; production uses receive_pack_raw pub async fn receive_pack( git_bin: &str, repo_path: &Path, @@ -244,6 +245,155 @@ pub async fn receive_pack( .body(Body::from(output))?) } +/// Run `git-receive-pack` and return the raw stdout bytes together with +/// the process exit status. Unlike [`receive_pack`], this does NOT bail +/// on a non-zero exit: the caller needs the stdout (which contains the +/// report-status with per-ref ok/ng results) even when the process +/// exits non-zero. A timeout still returns `Err`. +pub async fn receive_pack_raw( + git_bin: &str, + repo_path: &Path, + request_body: Bytes, + timeout: Duration, + admission: Option, +) -> Result<(Vec, bool)> { + let mut command = Command::new(git_bin); + command + .arg("receive-pack") + .arg("--stateless-rpc") + .arg(repo_path); + let (out, err, status, _admission) = drive_git_child_raw( + command, + request_body, + timeout, + "git-receive-pack", + admission, + ) + .await?; + // On timeout, drive_git_child_raw returns Err — we never reach here. + // On success/non-zero exit, we have the stdout and exit status. + if !status.success() { + let stderr = String::from_utf8_lossy(&err); + tracing::warn!(stderr = %stderr, "git-receive-pack exited non-zero"); + } + Ok((out, status.success())) +} + +/// Parse the git-receive-pack report-status output to determine per-ref +/// success/failure. Returns `(unpack_ok, per_ref_results)` where +/// `per_ref_results` is a list of `(ref_name, is_ok)`. +/// +/// The report-status format (after the sideband framing) is: +/// ```text +/// unpack ok\n (or: unpack fail\n) +/// ok \n (per successful ref) +/// ng \n (per rejected ref) +/// \n (empty line terminates) +/// ``` +/// +/// Returns `None` if the output cannot be parsed (e.g. the client did +/// not request report-status, or the output is truncated). In that +/// case the caller should treat all refs as uncertain. +pub fn parse_report_status(output: &[u8]) -> Option<(bool, Vec<(String, bool)>)> { + let text = std::str::from_utf8(output).ok()?; + // Strip sideband framing: each line starts with a pkt-line length + // prefix and a channel byte (1 = stdout, 2 = stderr). The actual + // data starts after the first `0000` flush packet or after we + // strip sideband bytes. + let stripped = strip_sideband(text)?; + // The report is framed TWICE when the client negotiated side-band-64k, + // which `git push` over smart HTTP does — so this is the common case, not + // an exotic one. The outer frame is the side-band envelope; band 1 carries + // the report-status stream, which is ITSELF pkt-line encoded. Real bytes + // from git 2.50.1 rejecting one ref of a two-ref push: + // + // 0057\x01000eunpack ok\n0028ng refs/heads/main non-fast-forward\n... + // + // After one pass the first line reads `000eunpack ok`, which fails the + // `unpack ok` check below and makes this return None — i.e. "no report", + // which the caller treats as inconclusive and keeps every declared ref. + // The per-ref gate would then be inert for exactly the pushes it exists to + // filter. A second pass removes the inner pkt-line framing; it is a no-op + // on the single-framed shape, because plain report text does not begin + // with four hex digits. + let stripped = strip_sideband(&stripped).unwrap_or(stripped); + let lines: Vec<&str> = stripped.lines().collect(); + if lines.is_empty() { + return None; + } + + // First non-empty line is "unpack ok" or "unpack fail". + let unpack_line = lines.iter().find(|l| !l.is_empty())?; + let unpack_ok = if unpack_line.starts_with("unpack ok") { + true + } else if unpack_line.starts_with("unpack fail") { + false + } else { + return None; + }; + + let mut results = Vec::new(); + for line in &lines[1..] { + let line = line.trim(); + if line.is_empty() { + break; + } + if let Some(rest) = line.strip_prefix("ok ") { + results.push((rest.to_string(), true)); + } else if let Some(rest) = line.strip_prefix("ng ") { + // "ng " — skip the reason + let ref_name = rest.split_whitespace().next()?.to_string(); + results.push((ref_name, false)); + } + } + + Some((unpack_ok, results)) +} + +/// Strip git sideband framing from a pkt-line encoded output. +/// Sideband-encoded lines start with a 4-hex-digit length, then a +/// channel byte (0x01=stdout, 0x02=stderr), then payload. Returns +/// the decoded payload lines concatenated, or `None` if the framing +/// is malformed. +fn strip_sideband(text: &str) -> Option { + let mut output = String::new(); + let mut pos = 0; + let bytes = text.as_bytes(); + + loop { + if pos + 4 > bytes.len() { + break; + } + let len_str = std::str::from_utf8(&bytes[pos..pos + 4]).ok()?; + let len = usize::from_str_radix(len_str, 16).ok()?; + if len == 0 { + // Flush packet — end of sideband stream + break; + } + if len < 4 || pos + len > bytes.len() { + break; + } + let pkt_data = std::str::from_utf8(&bytes[pos + 4..pos + len]).ok()?; + pos += len; + + // Sideband: first byte is channel (1=stdout, 2=stderr) + if let Some(payload) = pkt_data.strip_prefix('\x01') { + output.push_str(payload); + } else if pkt_data.starts_with('\x02') { + // stderr — skip (git error messages) + } else { + // Not sideband encoded — pass through + output.push_str(pkt_data); + } + } + + if output.is_empty() { + None + } else { + Some(output) + } +} + /// Sends SIGTERM to a child's whole process group on drop, unless disarmed first. /// /// A served `git upload-pack`/`receive-pack` forks helpers such as `pack-objects`. @@ -578,6 +728,98 @@ async fn drive_git_child( Ok((out, admission)) } +/// Like [`drive_git_child`], but returns stdout and stderr even on a +/// non-zero exit status. Used by [`receive_pack_raw`] so the caller +/// can parse the report-status output from a failed `git-receive-pack`. +async fn drive_git_child_raw( + mut command: Command, + input: Bytes, + timeout: Duration, + _what: &str, + admission: Option, +) -> Result<( + Vec, + Vec, + std::process::ExitStatus, + Option, +)> { + command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + #[cfg(unix)] + command.process_group(0); + + let mut child = command.spawn()?; + + let mut stdin = child.stdin.take(); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + + #[cfg(unix)] + let pgid = child.id().map(|id| id as i32); + #[cfg(unix)] + let mut group_guard = KillGroupOnDrop { + child: Some(child), + pgid, + admission, + }; + + let mut out = Vec::new(); + let mut err = Vec::new(); + + let interact = async { + let write = async { + match stdin.take() { + Some(mut s) => s.write_all(&input).await, + None => Ok(()), + } + }; + #[cfg(unix)] + let child_ref = group_guard.child_mut(); + #[cfg(not(unix))] + let child_ref = &mut child; + let (write_result, r_out, r_err, status) = tokio::join!( + write, + stdout.read_to_end(&mut out), + stderr.read_to_end(&mut err), + child_ref.wait(), + ); + r_out?; + r_err?; + Ok::<_, anyhow::Error>((write_result, status?)) + }; + + let timed = tokio::time::timeout(timeout, interact).await; + let (write_result, status, admission) = match timed { + Ok(result) => { + #[cfg(unix)] + let admission = group_guard.disarm(); + let (write_result, status) = result?; + (write_result, status, admission) + } + Err(_elapsed) => { + #[cfg(unix)] + { + reap_group_on_timeout(group_guard.child_mut()).await; + drop(group_guard.disarm()); + } + #[cfg(not(unix))] + { + let _ = child.start_kill(); + let _ = tokio::time::timeout(Duration::from_secs(2), child.wait()).await; + drop(admission); + } + return Err(GitServiceTimeout.into()); + } + }; + + write_result.context("failed to write to git stdin")?; + + Ok((out, err, status, admission)) +} + fn service_to_command(service: &str) -> &str { match service { "git-upload-pack" => "upload-pack", @@ -866,6 +1108,48 @@ mod tests { use std::process::Command; use tempfile::TempDir; + /// Byte-for-byte capture from `git receive-pack` 2.50.1 rejecting one ref of + /// a two-ref push, with side-band-64k negotiated — what `git push` over + /// smart HTTP actually sends. The report is framed twice: the outer + /// side-band envelope, then the report-status stream's own pkt-lines. + /// + /// This fixture is the point of the test. A hand-written single-framed + /// string parses fine with only one strip pass, so a fixture invented to + /// match the parser hides the exact case the parser is for. + const REAL_SIDEBAND_REPORT: &[u8] = + b"0057\x01000eunpack ok\n0028ng refs/heads/main non-fast-forward\n0018ok refs/heads/third\n0000"; + + #[test] + fn a_real_double_framed_sideband_report_is_read_not_treated_as_absent() { + let (unpack_ok, refs) = + parse_report_status(REAL_SIDEBAND_REPORT).expect("a real git report must parse"); + assert!(unpack_ok, "unpack line must be read through both frames"); + assert_eq!( + refs, + vec![ + ("refs/heads/main".to_string(), false), + ("refs/heads/third".to_string(), true), + ], + "the rejected ref must be distinguished from the accepted one" + ); + } + + /// The single-framed shape must keep working: the second pass has to be a + /// no-op there, not a corruption. Plain report text does not begin with + /// four hex digits, which is what makes that safe. + #[test] + fn a_single_framed_report_still_parses_after_the_second_pass() { + // Framed programmatically rather than by a hand-counted hex prefix: a + // wrong length makes the parser return None, which would look exactly + // like the regression this pair of tests exists to catch. + let payload = "\x01unpack ok\nok refs/heads/x\n"; + let single = format!("{:04x}{payload}0000", payload.len() + 4); + let (unpack_ok, refs) = + parse_report_status(single.as_bytes()).expect("single-framed must parse"); + assert!(unpack_ok); + assert_eq!(refs, vec![("refs/heads/x".to_string(), true)]); + } + /// List OIDs in a pack by writing it to a temp dir and running verify-pack. pub(super) fn pack_object_ids(pack: &[u8]) -> std::collections::HashSet { let dir = TempDir::new().unwrap(); diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 5617b4198..cd2f0f4ca 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -5,6 +5,14 @@ use std::process::Command; /// Initialize a new bare git repository with SHA-1 object format (default). /// /// SHA-1 is used for maximum compatibility with standard git clients. +/// +/// P1 (reviewer-2 round 4): `git config core.logAllRefUpdates true` +/// logs only `refs/heads/` and `refs/remotes/` — tag pushes +/// (`refs/tags/v1`) produce no reflog, and the reconcile gate at +/// `durable_outbox::reconcile_prepared_from_disk` requires a +/// reflog-entry proof to promote a row. Setting it to `always` +/// makes git log every ref update regardless of namespace, which is +/// what the recovery gate assumes. pub fn init_bare(path: &Path) -> Result<()> { if path.exists() { bail!("repository already exists at {}", path.display()); @@ -25,10 +33,188 @@ pub fn init_bare(path: &Path) -> Result<()> { // Write a default HEAD pointing to main std::fs::write(path.join("HEAD"), "ref: refs/heads/main\n")?; + // #26 Split PR 1: turn reflogs ON for this bare repo. `core.logAllRefUpdates` + // defaults to FALSE for bare repositories, so without this a bare repo keeps no + // record of what a ref did — only what it currently points at. + // + // The durable post-receive outbox's startup reconcile needs exactly that record. + // Its job is to decide whether a `prepared` transition (old -> new) actually + // LANDED after a crash, and the current SHA alone cannot answer that: a row + // claiming B -> A also "matches" a ref that was already sitting at A for some + // unrelated reason, and promoting it would write a push event, a certificate, + // and an anchor for a transition that never happened. The reflog is git's own + // per-ref landing record — one line per update carrying ` ` plus the + // time it happened — so [`ref_reflog_entries`] can prove the ref moved the way + // the row claims, and prove it moved AFTER the row was written. + // + // P1 (reviewer-2 round 4): the value MUST be `always`, not `true`. + // Under `true`, git logs only `refs/heads/` and `refs/remotes/` — + // tag pushes (`refs/tags/v1`) produce no reflog, and the + // reconcile gate can never promote a `refs/tags/*` row. The + // value `always` makes git log every ref update regardless of + // namespace, which is what the recovery gate assumes. I + // confirmed by execution in a bare repo: with `true`, an + // `update-ref refs/tags/v1 ` produced no `logs/refs/tags/` + // entry; with `always` it did. + // + // Failure is non-fatal on purpose: a repo without reflogs still serves every + // git operation, it only loses AUTOMATIC crash recovery for its outbox rows + // (the reconcile leaves those rows `prepared` for human-attended recovery + // rather than promoting something it cannot prove). + let config = Command::new("git") + .args(["config", "core.logAllRefUpdates", "always"]) + .current_dir(path) + .output(); + match config { + Ok(out) if !out.status.success() => { + tracing::warn!( + path = %path.display(), + stderr = %String::from_utf8_lossy(&out.stderr), + "failed to enable core.logAllRefUpdates=always; durable-outbox reconcile \ + will not be able to prove ref landings for this repo (tag pushes will \ + never auto-recover)" + ); + } + Err(e) => { + tracing::warn!( + path = %path.display(), + err = %e, + "failed to run git config core.logAllRefUpdates=always; durable-outbox \ + reconcile will not be able to prove ref landings for this repo" + ); + } + Ok(_) => {} + } + tracing::info!("initialized bare repo at {}", path.display()); Ok(()) } +/// One parsed reflog entry: the ` ` pair a single ref update recorded, +/// plus the unix timestamp git stamped it with. +/// +/// This is the unit of PER-REF LANDING PROOF the durable-outbox reconcile runs on. +/// A row that claims `old -> new` is only promoted when the ref's reflog carries an +/// entry with the same pair, stamped at or after the row was written; see +/// [`crate::durable_outbox::reconcile_prepared_from_disk`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReflogEntry { + pub old_sha: String, + pub new_sha: String, + /// Seconds since the unix epoch, as git wrote them. + pub at: i64, +} + +/// How many bytes of the END of a reflog file are read. A reflog line is roughly +/// 150 bytes, so this window holds on the order of two thousand of the most recent +/// updates to a single ref. +/// +/// The bound costs nothing in proving power, because of what the caller asks. The +/// gate only ever accepts an entry stamped at or after +/// `created_at - REFLOG_CLOCK_SKEW`, i.e. within about a minute of a row that is +/// itself inside `MAX_RECONCILE_AGE`; reflogs are append-ordered oldest-first, so +/// every entry that could possibly qualify is at the tail. What the window buys is +/// a ceiling: a reflog grows one line per ref update and how often a ref is updated +/// is PUSHER-controlled, so an unbounded read is an attacker-sized allocation taken +/// once per stranded row, at startup, which is the moment the node can least absorb +/// it. +const REFLOG_TAIL_BYTES: u64 = 256 * 1024; + +/// Read the tail of one ref's reflog in a bare repository, newest entry LAST. +/// +/// Reads `logs/` directly rather than shelling out to `git reflog`: the +/// file format is stable and documented, the reconcile may call this once per +/// stranded row at startup, and a plain file read cannot be defeated by the ref's +/// reflog having been expired out of the `git reflog show` default window. +/// +/// Only the last [`REFLOG_TAIL_BYTES`] are read, and only whole lines within that +/// window: when the file is longer, the bytes before the first newline inside the +/// window are a PARTIAL record and are discarded, so a record sliced mid-SHA can +/// never be tokenized into a bogus `old -> new` pair. An entry older than the +/// window reads as absent, which is the safe direction — the caller treats +/// "no matching entry" as unproven and leaves the row where it is, rather than +/// promoting it. +/// +/// Returns `Ok(None)` when the repo keeps no reflog for that ref — either because +/// `core.logAllRefUpdates` was off when the ref moved (repos created before +/// [`init_bare`] started enabling it) or because the ref was deleted (git removes a +/// deleted ref's reflog with it). `None` is NOT evidence that nothing landed; it is +/// the absence of evidence, and callers must treat it as "unproven", never as +/// "proven false". +/// +/// Line format (`git-check-ref-format`/`refs` docs): +/// ` \t` +pub fn ref_reflog_entries(repo_path: &Path, ref_name: &str) -> Result>> { + // Refuse anything that could climb out of `logs/`. Ref names are validated at + // the push edge, but this function takes a name off a DB row, so it re-checks + // rather than trusting the row. + if ref_name.is_empty() + || ref_name.contains("..") + || ref_name.starts_with('/') + || !ref_name.starts_with("refs/") + { + bail!("refusing to read a reflog for a non-refs/ ref name: {ref_name}"); + } + let path = repo_path.join("logs").join(ref_name); + let mut file = match std::fs::File::open(&path) { + Ok(f) => f, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e).context("failed to open reflog"), + }; + let len = file.metadata().context("failed to stat reflog")?.len(); + let truncated = len > REFLOG_TAIL_BYTES; + if truncated { + use std::io::Seek; + file.seek(std::io::SeekFrom::Start(len - REFLOG_TAIL_BYTES)) + .context("failed to seek to the reflog tail")?; + } + let mut buf = Vec::with_capacity(REFLOG_TAIL_BYTES.min(len) as usize); + { + use std::io::Read; + // Cap the read itself, not just the seek: the file can grow between the + // stat and the read, and the allocation must stay bounded either way. + file.take(REFLOG_TAIL_BYTES) + .read_to_end(&mut buf) + .context("failed to read reflog")?; + } + // A window into the middle of the file almost certainly starts mid-record. + // Drop everything before the first newline so only whole lines are parsed; + // a reflog is one record per line, so the first full record starts there. + let window: &[u8] = if truncated { + match buf.iter().position(|b| *b == b'\n') { + Some(i) => &buf[i + 1..], + // No newline in the whole window: every byte is part of one partial + // record, so there is nothing whole to parse. + None => &[], + } + } else { + &buf + }; + let raw = String::from_utf8_lossy(window); + let mut out = Vec::new(); + for line in raw.lines() { + // The message after the TAB can contain anything, including spaces and + // (in a `git commit -m` subject) tabs of its own, so split the header off + // at the FIRST tab and tokenize only that. + let header = line.split('\t').next().unwrap_or(line); + let tokens: Vec<&str> = header.split_whitespace().collect(); + // ` `: at minimum old, new, ts, tz. + if tokens.len() < 4 { + continue; + } + let at = match tokens[tokens.len() - 2].parse::() { + Ok(v) => v, + Err(_) => continue, + }; + out.push(ReflogEntry { + old_sha: tokens[0].to_string(), + new_sha: tokens[1].to_string(), + at, + }); + } + Ok(Some(out)) +} + /// Check if a path contains a valid bare git repository. #[allow(dead_code)] pub fn is_valid_bare(path: &Path) -> bool { @@ -63,6 +249,82 @@ pub fn list_refs(repo_path: &Path) -> Result> { Ok(refs) } +/// #26 Split PR 1 step 5 — read a single ref's value. Returns +/// `Ok(None)` for absent refs (git's exit code is 1; we treat that +/// as "not present" rather than a hard error). The value is the +/// full hex SHA — for a marker ref, that hex is the marker value +/// the live handler (or a test) wrote via `update-ref`, which is +/// a 40-char SHA-1 hex string. The reconcile compares two hex +/// strings. +pub fn read_ref(repo_path: &Path, ref_name: &str) -> Result> { + let output = Command::new("git") + .args(["show-ref", "--verify", "--hash", ref_name]) + .current_dir(repo_path) + .output() + .context("failed to run git show-ref")?; + if !output.status.success() { + // `git show-ref --verify` returns 1 when the ref is absent + // and non-zero (often 128) on other errors. We can't + // distinguish without inspecting stderr; the safe choice + // is to treat any non-zero as "absent" and let the caller + // (the reconcile gate) treat that as a quarantine signal. + return Ok(None); + } + let sha = String::from_utf8(output.stdout) + .context("git show-ref output is not utf-8")? + .trim() + .to_string(); + if sha.is_empty() { + return Ok(None); + } + Ok(Some(sha)) +} + +/// #26 Split PR 1 step 5 — compute the marker ref value for a +/// `request_bytes_hash`. Git's `update-ref` rejects arbitrary +/// 64-char hex; it only accepts 40-char SHA-1 hex that resolves +/// to an existing object. We sidestep both halves by feeding the +/// first 20 bytes of the 32-byte SHA-256 through `git +/// hash-object -w` (a blob object is content-addressed, so the +/// resulting 40-char SHA-1 is the marker value). The live +/// handler writes this; the reconcile's `read_ref` reads it +/// back; the gate compares hex strings. +/// +/// `repo_path` is the bare repo the marker ref lives in. The +/// blob is stored in the repo's object database so a later +/// `git show-ref --verify --hash` resolves cleanly. +pub fn marker_value_for(repo_path: &Path, request_bytes_hash: &[u8]) -> Result { + let mut content = Vec::with_capacity(20); + let n = 20.min(request_bytes_hash.len()); + content.extend_from_slice(&request_bytes_hash[..n]); + let mut child = Command::new("git") + .args(["hash-object", "-w", "--stdin"]) + .current_dir(repo_path) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .context("failed to spawn git hash-object")?; + use std::io::Write; + child + .stdin + .as_mut() + .context("stdin pipe")? + .write_all(&content) + .context("write to git hash-object stdin")?; + let out = child.wait_with_output().context("git hash-object wait")?; + if !out.status.success() { + anyhow::bail!( + "git hash-object failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + Ok(String::from_utf8(out.stdout) + .context("git hash-object output not utf-8")? + .trim() + .to_string()) +} + /// Read the current HEAD commit hash of a repository. /// Returns None if the repo is empty (no commits yet). pub fn head_commit(repo_path: &Path) -> Result> { @@ -882,6 +1144,235 @@ mod tests { use std::path::Path; use std::process::Command; + /// #26 split 1/4: a bare repo this node creates must KEEP REFLOGS, because + /// the durable-outbox reconcile has no other way to prove that a stranded + /// transition actually landed. `core.logAllRefUpdates` defaults to false for + /// bare repos, so without the explicit config a crashed push is unrecoverable + /// — the reconcile can see where a ref points, never how it got there. + /// + /// MUTATION (RED): drop the `git config core.logAllRefUpdates` call in + /// `init_bare` and no `logs/refs/heads/main` file appears. + #[test] + fn init_bare_keeps_reflogs_so_a_landing_can_be_proven() { + let td = tempfile::TempDir::new().unwrap(); + let bare = td.path().join("repo.git"); + super::init_bare(&bare).unwrap(); + + // Build a commit and move a ref onto it, the way receive-pack would. + let run = |args: &[&str]| -> String { + let out = Command::new("git") + .args(args) + .current_dir(&bare) + .stdin(std::process::Stdio::null()) + .env("GIT_AUTHOR_NAME", "test") + .env("GIT_AUTHOR_EMAIL", "test@example.com") + .env("GIT_COMMITTER_NAME", "test") + .env("GIT_COMMITTER_EMAIL", "test@example.com") + .output() + .unwrap(); + assert!( + out.status.success(), + "git {:?} failed: {}", + args, + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).unwrap().trim().to_string() + }; + let tree = run(&["mktree"]); + let commit = run(&["commit-tree", &tree, "-m", "root"]); + run(&["update-ref", "refs/heads/main", &commit]); + + let entries = super::ref_reflog_entries(&bare, "refs/heads/main") + .unwrap() + .expect("a repo created by init_bare keeps a reflog for its refs"); + assert_eq!(entries.len(), 1, "one update, one entry"); + assert_eq!( + entries[0].old_sha, "0000000000000000000000000000000000000000", + "the entry records where the ref came FROM — the half a current-SHA \ + check can never recover" + ); + assert_eq!(entries[0].new_sha, commit); + assert!( + entries[0].at > 0, + "the entry is timestamped, so proof can be required to postdate the intent" + ); + } + + /// One reflog record. Fixed-width `i` and timestamp keep every filler line the + /// same length, so the test below can place the window boundary on an exact + /// byte. + fn reflog_line(old: &str, new: &str, at: i64, msg: &str) -> String { + format!("{old} {new} tester {at} +0000\tpush: {msg}\n") + } + + fn filler_line(i: usize, pad: usize) -> String { + reflog_line( + &format!("{:040x}", i), + &format!("{:040x}", i + 1), + 1_600_000_000, + &format!("filler {i:06}{}", "x".repeat(pad)), + ) + } + + /// The bound is a ceiling on the READ, not on the proof: a ref hammered with + /// far more updates than the tail window can hold still yields its recent + /// entries, which is the only region the landing gate ever accepts from. And + /// the record the window CUTS THROUGH must be discarded whole, never + /// half-parsed. + /// + /// The file is laid out so the window boundary lands 20 bytes into a record's + /// old SHA — the hazardous alignment, where the surviving suffix still has + /// enough fields to tokenize and would yield a 20-character "old SHA" as a + /// bogus `old -> new` pair. Landing proof is an exact pair match, so a bogus + /// pair is a fabricated proof. + /// + /// MUTATION (RED): read the first `REFLOG_TAIL_BYTES` instead of the last and + /// the recent entry disappears; keep the leading partial line instead of + /// discarding it and the 20-character SHA appears. + #[test] + fn ref_reflog_entries_reads_whole_records_from_the_recent_tail() { + let td = tempfile::TempDir::new().unwrap(); + let bare = td.path().join("repo.git"); + super::init_bare(&bare).unwrap(); + + let window = super::REFLOG_TAIL_BYTES as usize; + // The record the boundary will slice, and the recent one a reconcile + // would actually be looking for. + let hazard = reflog_line(&"a".repeat(40), &"b".repeat(40), 1_700_000_000, "hazard"); + let recent = reflog_line(&"c".repeat(40), &"d".repeat(40), 1_700_009_999, "recent"); + + // Everything from the hazard record to EOF must measure exactly + // `window + 20`, so the read starts 20 bytes into the hazard's old SHA. + let tail_target = window + 20; + let needed = tail_target - hazard.len() - recent.len(); + let base = filler_line(0, 0).len(); + assert!(needed > 2 * base, "layout math needs room for filler"); + let full = needed / base - 1; + let pad = needed - (full + 1) * base; + let mut tail = hazard.clone(); + for i in 0..full { + tail.push_str(&filler_line(i, 0)); + } + tail.push_str(&filler_line(full, pad)); + tail.push_str(&recent); + assert_eq!( + tail.len(), + tail_target, + "the tail must measure exactly window + 20 for the boundary to land \ + inside the hazard record's old SHA" + ); + + // Anything before the hazard record is outside the window entirely. + let mut body = String::new(); + for i in 0..8 { + body.push_str(&filler_line(1_000 + i, 0)); + } + let lead = body.len(); + body.push_str(&tail); + + let log_path = bare.join("logs/refs/heads/busy"); + std::fs::create_dir_all(log_path.parent().unwrap()).unwrap(); + std::fs::write(&log_path, &body).unwrap(); + assert_eq!( + body.len() - window, + lead + 20, + "the read must begin 20 bytes into the hazard record" + ); + + let entries = super::ref_reflog_entries(&bare, "refs/heads/busy") + .unwrap() + .expect("the reflog exists"); + + // The tail is what got read. + let last = entries.last().expect("the window holds whole records"); + assert_eq!( + (last.old_sha.as_str(), last.new_sha.as_str(), last.at), + ( + "c".repeat(40).as_str(), + "d".repeat(40).as_str(), + 1_700_009_999 + ), + "the newest entry — the only region the landing gate accepts — survives \ + the bound intact" + ); + assert!( + entries.len() < body.lines().count(), + "the read is bounded: not every record in the file is parsed" + ); + + // And the sliced record was dropped rather than half-parsed. Both halves + // matter: no truncated SHA may appear, and the hazard's pair must not be + // reconstructed from a partial line either. + for e in &entries { + assert_eq!( + e.old_sha.len(), + 40, + "a partial record was parsed into a bogus old SHA: {e:?}" + ); + assert_eq!(e.new_sha.len(), 40, "a partial record was parsed: {e:?}"); + } + assert!( + !entries.iter().any(|e| e.new_sha == "b".repeat(40)), + "the record the window cut through must not contribute a pair at all" + ); + } + + /// The safe direction of the same bound. An entry that sits only in the + /// discarded older region reads as absent, and absent means UNPROVEN — the + /// reconcile leaves such a row where it is instead of promoting it, which is + /// the failure mode a bounded read is allowed to have. + #[test] + fn an_entry_older_than_the_reflog_tail_reads_as_unproven() { + let td = tempfile::TempDir::new().unwrap(); + let bare = td.path().join("repo.git"); + super::init_bare(&bare).unwrap(); + + // The sought pair is written FIRST, then buried under enough later + // updates to push it clear out of the window. + let buried = ("c".repeat(40), "d".repeat(40), 1_650_000_000i64); + let log_path = bare.join("logs/refs/heads/buried"); + std::fs::create_dir_all(log_path.parent().unwrap()).unwrap(); + let mut body = reflog_line(&buried.0, &buried.1, buried.2, "the buried landing"); + let mut i = 0; + while body.len() <= super::REFLOG_TAIL_BYTES as usize * 2 { + body.push_str(&filler_line(i, 0)); + i += 1; + } + std::fs::write(&log_path, &body).unwrap(); + + let entries = super::ref_reflog_entries(&bare, "refs/heads/buried") + .unwrap() + .expect("the reflog exists"); + assert!( + !entries + .iter() + .any(|e| e.old_sha == buried.0 && e.new_sha == buried.1), + "an entry outside the tail window is simply not seen — the caller then \ + treats the landing as unproven and leaves the row alone, never the \ + other way round" + ); + } + + /// A ref with no reflog reads as `None` — "no evidence", which callers must + /// treat as unproven rather than as proof of nothing having happened. + #[test] + fn ref_reflog_entries_is_none_when_the_repo_kept_no_log() { + let td = tempfile::TempDir::new().unwrap(); + let bare = td.path().join("repo.git"); + super::init_bare(&bare).unwrap(); + assert!( + super::ref_reflog_entries(&bare, "refs/heads/never-existed") + .unwrap() + .is_none(), + "a missing reflog is None, not an empty proof set" + ); + // A name that could climb out of `logs/` is refused outright. + assert!( + super::ref_reflog_entries(&bare, "../../etc/passwd").is_err(), + "reflog lookups take a ref name off a DB row, so the path is re-checked" + ); + } + #[test] fn branch_diff_names_lists_changed_paths() { let td = tempfile::TempDir::new().unwrap(); diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 66bfa0961..d08676efd 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -5,6 +5,7 @@ mod bootstrap; mod cert; mod config; mod db; +mod durable_outbox; mod encrypted_pin; mod error; mod git; @@ -585,6 +586,7 @@ async fn main() -> Result<()> { } let _legacy_cid_sweep = spawn_legacy_cid_sweep(&state, &config); + let _queue_lifecycle_sweep = spawn_queue_lifecycle_sweep(&state, &config); let router = server::build_router(state.clone()); // Re-register the socket bound at startup — same fd, so there was never a @@ -676,6 +678,58 @@ async fn main() -> Result<()> { let grace = std::time::Duration::from_secs(config.shutdown_grace_secs); info!(grace_secs = config.shutdown_grace_secs, "axum server ready"); + // #26 Split PR 1: drain any `applied` rows left by a previous + // process that crashed after Git applied a ref but before the + // bookkeeping landed. Runs once, BEFORE the server accepts new + // pushes, so a recovery re-derivation does not race a fresh push. + // Non-fatal: a transient drain failure is logged and the rows + // remain `applied` for the next startup to pick up. + // + // P1-A: the reconcile step runs FIRST and promotes any `prepared` + // or `uncertain` row whose target SHA actually landed on disk. + // This is the path that recovers a ref when the post-receive + // `mark_pending_ref_transitions_applied` call errored or was + // interrupted after `receive_pack` returned Ok, or when + // receive-pack returned Err but some refs may have landed + // (the `uncertain` state). Without this step, the drain (gated + // on `state = 'applied'`) would never see those rows. + // + // P2 (reviewer-1/2 round 3): use the multi-pass reconcile so + // prepared/uncertain rows beyond the first 1000-row page are + // processed in the same startup, rather than waiting for the + // next restart (where they might age out of MAX_RECONCILE_AGE). + match durable_outbox::reconcile_prepared_from_disk_all( + state.clone(), + durable_outbox::DRAIN_PER_PASS_LIMIT, + durable_outbox::DRAIN_MAX_PASSES, + ) + .await + { + Ok(0) => {} + Ok(n) => info!( + n, + "reconciled prepared/uncertain -> applied via on-disk ref match" + ), + Err(e) => warn!( + err = %e, + "pending ref transition reconcile failed at startup (non-fatal; will retry on next start)" + ), + } + match durable_outbox::drain_receive_pack_requests_all( + state.clone(), + durable_outbox::DRAIN_PER_PASS_LIMIT, + durable_outbox::DRAIN_MAX_PASSES, + ) + .await + { + Ok(0) => {} + Ok(n) => info!(n, "drained pending ref transitions from prior run"), + Err(e) => warn!( + err = %e, + "pending ref transition drain failed at startup (non-fatal; will retry on next start)" + ), + } + // `into_make_service_with_connect_info` exposes the socket peer address as // `ConnectInfo` so the push limiter can key on the real client // when no trusted proxy header applies (see `rate_limit::client_key`). @@ -746,6 +800,43 @@ fn spawn_legacy_cid_sweep(state: &AppState, config: &Config) -> tokio::task::Joi }) } +/// #26 Split PR 1 step 4 — periodic queue-lifecycle purge. Runs on +/// the same detached task pattern as `spawn_legacy_cid_sweep`: +/// tokio::spawn with a shutdown watcher, never on the boot path. The +/// interval is fixed at 24 hours (the spec calls for "one per +/// cluster per day"); the inter-batch delay is implicit in the +/// batch size plus the wall-clock cost of each pass. The drain +/// (`drain_receive_pack_requests_all`) and the purge +/// (`purge_request_queue`) share the same `DRAIN_PER_PASS_LIMIT` +/// budget so a 1000-row purge pass takes roughly the same time as +/// a 1000-row drain pass. +fn spawn_queue_lifecycle_sweep(state: &AppState, config: &Config) -> tokio::task::JoinHandle<()> { + let db = state.db.clone(); + let retention_days = config.queue_retention_days; + let batch = config.queue_purge_batch; + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(24 * 3600)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + _ = interval.tick() => { + if let Err(e) = durable_outbox::purge_request_queue( + &db, + retention_days, + batch, + ).await { + tracing::warn!(err = %e, "queue lifecycle purge failed; will retry on next tick"); + } + } + _ = shutdown_rx.changed() => { + break; + } + } + } + }) +} + fn spawn_shutdown_signal(tx: watch::Sender) { tokio::spawn(async move { #[cfg(unix)] diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 48381e3dc..df5f50cc4 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -415,8 +415,8 @@ fn f3_second_writer_leased_until_reap() { NOT RepoWriteGuard (which drops at the disconnect instant, reopening F3).", ); let receive = repos_production - .find("smart_http::receive_pack(") - .expect("F3 gate stale: git_receive_pack no longer calls smart_http::receive_pack"); + .find("smart_http::receive_pack_raw(") + .expect("F3 gate stale: git_receive_pack no longer calls smart_http::receive_pack_raw"); assert!( lease_acquire < with_lease && with_lease < receive, "F3 gate bypassed: the write lease must be acquired, then carried by the \ @@ -533,10 +533,8 @@ fn inv22_replication_tail_spawns_at_the_durability_boundary() { .expect("split always yields a first chunk"); let success_flag = production - .find("let push_succeeded = receive_result.is_ok();") - .expect( - "U5 gate missing: the tail's success gate must be bound from receive_result.is_ok()", - ); + .find("let push_succeeded = ") + .expect("U5 gate missing: the tail's success flag must be bound before the gate"); let gate_open = production .find("if push_succeeded {") .expect("U5 gate missing: the tail spawn must be gated on the push having succeeded"); @@ -549,9 +547,15 @@ fn inv22_replication_tail_spawns_at_the_durability_boundary() { let touch = production .find("state.db.touch_repo(") .expect("U5 gate stale: git_receive_pack no longer calls touch_repo"); - let webhook = production - .find("webhooks::fire_event(") - .expect("U5 gate stale: git_receive_pack no longer fires push webhooks"); + // #26 Split PR 1 step 3 — the webhook fan-out moved into + // `durable_outbox::apply_request_effects`, which the live + // handler calls inline (the recovery drain calls the same + // function on the next startup). The gate now pins that the + // handler is wired to the executor, not to a per-ref inline + // webhook call. + let effects_executor = production + .find("apply_request_effects(&state, &request_id)") + .expect("U5 gate stale: git_receive_pack no longer calls apply_request_effects inline"); assert!( success_flag < gate_open && gate_open < spawn, @@ -565,9 +569,259 @@ fn inv22_replication_tail_spawns_at_the_durability_boundary() { rejected push now spawns a tail" ); assert!( - spawn < release && spawn < touch && spawn < webhook, + spawn < release && spawn < touch && spawn < effects_executor, "U5 gate bypassed: the tail must be spawned BEFORE guard.release, touch_repo \ - and the webhook fan-out, so a disconnect in any of those windows cannot drop \ + and the effect executor, so a disconnect in any of those windows cannot drop \ this push's pins, recovery copy, and announcements" ); } + +/// #26 Split PR 1 step 3 — drain + handler share a single effect +/// executor. The live handler (api/repos.rs) and the recovery +/// drain (`durable_outbox::drain_receive_pack_requests_with`) both +/// call `apply_request_effects`, so the per-ref effects fan-out +/// lives in exactly one place. The v29 per-ref walk +/// (`derive_one`, `drain_pending_ref_transitions_all`, +/// `lookup_accepted_ordinal`) is dead code: any caller reintroduced +/// would be the per-ref walk the step-3 PR removed. This gate +/// fails if a call site slips back in or the new seam is bypassed. +#[test] +fn inv26_step3_live_and_drain_share_apply_request_effects() { + let repos = src("api/repos.rs"); + let outbox = src("durable_outbox.rs"); + + // The live handler calls `apply_request_effects`. Split the + // file at the test attribute so test code can't satisfy the + // gate by itself. + let production_repos = repos + .split("\nmod tests {") + .next() + .expect("split always yields a first chunk"); + + assert!( + production_repos.contains("apply_request_effects(&state, &request_id)"), + "live handler must call `apply_request_effects(&state, &request_id)`; \ + reverting to a per-ref inline fan-out splits live and recovery" + ); + + // The drain's per-request seam calls the same executor. Test + // code lives below `mod drain_tests`, so split there too. + let production_outbox = outbox + .split("\nmod drain_tests {") + .next() + .expect("split always yields a first chunk"); + + assert!( + production_outbox.contains("drain_receive_pack_requests_with"), + "drain seam `drain_receive_pack_requests_with` must exist; \ + removing it forces a per-ref walk back into the drain" + ); + assert!( + production_outbox.contains("apply_request_effects"), + "durable_outbox production code must define `apply_request_effects`; \ + removing it splits the executor between live and recovery" + ); + + // The drain's per-request walker is wired to the executor. + // The closure body of `drain_receive_pack_requests_with` is the + // only call site — if a future change calls `derive_one` + // instead, this assertion fires. + let drain_seam_open = production_outbox + .find("pub async fn drain_receive_pack_requests_with") + .expect("drain seam must be defined in the production half"); + let drain_seam_close = production_outbox[drain_seam_open..] + .find("\n}\n") + .expect("drain seam body must close"); + let drain_seam_body = &production_outbox[drain_seam_open..drain_seam_open + drain_seam_close]; + assert!( + drain_seam_body.contains("apply_request_effects"), + "drain_receive_pack_requests_with must call apply_request_effects; \ + wiring it to a per-ref helper reintroduces the v29 walk" + ); + + // The deleted per-ref drain must have zero call sites in the + // production half. A regression that re-adds a caller would + // bring back the per-ref fan-out. + assert!( + !production_outbox.contains("drain_pending_ref_transitions_all("), + "deleted `drain_pending_ref_transitions_all` must have zero production call sites; \ + a per-ref walk is reintroduced" + ); + assert!( + !production_outbox.contains("derive_one("), + "deleted `derive_one` must have zero production call sites; \ + the per-ref fan-out is reintroduced" + ); + assert!( + !production_outbox.contains("lookup_accepted_ordinal("), + "deleted `lookup_accepted_ordinal` must have zero production call sites; \ + the per-ref ordinal lookup is reintroduced" + ); + assert!( + !production_repos.contains("derive_one("), + "deleted `derive_one` must have zero live-handler call sites" + ); + assert!( + !production_repos.contains("drain_pending_ref_transitions"), + "deleted per-ref drain functions must have zero live-handler call sites" + ); +} + +/// #26 Split PR 1 step 4 — the periodic queue-lifecycle purge is +/// wired in `main.rs` and the contract is enforced by the `idx_receive_pack_requests_completed_at` +/// partial index from v30. +/// +/// Assertions: +/// 1. `main.rs` calls `purge_request_queue` once at boot (well, on +/// the spawn-task interval) and the spawn function exists. +/// 2. The DB helpers `purge_completed_receive_pack_requests` and +/// `purge_completed_pending_ref_transitions` exist in `db/mod.rs`. +/// 3. The config knobs `queue_retention_days` and `queue_purge_batch` exist +/// on `Config`. +/// 4. `quarantined` is NOT in the purge WHERE clause (step 4 does not +/// introduce the state, but the invariant holds for the future). +#[test] +fn inv26_step4_queue_lifecycle_purge_is_wired() { + let main_src = src("main.rs"); + assert!( + main_src.contains("spawn_queue_lifecycle_sweep"), + "main.rs must spawn the periodic queue-lifecycle purge" + ); + assert!( + main_src.contains("purge_request_queue"), + "main.rs must call purge_request_queue on the periodic sweep" + ); + + let db_src = src("db/mod.rs"); + assert!( + db_src.contains("purge_completed_receive_pack_requests"), + "db/mod.rs must expose purge_completed_receive_pack_requests" + ); + assert!( + db_src.contains("purge_completed_pending_ref_transitions"), + "db/mod.rs must expose purge_completed_pending_ref_transitions" + ); + + let config_src = src("config.rs"); + assert!( + config_src.contains("queue_retention_days"), + "Config must expose queue_retention_days" + ); + assert!( + config_src.contains("queue_purge_batch"), + "Config must expose queue_purge_batch" + ); +} + +/// #26 Split PR 1 step 5 — the marker gate and the retry-bound +/// quarantine are wired end-to-end. A missing or hash-mismatched +/// marker on disk quarantines the request; a Retry over the bound +/// does too. This gate pins every load-bearing seam, against the +/// production half of each file (the test modules name the same +/// identifiers in their own harnesses). +/// +/// Assertions: +/// 1. `reconcile_prepared_page` calls `mark_request_quarantined` +/// and `mark_children_rejected_for_quarantined_parent` on +/// marker-gate failure. +/// 2. The drain's `EffectsOutcome::Retry` arm checks +/// `effects_max_attempts`. +/// 3. `git::store::read_ref` exists in `git/store.rs`. +/// 4. `db::mark_request_quarantined`, +/// `db::mark_children_rejected_for_quarantined_parent`, +/// `db::get_receive_pack_requests_by_ids`, and +/// `db::request_state::QUARANTINED` all exist. +/// 5. The handler writes the marker ref BEFORE calling +/// `smart_http::receive_pack_raw` (the durability window). +#[test] +fn inv26_step5_marker_quarantine_and_bound_are_wired() { + let outbox = src("durable_outbox.rs"); + let store = src("git/store.rs"); + let db = src("db/mod.rs"); + let repos = src("api/repos.rs"); + + // Split at the TEST MODULE for the production-only assertions. + let production_outbox = outbox + .split("\nmod drain_tests {") + .next() + .expect("split always yields a first chunk"); + let production_repos = repos + .split("\nmod tests {") + .next() + .expect("split always yields a first chunk"); + + // (1) The reconcile's marker gate quarantines via the DB helpers. + assert!( + production_outbox.contains("mark_request_quarantined"), + "reconcile_prepared_page must call mark_request_quarantined on marker-gate failure" + ); + assert!( + production_outbox.contains("mark_children_rejected_for_quarantined_parent"), + "reconcile_prepared_page must call mark_children_rejected_for_quarantined_parent \ + so quarantined parents cancel their children" + ); + + // The gate reads the marker ref and compares against the parent's + // `request_bytes_hash` via `git::store::read_ref` / + // `git::store::marker_value_for`. Reverting either reintroduces + // the DoS window the marker gate exists to close. + assert!( + production_outbox.contains("git::store::read_ref"), + "reconcile_prepared_page must read the marker ref via git::store::read_ref" + ); + assert!( + production_outbox.contains("marker_value_for"), + "reconcile_prepared_page must compute the expected marker value via \ + git::store::marker_value_for" + ); + + // (2) The drain's `EffectsOutcome::Retry` arm checks the bound. + assert!( + production_outbox.contains("effects_max_attempts"), + "drain_receive_pack_requests_with must consult effects_max_attempts on Retry" + ); + + // (3) `git::store::read_ref` is the read seam the gate depends on. + assert!( + store.contains("pub fn read_ref("), + "git::store::read_ref must exist; the marker gate reads through it" + ); + assert!( + store.contains("pub fn marker_value_for("), + "git::store::marker_value_for must exist; the marker gate computes the expected \ + value with it (and the live handler writes the value via the same helper)" + ); + + // (4) DB-side seams the gate depends on. + assert!( + db.contains("pub async fn mark_request_quarantined"), + "Db::mark_request_quarantined must exist" + ); + assert!( + db.contains("pub async fn mark_children_rejected_for_quarantined_parent"), + "Db::mark_children_rejected_for_quarantined_parent must exist" + ); + assert!( + db.contains("pub async fn get_receive_pack_requests_by_ids"), + "Db::get_receive_pack_requests_by_ids must exist (avoids N+1 in the marker gate)" + ); + assert!( + db.contains("pub const QUARANTINED: &str = \"quarantined\""), + "request_state::QUARANTINED must be defined" + ); + + // (5) The handler writes the marker ref BEFORE the durability + // boundary (smart_http::receive_pack_raw). Severing the + // ordering re-opens the marker-gate DoS window for live pushes. + let marker_write = production_repos + .find("git::store::marker_value_for") + .expect("U5 gate stale: the live handler no longer computes the marker value"); + let receive_raw = production_repos + .find("smart_http::receive_pack_raw(") + .expect("U5 gate stale: git_receive_pack no longer calls smart_http::receive_pack_raw"); + assert!( + marker_write < receive_raw, + "U5 gate bypassed: the marker ref must be written BEFORE receive_pack_raw so the \ + reconcile's gate has evidence of the live push" + ); +}