From 59f3a2a9f96a730c0110e48be55a7c602a1378d6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 14 Nov 2026 09:40:46 +0000 Subject: [PATCH 1/5] fix(demos): capture logs before grep so leak checks can't miss a match Under `set -o pipefail`, `logs | grep -q X` returns 141 when grep matches early and the writer gets SIGPIPE, so the `if` reads a found secret as "not found". The phase 4 token check and phase 6 password check could therefore pass with a secret in the logs; the public-cert demo had the mirror problem (a spurious "no warning" failure). Grep a captured copy. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SEiXYZ8UQNGWJJHgaqCU64 --- demos/demo-phase4.sh | 5 ++++- demos/demo-phase6.sh | 4 +++- demos/demo-public-cert.sh | 7 +++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/demos/demo-phase4.sh b/demos/demo-phase4.sh index b5024ee8..0e3535bc 100755 --- a/demos/demo-phase4.sh +++ b/demos/demo-phase4.sh @@ -189,8 +189,11 @@ print(" denials:", dict(reasons)) denied = next(e for e in events if e["kind"] == "auth.denied" and e["data"].get("reason") == "no bot matches this token" and e["data"]["ci"].get("ref") == "refs/heads/feature") print(" the refused run is recorded:", denied["data"]["ci"]["repository"], denied["data"]["ci"]["ref"]) PY +# Capture the logs first: with pipefail, `logs | grep -q` returns SIGPIPE's +# 141 when grep finds a match early, and the leak would go unreported. +service_logs="$("${compose[@]}" logs --no-color control-plane proxy worker 2>&1)" for f in "$state"/phase4/token-*.jwt; do - if "${compose[@]}" logs --no-color control-plane proxy worker 2>&1 | grep -qF "$(cat "$f")"; then fail "a token appears in the service logs: $f"; fi + if grep -qF -- "$(cat "$f")" <<<"$service_logs"; then fail "a token appears in the service logs: $f"; fi done pass "audited with the CI run; no token in the trail or in the logs" diff --git a/demos/demo-phase6.sh b/demos/demo-phase6.sh index d567ef18..99d27cab 100755 --- a/demos/demo-phase6.sh +++ b/demos/demo-phase6.sh @@ -204,7 +204,9 @@ print(f" db.query={len(queries)} db.query.end={len(qends)} db.bootstrap={len(bo for e in (pg, my): print(f" {e['data']['protocol']:8} {e['user']} {e['data']['db_role']}@{e['data']['database']}: {e['data']['statement']}") PY -if "${compose[@]}" logs worker 2>&1 | grep -qF "$pw_pg"; then fail "a password is in the worker log"; fi +# Capture first: under pipefail an early grep match makes `logs | grep -q` fail with SIGPIPE. +worker_logs="$("${compose[@]}" logs worker 2>&1)" +if grep -qF -- "$pw_pg" <<<"$worker_logs"; then fail "a password is in the worker log"; fi pass "sessions, statements, outcomes, and bootstrap changes are in the trail, linked by id; no password anywhere" printf '\n\033[32mPhase 6 demo passed.\033[0m\n' diff --git a/demos/demo-public-cert.sh b/demos/demo-public-cert.sh index e9215a38..e58bcf2b 100755 --- a/demos/demo-public-cert.sh +++ b/demos/demo-public-cert.sh @@ -162,8 +162,11 @@ step "A broken replacement is refused and the last good certificate stays" good_key="$(cat "$pub/key.pem")" printf 'not a key\n' >"$pub/key.pem" docker kill -s HUP "$proxy_container" >/dev/null -for _ in $(seq 1 30); do docker logs "$proxy_container" 2>&1 | grep -q "public certificate not reloaded" && break; sleep 1; done -docker logs "$proxy_container" 2>&1 | grep -q "public certificate not reloaded" || fail "no warning about the bad key" +# Capture the logs before grepping: under pipefail an early `grep -q` match +# makes `docker logs | grep -q` exit 141 (SIGPIPE) and read as "not found". +warned() { grep -q "public certificate not reloaded" <<<"$(docker logs "$proxy_container" 2>&1)"; } +for _ in $(seq 1 30); do warned && break; sleep 1; done +warned || fail "no warning about the bad key" [ "$(presented serial h2)" = "$after" ] || fail "the certificate changed even though the key was bad" limen status >/dev/null || fail "the CLI should still work" curl -sf "http://localhost:$LIMEN_PORT_HEALTH/metrics" | grep -q "limen_proxy_public_cert_reload_failures_total" || fail "no failure counter" From 6c3c5bb9bf976a2bd7143a98ea0d11a493e483a9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 14 Nov 2026 09:42:12 +0000 Subject: [PATCH 2/5] fix(ca): refuse component labels that would make an unverifiable cert `Labels` allows an empty value, but `IdentityUri::parse` rejects an empty segment, so a join token labelled `{"zone": ""}` enrolled a component whose certificate then failed every verification as "malformed identity URI". Validate label keys and values in `issue_component` as `issue_user` already does for CI facts; the refusal happens before the join token is consumed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SEiXYZ8UQNGWJJHgaqCU64 --- crates/limen-ca/src/authority.rs | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/crates/limen-ca/src/authority.rs b/crates/limen-ca/src/authority.rs index 79a9f78e..3e26e6ac 100644 --- a/crates/limen-ca/src/authority.rs +++ b/crates/limen-ca/src/authority.rs @@ -218,6 +218,15 @@ impl CertificateAuthority { pub async fn issue_component(&self, req: ComponentCertRequest) -> Result { self.require(CaPurpose::Internal)?; validate_name("component id", &req.id)?; + for (key, value) in &req.labels { + validate_name("label key", key)?; + // As for CI facts: an empty value would write a label SAN that + // does not parse, so the certificate could never verify. + if value.is_empty() || value.len() > MAX_CI_VALUE || value.chars().any(char::is_control) + { + return Err(CaError::InvalidRequest(format!("invalid label {key}"))); + } + } let csr = parse_csr(&req.csr_der)?; let mut params = leaf_params(req.ttl)?; @@ -548,6 +557,30 @@ mod tests { } } + #[tokio::test] + async fn component_labels_must_round_trip_through_the_certificate() { + let internal = ca(CaPurpose::Internal).await; + let mut ok = component("w1", &[]); + ok.labels = [("zone".to_owned(), "eu-west-1a".to_owned())].into(); + let issued = internal.issue_component(ok).await.expect("issue"); + let identity = crate::verify::verify_component(&issued.der, internal.cert_der(), false) + .expect("a labelled component cert verifies"); + assert_eq!( + identity.labels.get("zone").map(String::as_str), + Some("eu-west-1a") + ); + for (key, value) in [ + ("zone", String::new()), + ("zone", "a\nb".to_owned()), + ("zone", "a".repeat(MAX_CI_VALUE + 1)), + ("", "x".to_owned()), + ] { + let mut bad = component("w1", &[]); + bad.labels = [(key.to_owned(), value)].into(); + assert!(is_invalid(&internal.issue_component(bad).await), "{key:?}"); + } + } + #[tokio::test] async fn component_dns_names_must_be_valid() { let internal = ca(CaPurpose::Internal).await; From 73edcf33e0e6351c9c9ea8d9b46c5b1afb0dd4af Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 14 Nov 2026 09:53:37 +0000 Subject: [PATCH 3/5] fix(audit): back off when the spool commit fails after a send A send that succeeded but could not be committed reset the backoff and looped straight back: the offset hadn't moved, so the same batch was re-sent immediately, forever, while the commit kept failing (a full disk, an unwritable offsets directory). The new test saw ~40k sends in 300 ms. Retry with the shipper's jittered backoff instead, and reset it only once a commit lands. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SEiXYZ8UQNGWJJHgaqCU64 --- crates/limen-audit/src/ship.rs | 65 +++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/crates/limen-audit/src/ship.rs b/crates/limen-audit/src/ship.rs index 0e9a67ac..ca12cd92 100644 --- a/crates/limen-audit/src/ship.rs +++ b/crates/limen-audit/src/ship.rs @@ -133,12 +133,23 @@ pub async fn run_consumer( }; match outcome { Ok(()) => { - backoff.reset(); - if let Err(e) = spool.commit(&config.name, upto) { - tracing::error!(consumer = %config.name, error = %e, "spool commit failed; the batch will be shipped again"); - } metrics::counter!("limen_audit_shipped_total", "consumer" => config.name.clone()) .increment(count as u64); + if let Err(e) = spool.commit(&config.name, upto) { + // The offset did not move, so the next read returns the + // same batch. Back off rather than re-send it in a loop. + let mut delay = backoff.next_delay(rand::random()); + if *shutdown.borrow() { + delay = delay.min(Duration::from_secs(1)); + } + tracing::error!(consumer = %config.name, error = %e, retry_in = ?delay, "spool commit failed; the batch will be shipped again"); + tokio::select! { + () = tokio::time::sleep(delay) => {} + () = flipped(&mut deadline, false) => {} + } + } else { + backoff.reset(); + } } Err(ShipError::Retry(reason)) => { let mut delay = backoff.next_delay(rand::random()); @@ -229,6 +240,52 @@ mod tests { assert_eq!(spool.pending("out"), 0); } + /// A send that succeeds but can't be committed leaves the offset where + /// it was. The batch must then be retried with backoff, not re-sent in a + /// tight loop that floods the sink with duplicates. Multi-threaded, so a + /// regression (a loop that never yields) fails the assertion instead of + /// starving the test's own timer. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_failed_commit_backs_off_instead_of_resending_at_once() { + let dir = tempfile::tempdir().expect("dir"); + let mut c = SpoolConfig::new(dir.path(), &["out"]); + c.fsync_interval = Duration::from_millis(2); + let spool = Spool::open(c).expect("open"); + spool.start(); + spool.append(b"one").await.expect("append"); + // Make the offsets directory unusable, so every commit fails. + let offsets = dir.path().join("offsets"); + std::fs::rename(&offsets, dir.path().join("offsets.moved")).expect("move offsets"); + std::fs::write(&offsets, b"not a directory").expect("block offsets"); + + let sends = Arc::new(AtomicUsize::new(0)); + let (_tx, rx) = tokio::sync::watch::channel(false); + let (deadline_tx, deadline_rx) = tokio::sync::watch::channel(false); + let s = Arc::clone(&sends); + let task = tokio::spawn(run_consumer( + spool.clone(), + ConsumerConfig { + name: "out".into(), + batch_max: 10, + linger: Duration::ZERO, + }, + move |_batch| { + s.fetch_add(1, Ordering::SeqCst); + async { Ok(()) } + }, + rx, + deadline_rx, + )); + tokio::time::sleep(Duration::from_millis(300)).await; + let _ = deadline_tx.send(true); + let _ = task.await; + let sent = sends.load(Ordering::SeqCst); + // With full-jitter backoff a few early retries can be short, but a + // tight loop would send thousands of times in 300ms. + assert!((1..=10).contains(&sent), "sent {sent} times"); + assert_eq!(spool.pending("out"), 1, "the record is still unshipped"); + } + /// The backlog gauge must count records waiting out the linger, not only /// what is left after a send: otherwise a replica holding acked, /// unshipped events reports 0 for the whole flush interval. From c317ff9662bf4fe81144a8d5e217ec43a747fb36 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 14 Nov 2026 09:58:19 +0000 Subject: [PATCH 4/5] fix(idp-check): only spread the batch claim upward MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The claim on a batch of logins is sized to outlast 50 sequential IdP calls, so another replica can't re-claim and redeem the same rotating refresh token mid-batch (the second redemption reads as a refusal and locks the person). It was passed through the ±10% check jitter, which could shorten it to 90% of that bound. Spread it by 0..+10% instead. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SEiXYZ8UQNGWJJHgaqCU64 --- crates/limen-control-plane/src/idp_check.rs | 29 +++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/crates/limen-control-plane/src/idp_check.rs b/crates/limen-control-plane/src/idp_check.rs index b394dbd8..7f68aa5d 100644 --- a/crates/limen-control-plane/src/idp_check.rs +++ b/crates/limen-control-plane/src/idp_check.rs @@ -160,13 +160,21 @@ pub async fn note_user( .map(|_| ()) } +/// How long a claim holds its rows, in seconds. It must outlast a whole +/// batch of slow calls, so it is spread only upward (by up to 10%): a +/// downward spread could let another replica re-claim the tail of a batch +/// still being checked. +fn claim_secs(interval: Duration) -> f64 { + let claim_for = interval.max(CALL_TIMEOUT * u32::try_from(BATCH).unwrap_or(50) + CALL_TIMEOUT); + claim_for.as_secs_f64() * (1.0 + rand::random::() * 0.1) +} + /// Claim the logins due for a check, pushing their next check out first, so /// replicas never take the same login and a crash mid-check only delays it. async fn claim_due(db: &Db, interval: Duration) -> Result, sqlx::Error> { // Long enough that a whole batch of slow calls finishes before another // replica can claim the same rows: two redemptions of one rotating token // would make the second look like a refusal. - let claim_for = interval.max(CALL_TIMEOUT * u32::try_from(BATCH).unwrap_or(50) + CALL_TIMEOUT); sqlx::query( "UPDATE idp_logins SET next_check_at = now() + make_interval(secs => $2) WHERE login_id IN ( @@ -175,7 +183,7 @@ async fn claim_due(db: &Db, interval: Duration) -> Result, sqlx::Er RETURNING login_id, username, refresh_token, groups, expires_at, last_ok_at", ) .bind(BATCH) - .bind(jittered_secs(claim_for)) + .bind(claim_secs(interval)) .fetch_all(db) .await? .iter() @@ -854,6 +862,23 @@ mod tests { const KEY: [u8; 32] = [7; 32]; + #[test] + fn a_claim_always_outlasts_a_whole_batch_of_slow_calls() { + let batch = + (CALL_TIMEOUT * u32::try_from(BATCH).expect("batch") + CALL_TIMEOUT).as_secs_f64(); + for interval in [Duration::from_secs(60), Duration::from_secs(300)] { + for _ in 0..1000 { + let c = claim_secs(interval); + assert!(c >= batch && c <= batch * 1.1, "{c} for {interval:?}"); + } + } + // A longer interval is the floor when it exceeds the batch time. + let long = Duration::from_secs(3600); + for _ in 0..1000 { + assert!(claim_secs(long) >= long.as_secs_f64()); + } + } + #[test] fn a_sealed_token_opens_only_with_its_key_and_login() { let sealed = seal(&KEY, "login-1", "refresh-secret").expect("seal"); From 4f23a549d8e1edc9ba417e3ec2573a73b1a1297a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 14 Nov 2026 09:58:19 +0000 Subject: [PATCH 5/5] fix(verify): don't overflow on a seq of u64::MAX from the bucket `seq` is read from event lines in S3, which a tamperer can write. Gap detection did `seq + 1`, which panics in debug builds and wraps in release. Saturate instead; the line still shows up as a gap. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SEiXYZ8UQNGWJJHgaqCU64 --- crates/limen-verify/src/bucket.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/limen-verify/src/bucket.rs b/crates/limen-verify/src/bucket.rs index 5b6ae322..66383624 100644 --- a/crates/limen-verify/src/bucket.rs +++ b/crates/limen-verify/src/bucket.rs @@ -156,7 +156,8 @@ impl Tally { missing_to: seq - 1, }); } - next = seq + 1; + // `seq` comes from the bucket, so it may be anything. + next = seq.saturating_add(1); } } gaps @@ -570,6 +571,25 @@ mod tests { ); } + /// `seq` is read from the bucket, so a tampered or corrupt line can say + /// anything, `u64::MAX` included: that is a gap, never a panic. + #[test] + fn the_largest_seq_is_a_gap_not_an_overflow() { + let mut t = Tally::default(); + for l in [line("p/a", 0), line("p/a", u64::MAX)] { + t.add(&l).expect("event"); + } + assert_eq!( + t.gaps(), + vec![SeqGap { + component: "proxy".into(), + instance_id: "p/a".into(), + missing_from: 1, + missing_to: u64::MAX - 1, + }] + ); + } + /// One producer can't give one `seq` to two events. #[test] fn a_seq_used_by_two_events_is_a_conflict() {