Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 61 additions & 4 deletions crates/limen-audit/src/ship.rs
Original file line number Diff line number Diff line change
Expand Up @@ -133,12 +133,23 @@ pub async fn run_consumer<F, Fut>(
};
match outcome {
Ok(()) => {
backoff.reset();
if let Err(e) = spool.commit(&config.name, upto) {
tracing::error!(consumer = %config.name, error = %e, "spool commit failed; the batch will be shipped again");
}
metrics::counter!("limen_audit_shipped_total", "consumer" => config.name.clone())
.increment(count as u64);
if let Err(e) = spool.commit(&config.name, upto) {
// The offset did not move, so the next read returns the
// same batch. Back off rather than re-send it in a loop.
let mut delay = backoff.next_delay(rand::random());
if *shutdown.borrow() {
delay = delay.min(Duration::from_secs(1));
}
tracing::error!(consumer = %config.name, error = %e, retry_in = ?delay, "spool commit failed; the batch will be shipped again");
tokio::select! {
() = tokio::time::sleep(delay) => {}
() = flipped(&mut deadline, false) => {}
}
} else {
backoff.reset();
}
}
Err(ShipError::Retry(reason)) => {
let mut delay = backoff.next_delay(rand::random());
Expand Down Expand Up @@ -229,6 +240,52 @@ mod tests {
assert_eq!(spool.pending("out"), 0);
}

/// A send that succeeds but can't be committed leaves the offset where
/// it was. The batch must then be retried with backoff, not re-sent in a
/// tight loop that floods the sink with duplicates. Multi-threaded, so a
/// regression (a loop that never yields) fails the assertion instead of
/// starving the test's own timer.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_failed_commit_backs_off_instead_of_resending_at_once() {
let dir = tempfile::tempdir().expect("dir");
let mut c = SpoolConfig::new(dir.path(), &["out"]);
c.fsync_interval = Duration::from_millis(2);
let spool = Spool::open(c).expect("open");
spool.start();
spool.append(b"one").await.expect("append");
// Make the offsets directory unusable, so every commit fails.
let offsets = dir.path().join("offsets");
std::fs::rename(&offsets, dir.path().join("offsets.moved")).expect("move offsets");
std::fs::write(&offsets, b"not a directory").expect("block offsets");

let sends = Arc::new(AtomicUsize::new(0));
let (_tx, rx) = tokio::sync::watch::channel(false);
let (deadline_tx, deadline_rx) = tokio::sync::watch::channel(false);
let s = Arc::clone(&sends);
let task = tokio::spawn(run_consumer(
spool.clone(),
ConsumerConfig {
name: "out".into(),
batch_max: 10,
linger: Duration::ZERO,
},
move |_batch| {
s.fetch_add(1, Ordering::SeqCst);
async { Ok(()) }
},
rx,
deadline_rx,
));
tokio::time::sleep(Duration::from_millis(300)).await;
let _ = deadline_tx.send(true);
let _ = task.await;
let sent = sends.load(Ordering::SeqCst);
// With full-jitter backoff a few early retries can be short, but a
// tight loop would send thousands of times in 300ms.
assert!((1..=10).contains(&sent), "sent {sent} times");
assert_eq!(spool.pending("out"), 1, "the record is still unshipped");
}

/// The backlog gauge must count records waiting out the linger, not only
/// what is left after a send: otherwise a replica holding acked,
/// unshipped events reports 0 for the whole flush interval.
Expand Down
33 changes: 33 additions & 0 deletions crates/limen-ca/src/authority.rs
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,15 @@ impl CertificateAuthority {
pub async fn issue_component(&self, req: ComponentCertRequest) -> Result<IssuedCert, CaError> {
self.require(CaPurpose::Internal)?;
validate_name("component id", &req.id)?;
for (key, value) in &req.labels {
validate_name("label key", key)?;
// As for CI facts: an empty value would write a label SAN that
// does not parse, so the certificate could never verify.
if value.is_empty() || value.len() > MAX_CI_VALUE || value.chars().any(char::is_control)
{
return Err(CaError::InvalidRequest(format!("invalid label {key}")));
}
}
let csr = parse_csr(&req.csr_der)?;

let mut params = leaf_params(req.ttl)?;
Expand Down Expand Up @@ -548,6 +557,30 @@ mod tests {
}
}

#[tokio::test]
async fn component_labels_must_round_trip_through_the_certificate() {
let internal = ca(CaPurpose::Internal).await;
let mut ok = component("w1", &[]);
ok.labels = [("zone".to_owned(), "eu-west-1a".to_owned())].into();
let issued = internal.issue_component(ok).await.expect("issue");
let identity = crate::verify::verify_component(&issued.der, internal.cert_der(), false)
.expect("a labelled component cert verifies");
assert_eq!(
identity.labels.get("zone").map(String::as_str),
Some("eu-west-1a")
);
for (key, value) in [
("zone", String::new()),
("zone", "a\nb".to_owned()),
("zone", "a".repeat(MAX_CI_VALUE + 1)),
("", "x".to_owned()),
] {
let mut bad = component("w1", &[]);
bad.labels = [(key.to_owned(), value)].into();
assert!(is_invalid(&internal.issue_component(bad).await), "{key:?}");
}
}

#[tokio::test]
async fn component_dns_names_must_be_valid() {
let internal = ca(CaPurpose::Internal).await;
Expand Down
29 changes: 27 additions & 2 deletions crates/limen-control-plane/src/idp_check.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,13 +160,21 @@ pub async fn note_user(
.map(|_| ())
}

/// How long a claim holds its rows, in seconds. It must outlast a whole
/// batch of slow calls, so it is spread only upward (by up to 10%): a
/// downward spread could let another replica re-claim the tail of a batch
/// still being checked.
fn claim_secs(interval: Duration) -> f64 {
let claim_for = interval.max(CALL_TIMEOUT * u32::try_from(BATCH).unwrap_or(50) + CALL_TIMEOUT);
claim_for.as_secs_f64() * (1.0 + rand::random::<f64>() * 0.1)
}

/// Claim the logins due for a check, pushing their next check out first, so
/// replicas never take the same login and a crash mid-check only delays it.
async fn claim_due(db: &Db, interval: Duration) -> Result<Vec<Tracked>, sqlx::Error> {
// Long enough that a whole batch of slow calls finishes before another
// replica can claim the same rows: two redemptions of one rotating token
// would make the second look like a refusal.
let claim_for = interval.max(CALL_TIMEOUT * u32::try_from(BATCH).unwrap_or(50) + CALL_TIMEOUT);
sqlx::query(
"UPDATE idp_logins SET next_check_at = now() + make_interval(secs => $2)
WHERE login_id IN (
Expand All @@ -175,7 +183,7 @@ async fn claim_due(db: &Db, interval: Duration) -> Result<Vec<Tracked>, sqlx::Er
RETURNING login_id, username, refresh_token, groups, expires_at, last_ok_at",
)
.bind(BATCH)
.bind(jittered_secs(claim_for))
.bind(claim_secs(interval))
.fetch_all(db)
.await?
.iter()
Expand Down Expand Up @@ -854,6 +862,23 @@ mod tests {

const KEY: [u8; 32] = [7; 32];

#[test]
fn a_claim_always_outlasts_a_whole_batch_of_slow_calls() {
let batch =
(CALL_TIMEOUT * u32::try_from(BATCH).expect("batch") + CALL_TIMEOUT).as_secs_f64();
for interval in [Duration::from_secs(60), Duration::from_secs(300)] {
for _ in 0..1000 {
let c = claim_secs(interval);
assert!(c >= batch && c <= batch * 1.1, "{c} for {interval:?}");
}
}
// A longer interval is the floor when it exceeds the batch time.
let long = Duration::from_secs(3600);
for _ in 0..1000 {
assert!(claim_secs(long) >= long.as_secs_f64());
}
}

#[test]
fn a_sealed_token_opens_only_with_its_key_and_login() {
let sealed = seal(&KEY, "login-1", "refresh-secret").expect("seal");
Expand Down
22 changes: 21 additions & 1 deletion crates/limen-verify/src/bucket.rs
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,8 @@ impl Tally {
missing_to: seq - 1,
});
}
next = seq + 1;
// `seq` comes from the bucket, so it may be anything.
next = seq.saturating_add(1);
}
}
gaps
Expand Down Expand Up @@ -570,6 +571,25 @@ mod tests {
);
}

/// `seq` is read from the bucket, so a tampered or corrupt line can say
/// anything, `u64::MAX` included: that is a gap, never a panic.
#[test]
fn the_largest_seq_is_a_gap_not_an_overflow() {
let mut t = Tally::default();
for l in [line("p/a", 0), line("p/a", u64::MAX)] {
t.add(&l).expect("event");
}
assert_eq!(
t.gaps(),
vec![SeqGap {
component: "proxy".into(),
instance_id: "p/a".into(),
missing_from: 1,
missing_to: u64::MAX - 1,
}]
);
}

/// One producer can't give one `seq` to two events.
#[test]
fn a_seq_used_by_two_events_is_a_conflict() {
Expand Down
5 changes: 4 additions & 1 deletion demos/demo-phase4.sh
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,11 @@ print(" denials:", dict(reasons))
denied = next(e for e in events if e["kind"] == "auth.denied" and e["data"].get("reason") == "no bot matches this token" and e["data"]["ci"].get("ref") == "refs/heads/feature")
print(" the refused run is recorded:", denied["data"]["ci"]["repository"], denied["data"]["ci"]["ref"])
PY
# Capture the logs first: with pipefail, `logs | grep -q` returns SIGPIPE's
# 141 when grep finds a match early, and the leak would go unreported.
service_logs="$("${compose[@]}" logs --no-color control-plane proxy worker 2>&1)"
for f in "$state"/phase4/token-*.jwt; do
if "${compose[@]}" logs --no-color control-plane proxy worker 2>&1 | grep -qF "$(cat "$f")"; then fail "a token appears in the service logs: $f"; fi
if grep -qF -- "$(cat "$f")" <<<"$service_logs"; then fail "a token appears in the service logs: $f"; fi
done
pass "audited with the CI run; no token in the trail or in the logs"

Expand Down
4 changes: 3 additions & 1 deletion demos/demo-phase6.sh
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,9 @@ print(f" db.query={len(queries)} db.query.end={len(qends)} db.bootstrap={len(bo
for e in (pg, my):
print(f" {e['data']['protocol']:8} {e['user']} {e['data']['db_role']}@{e['data']['database']}: {e['data']['statement']}")
PY
if "${compose[@]}" logs worker 2>&1 | grep -qF "$pw_pg"; then fail "a password is in the worker log"; fi
# Capture first: under pipefail an early grep match makes `logs | grep -q` fail with SIGPIPE.
worker_logs="$("${compose[@]}" logs worker 2>&1)"
if grep -qF -- "$pw_pg" <<<"$worker_logs"; then fail "a password is in the worker log"; fi
pass "sessions, statements, outcomes, and bootstrap changes are in the trail, linked by id; no password anywhere"

printf '\n\033[32mPhase 6 demo passed.\033[0m\n'
7 changes: 5 additions & 2 deletions demos/demo-public-cert.sh
Original file line number Diff line number Diff line change
Expand Up @@ -162,8 +162,11 @@ step "A broken replacement is refused and the last good certificate stays"
good_key="$(cat "$pub/key.pem")"
printf 'not a key\n' >"$pub/key.pem"
docker kill -s HUP "$proxy_container" >/dev/null
for _ in $(seq 1 30); do docker logs "$proxy_container" 2>&1 | grep -q "public certificate not reloaded" && break; sleep 1; done
docker logs "$proxy_container" 2>&1 | grep -q "public certificate not reloaded" || fail "no warning about the bad key"
# Capture the logs before grepping: under pipefail an early `grep -q` match
# makes `docker logs | grep -q` exit 141 (SIGPIPE) and read as "not found".
warned() { grep -q "public certificate not reloaded" <<<"$(docker logs "$proxy_container" 2>&1)"; }
for _ in $(seq 1 30); do warned && break; sleep 1; done
warned || fail "no warning about the bad key"
[ "$(presented serial h2)" = "$after" ] || fail "the certificate changed even though the key was bad"
limen status >/dev/null || fail "the CLI should still work"
curl -sf "http://localhost:$LIMEN_PORT_HEALTH/metrics" | grep -q "limen_proxy_public_cert_reload_failures_total" || fail "no failure counter"
Expand Down
Loading