From 8ce997ccd8677edc3f90f8bd7ebd9067c0d8d898 Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Mon, 31 Aug 2026 21:23:56 +0800 Subject: [PATCH] fix(github): auto-refreshing installation token + mention resolution backoff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by live acceptance: one hour after serve start, every GitHub API call began failing 401 'Bad credentials' — the installation client was built once with a fixed personal_token (installation tokens expire after 1h) and never refreshed, silently wedging mention resolution, reactions, and the write outbox. The client now uses octocrab's installation auth state, which caches and refreshes tokens with an expiry buffer. Also: mention-authority resolution retried every 250ms tick on persistent failure (~4 rps against a dead credential); it now backs off exponentially (2s doubling to a 60s cap) and stops iterating the batch on first error. --- CHANGELOG.md | 12 ++++++++++++ src/github.rs | 12 +++++++----- src/producer/ingress.rs | 40 ++++++++++++++++++++++++++++++---------- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 277d05d..8997bf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,18 @@ All notable changes to Braid are recorded here. The project follows Semantic Versioning once release artifacts are published. +## [0.3.1] - unreleased + +### Fixed + +- The GitHub installation client no longer pins the initial installation + token: it is built via octocrab's installation auth state, which caches and + auto-refreshes the token. Previously every API call began failing with 401 + "Bad credentials" one hour after `serve` started (token expiry), silently + wedging mention resolution, reactions, and the write outbox until restart. +- Mention-authority resolution now backs off exponentially (2s to 60s) on + persistent GitHub errors instead of retrying every 250ms scheduler tick. + ## [0.3.0] - 2026-08-31 ### Added diff --git a/src/github.rs b/src/github.rs index b5346b1..f6b873d 100644 --- a/src/github.rs +++ b/src/github.rs @@ -168,13 +168,16 @@ impl GitHubClient { } let installation = app.apps().get_repository_installation(&repository.owner, &repository.name).await?; - let installation_id = installation.id.into_inner(); + // Keep the raw id for the auto-refreshing installation client; a + // fixed personal_token client would die permanently when the token + // expires after one hour. + let installation_raw_id = installation.id; + let installation_id = installation_raw_id.into_inner(); let access: AccessTokenResponse = app .post(&format!("/app/installations/{installation_id}/access_tokens"), None::<&()>) .await?; - let installation_client = Octocrab::builder() - .personal_token(access.token.clone()) - .build() + let installation_client = app + .installation(installation_raw_id) .map_err(|error| GitHubError::Client(error.to_string()))?; let repository_info = repository_identity(&installation_client, repository).await?; let actor = viewer_identity(&installation_client).await?; @@ -600,7 +603,6 @@ struct GraphQlError { #[derive(Deserialize)] struct AccessTokenResponse { - token: String, expires_at: String, #[serde(default)] permissions: BTreeMap, diff --git a/src/producer/ingress.rs b/src/producer/ingress.rs index 65e2f7c..8b9a30e 100644 --- a/src/producer/ingress.rs +++ b/src/producer/ingress.rs @@ -105,6 +105,11 @@ pub(crate) async fn event_worker( ) { let mut tick = tokio::time::interval(Duration::from_millis(250)); tick.set_missed_tick_behavior(MissedTickBehavior::Delay); + // Mention-authority resolution talks to GitHub; on persistent failure + // (e.g. token expiry before the client refreshes) back off exponentially + // instead of hammering the API every tick. + let mut mention_failures: u32 = 0; + let mut mention_cooldown_until = tokio::time::Instant::now(); loop { tokio::select! { _ = shutdown.changed() => break, @@ -112,21 +117,36 @@ pub(crate) async fn event_worker( if let Err(error) = store.advance_scheduler() { tracing::error!(%error, "cannot advance scheduler"); } - match store.mention_candidates(16) { - Ok(candidates) => { - for candidate in candidates { - match github.repository_permission(&candidate.actor_login).await { - Ok(role) => { - let trusted = matches!(role.to_ascii_lowercase().as_str(), "maintain" | "admin"); - if let Err(error) = store.resolve_mention(candidate.event_id, trusted, policy) { - tracing::error!(%error, "cannot resolve mention authority"); + if tokio::time::Instant::now() >= mention_cooldown_until { + match store.mention_candidates(16) { + Ok(candidates) => { + let mut failed = false; + for candidate in candidates { + match github.repository_permission(&candidate.actor_login).await { + Ok(role) => { + let trusted = matches!(role.to_ascii_lowercase().as_str(), "maintain" | "admin"); + if let Err(error) = store.resolve_mention(candidate.event_id, trusted, policy) { + tracing::error!(%error, "cannot resolve mention authority"); + } + } + Err(error) => { + tracing::warn!(%error, actor = %candidate.actor_login, "mention authority remains unresolved"); + failed = true; + break; } } - Err(error) => tracing::warn!(%error, actor = %candidate.actor_login, "mention authority remains unresolved"), + } + if failed { + mention_failures = (mention_failures + 1).min(6); + let backoff = Duration::from_secs(2u64.pow(mention_failures).min(60)); + mention_cooldown_until = tokio::time::Instant::now() + backoff; + tracing::debug!(?backoff, "mention authority resolution backing off"); + } else { + mention_failures = 0; } } + Err(error) => tracing::error!(%error, "cannot load mention candidates"), } - Err(error) => tracing::error!(%error, "cannot load mention candidates"), } drain_one_write(&store, &github).await; }