fix(codex): stop retrying a doomed pool credential refresh, and say it was local (#4546)

scripts/test-layout/layout.json

@@ -1450,7 +1450,8 @@
     "adapter-input-media-guard.test.ts": "adapters",
     "chat-media-translation.test.ts": "responses",
     "execution-budget-permits.test.ts": "lib",
-    "spend-instrumentation-log.test.ts": "server"
+    "spend-instrumentation-log.test.ts": "server",
+    "codex-pool-refresh-backoff.test.ts": "codex-integration"
   },
   "migrated": [
     "adapters",

src/codex/account-lifecycle.ts

@@ -11,6 +11,8 @@ import { getMainChatgptAccountId, readCodexTokensResult } from "./auth-collision
 import { MAIN_CODEX_ACCOUNT_ID, setMainAccountPlan } from "./main-account";
 import { clearAccountQuota } from "./quota";
 import { clearCodexUpstreamHealthForAccount, clearThreadAccountMapForAccount } from "./routing";
+
+import { clearCodexPoolRefreshFailure } from "./pool-refresh-backoff";
 import { invalidateCodexWebSocketsForAccount } from "./websocket-registry";
 import { clearMainAccountCredentialPresence, clearMainAccountInfoCache, observeMainQuotaCredential, observeMainQuotaIdentity } from "./main-account-cache";
 import { extractAccountIdClaims } from "../oauth/chatgpt";
@@ -41,6 +43,7 @@ export function purgeCodexAccountRuntimeState(accountId: string): void {
   clearAccountQuota(accountId);
   clearThreadAccountMapForAccount(accountId);
   clearCodexUpstreamHealthForAccount(accountId);
+  clearCodexPoolRefreshFailure(accountId);
   if (accountId === MAIN_CODEX_ACCOUNT_ID) {
     clearMainAccountInfoCache();
     clearMainAccountCredentialPresence();

src/codex/account-store.ts

@@ -17,6 +17,13 @@ import { isValidCodexAccountId } from "./account-id";
 import type { PoolQuotaWriter } from "./quota-types";
 import { CODEX_REFRESH_FLIGHT_CEILING_MS } from "./quota-recovery-timing";
 
+import {
+  CodexPoolRefreshCooldownError,
+  clearCodexPoolRefreshFailure,
+  isCodexPoolRefreshCooling,
+  noteCodexPoolRefreshFailure,
+} from "./pool-refresh-backoff";
+
 type LegacyCodexAccountStore = Record<string, CodexAccountCredentials>;
 type CodexAccountStore = Record<string, CodexAccountCredentialRecord>;
 type RawCodexAccountStore = Record<string, CodexAccountCredentials | CodexAccountCredentialRecord>;
@@ -480,6 +487,17 @@ export class TokenRefreshError extends Error {
   }
 }
 
+/**
+ * The stored record or its refresh-grant fingerprint is gone. Retrying cannot
+ * conjure a missing credential, so callers must treat this as terminal.
+ */
+export class CodexCredentialUnavailableError extends Error {
+  constructor(message = "Codex account credential is unavailable; reauthenticate the account.") {
+    super(message);
+    this.name = "CodexCredentialUnavailableError";
+  }
+}
+
 export class CodexCredentialGenerationConflictError extends Error {
   constructor(message = "Codex account changed during refresh") {
     super(message);
@@ -514,6 +532,30 @@ export class CodexCredentialRefreshStaleError extends Error {
   }
 }
 
+/**
+ * Terminal means the grant itself is dead, or there is no grant to refresh.
+ * Token-endpoint 5xx (`unknown`) and a generation CAS loss stay transient
+ * because those genuinely may clear (#2887).
+ */
+export function isTerminalCodexPoolRefreshFailure(error: unknown): boolean {
+  return (error instanceof TokenRefreshError && (error.reason === "revoked" || error.reason === "expired"))
+    || error instanceof CodexCredentialUnavailableError;
+}
+
+function isOperationalCodexPoolRefreshFailure(error: unknown): boolean {
+  if (error instanceof CodexPoolRefreshCooldownError) return true;
+  if (error instanceof CodexCredentialRefreshBusyError) return true;
+  if (error instanceof CodexCredentialRefreshStaleError) return true;
+  if (error instanceof CodexCredentialRefreshLockTimeoutError) return true;
+  return error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError");
+}
+
+function classifyCodexPoolRefreshFailureReason(error: unknown): string {
+  if (error instanceof TokenRefreshError) return error.reason;
+  if (error instanceof CodexCredentialGenerationConflictError) return "generation_conflict";
+  return "network";
+}
+
 /** Credential writers share the config mutation coordinator; contention is transient, not reauth. */
 function withCredentialMutationLockSync<T>(fn: () => T): T {
   try {
@@ -797,6 +839,11 @@ export async function forceRefreshCodexPoolToken(
     settle({ kind: "failed", error: options.signal.reason });
     throw options.signal.reason;
   }
+  if (isCodexPoolRefreshCooling(id)) {
+    const error = new CodexPoolRefreshCooldownError();
+    settle({ kind: "failed", error });
+    throw error;
+  }
   const completion = resolveCodexToken(
     id,
     { rejectedGeneration: options.rejectedGeneration, rejectedAccessToken: options.rejectedAccessToken },
@@ -805,13 +852,23 @@ export async function forceRefreshCodexPoolToken(
     undefined,
   );
   completion.then(
-    resolved => settle({
-      kind: "resolved",
-      provenance: classify(resolved),
-      generation: resolved.generation,
-      rotated: resolved.accessToken !== options.rejectedAccessToken,
-    }),
-    error => settle({ kind: "failed", error }),
+    resolved => {
+      clearCodexPoolRefreshFailure(id);
+      settle({
+        kind: "resolved",
+        provenance: classify(resolved),
+        generation: resolved.generation,
+        rotated: resolved.accessToken !== options.rejectedAccessToken,
+      });
+    },
+    error => {
+      if (isTerminalCodexPoolRefreshFailure(error) || isOperationalCodexPoolRefreshFailure(error)) {
+        if (isTerminalCodexPoolRefreshFailure(error)) clearCodexPoolRefreshFailure(id);
+      } else {
+        noteCodexPoolRefreshFailure(id, classifyCodexPoolRefreshFailureReason(error));
+      }
+      settle({ kind: "failed", error });
+    },
   );
   const result = await awaitOwnCancellation(completion, options.signal);
   const provenance = classify(result);
@@ -851,9 +908,9 @@ async function resolveCodexToken(
   if (callerSignal?.aborted) throw callerSignal.reason;
   const record = readCodexAccountRecord(id);
   const cred = record?.deletedAt == null ? record?.credential : undefined;
-  if (!record || !cred) throw new Error("Codex account credential is unavailable; reauthenticate the account.");
+  if (!record || !cred) throw new CodexCredentialUnavailableError();
   const refreshGrantFingerprint = recordGrantFingerprint(record);
-  if (!refreshGrantFingerprint) throw new Error("Codex account credential is unavailable; reauthenticate the account.");
+  if (!refreshGrantFingerprint) throw new CodexCredentialUnavailableError();
 
   // The freshness shortcut is exactly what makes a 401 on a time-valid token
   // unrecoverable, so a forced caller skips it — but only while the stored credential

src/codex/pool-refresh-backoff.ts

@@ -0,0 +1,127 @@
+
+/**
+ * Per-account cooldown for a stored Codex pool credential whose forced refresh
+ * failed without proving the grant is dead.
+ *
+ * A token-endpoint 5xx, a generation CAS loss, or a network blip is transient
+ * (#2887): it must not quarantine the account or drop its binding. Retrying the
+ * same doomed refresh on every request, though, is how a single unhealthy
+ * account pinned the pool at 503 while healthy siblings sat idle. Consecutive
+ * non-terminal failures open a bounded growing cooldown; during that window no
+ * new forced refresh starts, and selection prefers a sibling. The first
+ * successful refresh clears it.
+ */
+
+import { fallbackCodexAccountLogLabel } from "./account-label";
+
+export const CODEX_POOL_REFRESH_INCOMPLETE_LOG_REASON = "codex_pool_refresh_incomplete";
+
+/** Growing delays between forced-refresh attempts for one account. */
+export const CODEX_POOL_REFRESH_FAILURE_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
+
+export class CodexPoolRefreshCooldownError extends Error {
+  readonly retryable = true;
+  readonly code = "CODEX_REFRESH_COOLING";
+
+  constructor(message = "Codex credential refresh is cooling down") {
+    super(message);
+    this.name = "CodexPoolRefreshCooldownError";
+  }
+}
+
+type RefreshFailureBackoff = {
+  consecutiveFailures: number;
+  cooldownUntil: number;
+  reason: string;
+};
+
+const backoffByAccount = new Map<string, RefreshFailureBackoff>();
+let nowOverride: number | undefined;
+
+export function setCodexPoolRefreshFailureNowForTests(now?: number): void {
+  nowOverride = now;
+}
+
+export function resetCodexPoolRefreshFailureBackoffForTests(): void {
+  backoffByAccount.clear();
+  nowOverride = undefined;
+}
+
+export function clearCodexPoolRefreshFailure(accountId: string): void {
+  backoffByAccount.delete(accountId);
+}
+
+/**
+ * Drop every remembered failure. Called when the routing layer discards its per-account state,
+ * because a cooldown outliving the binding it was learned alongside would keep an account out of
+ * selection for a roster the operator has already replaced.
+ */
+export function clearAllCodexPoolRefreshFailures(): void {
+  backoffByAccount.clear();
+}
+
+function currentNow(now?: number): number {
+  return now ?? nowOverride ?? Date.now();
+}
+
+function delayFor(consecutiveFailures: number): number {
+  const index = Math.min(Math.max(consecutiveFailures, 1), CODEX_POOL_REFRESH_FAILURE_BACKOFF_MS.length) - 1;
+  return CODEX_POOL_REFRESH_FAILURE_BACKOFF_MS[index]!;
+}
+
+/**
+ * How many consecutive non-terminal failures must land before a refresh is WITHHELD.
+ *
+ * Withholding on the first failure was wrong twice over. A single token-endpoint blip is the
+ * ordinary case that the very next attempt clears, and -- worse -- a withheld refresh never runs,
+ * so an account whose grant is actually revoked can no longer discover that: the terminal 401 it
+ * owes the operator turns into a retryable 503 that never resolves. The cooldown exists for the
+ * account that keeps failing, not for the one that failed once.
+ */
+export const CODEX_POOL_REFRESH_COOLDOWN_AFTER_FAILURES = 3;
+
+export function getCodexPoolRefreshCooldownUntil(accountId: string, now = currentNow()): number | null {
+  const entry = backoffByAccount.get(accountId);
+  if (!entry) return null;
+  if (entry.consecutiveFailures < CODEX_POOL_REFRESH_COOLDOWN_AFTER_FAILURES) return null;
+  return entry.cooldownUntil > now ? entry.cooldownUntil : null;
+}
+
+export function isCodexPoolRefreshCooling(accountId: string, now = currentNow()): boolean {
+  return getCodexPoolRefreshCooldownUntil(accountId, now) !== null;
+}
+
+/**
+ * Record a non-terminal forced-refresh failure. Already-cooling accounts do not
+ * grow the window: growth requires another real attempt after the previous one
+ * expired. Logs the classified reason once per account per window, with the
+ * durable hash label — never a token and never an email.
+ */
+export function noteCodexPoolRefreshFailure(
+  accountId: string,
+  reason: string,
+  now = currentNow(),
+): { consecutiveFailures: number; cooldownUntil: number; openedWindow: boolean } {
+  const existing = backoffByAccount.get(accountId);
+  // The "do not grow inside an open window" rule applies only once the window is actually
+  // WITHHOLDING. Below the threshold no refresh is being withheld, so every failure is a real
+  // attempt that really failed and must count -- otherwise a client retrying the 503 once a
+  // second can never reach the threshold the cooldown is meant to protect against.
+  const withholding = existing !== undefined
+    && existing.consecutiveFailures >= CODEX_POOL_REFRESH_COOLDOWN_AFTER_FAILURES;
+  if (existing && withholding && existing.cooldownUntil > now) {
+    return {
+      consecutiveFailures: existing.consecutiveFailures,
+      cooldownUntil: existing.cooldownUntil,
+      openedWindow: false,
+    };
+  }
+  const consecutiveFailures = (existing?.consecutiveFailures ?? 0) + 1;
+  const cooldownUntil = now + delayFor(consecutiveFailures);
+  backoffByAccount.set(accountId, { consecutiveFailures, cooldownUntil, reason });
+  const label = fallbackCodexAccountLogLabel(accountId);
+  console.warn(
+    `[codex-auth] Codex pool account ${label} credential refresh failed (${reason})`,
+  );
+  return { consecutiveFailures, cooldownUntil, openedWindow: true };
+}

src/codex/routing.ts

@@ -38,6 +38,8 @@ import { isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers";
 import { retainedUtf8Bytes } from "../lib/admission";
 import { recordUpstreamHostFailure } from "./upstream-host-health";
 
+import { clearAllCodexPoolRefreshFailures, isCodexPoolRefreshCooling } from "./pool-refresh-backoff";
+
 type ThreadAffinityEntry = {
   accountId: string;
   generation: number;
@@ -428,6 +430,9 @@ export function listLiveCodexAccountIds(config: OcxConfig): ReadonlySet<string>
 export function clearThreadAccountMap(): void {
   threadAccountMap.clear();
   threadAffinityEntryTotal = 0;
+  // A refresh cooldown is per-account runtime state learned alongside these bindings. Leaving it
+  // behind here keeps an account out of selection after the roster it belonged to is gone.
+  clearAllCodexPoolRefreshFailures();
 }
 
 export function clearThreadAccountMapForAccount(
@@ -1335,6 +1340,7 @@ function isCodexAccountSelectable(
     && getCodexQuotaHealthSnapshot(accountId, quotaScope, now) === null
     && !isCodexQuotaAvoided(accountId, quotaScope, now)
     && !isCodexAccountSoftAvoided(accountId, now)
+    && !isCodexPoolRefreshCooling(accountId, now)
     && isCodexAccountUsable(config, accountId, selectionOptions);
 }
 
@@ -1360,6 +1366,7 @@ function codexAccountBlockReason(
   if (getCodexQuotaHealthSnapshot(accountId, quotaScope, now) !== null) return "cooldown";
   if (isCodexQuotaAvoided(accountId, quotaScope, now)) return "quota_avoided";
   if (isCodexAccountSoftAvoided(accountId, now)) return "transient";
+  if (isCodexPoolRefreshCooling(accountId, now)) return "transient";
   if (!isCodexAccountUsable(config, accountId, selectionOptions)) return "unusable";
   return undefined;
 }
@@ -1600,6 +1607,7 @@ function getEligiblePoolAccounts(
     .filter(account => getCodexQuotaHealthSnapshot(account.id, quotaScope, now) === null)
     .filter(account => !isCodexAccountSoftAvoided(account.id, now))
     .filter(account => !isCodexQuotaAvoided(account.id, quotaScope, now))
+    .filter(account => !isCodexPoolRefreshCooling(account.id, now))
     .filter(account => isCodexAccountUsable(config, account.id, selectionOptions))
     .map(account => account.id);
   // The main Codex account is not stored in config.codexAccounts; include it as a
@@ -1616,6 +1624,7 @@ function getEligiblePoolAccounts(
     // earned it: the cooldown caps at fifteen minutes, the window runs up to six hours, and
     // in between the main account returns as a first-class candidate.
     && !isCodexQuotaAvoided(MAIN_CODEX_ACCOUNT_ID, quotaScope, now)
+    && !isCodexPoolRefreshCooling(MAIN_CODEX_ACCOUNT_ID, now)
     && (!skipFailoverReadyCandidates || !shouldFailover(config, MAIN_CODEX_ACCOUNT_ID, now))
     && isCodexAccountUsable(config, MAIN_CODEX_ACCOUNT_ID, selectionOptions)
   ) {
@@ -1725,7 +1734,9 @@ function isTransientOnlyAffinityBlock(
   if (!isCodexAccountUsable(config, entry.accountId, selectionOptions)) return false;
   if (getCodexQuotaHealthSnapshot(entry.accountId, quotaScope, now) !== null) return false;
   if (isCodexQuotaAvoided(entry.accountId, quotaScope, now)) return false;
-  return shouldFailover(config, entry.accountId, now) || isCodexAccountSoftAvoided(entry.accountId, now);
+  return shouldFailover(config, entry.accountId, now)
+    || isCodexAccountSoftAvoided(entry.accountId, now)
+    || isCodexPoolRefreshCooling(entry.accountId, now);
 }
 
 /** Has a held binding waited longer than a transient failure can reasonably explain? */

src/server/request-log.ts

@@ -254,7 +254,11 @@ export interface RequestLogEntry {
   affinityReason?: CodexAffinityReason;
   /** Where the upstream terminal/failure was observed. */
   transportPhase?: "pre_headers" | "mid_stream" | "terminal_sse";
-  /** Whether the terminal came from a real upstream SSE event or a proxy synthetic tail. */
+  /**
+   * Whether the HTTP status and message originated upstream or were synthesized by this
+   * proxy. Covers SSE tails and pre-stream JSON refusals. Management surfaces this so a
+   * local refusal cannot be presented as an upstream reason.
+   */
   terminalSource?: "upstream" | "synthetic";
   /** Bounded route-decision trace (RI-01); never contains secrets. */
   routeDecision?: RouteDecisionTraceV1;
@@ -815,6 +819,15 @@ export function usageFromResponsesPayload(usage: unknown): OcxUsage | undefined
   return undefined;
 }
 
+/**
+ * Mark a refusal this proxy synthesized locally. Sets origin to `synthetic` and a
+ * distinct local reason so the request log cannot be read as an upstream overload.
+ */
+export function markLocalRequestLogRefusal(logCtx: RequestLogContext, reason: string): void {
+  logCtx.localTerminalReason = reason;
+  logCtx.terminalSource = "synthetic";
+}
+
 export function inspectResponseLogJson(logCtx: RequestLogContext, text: string): void {
   try {
     applyResponseLogMetadata(logCtx, JSON.parse(text));