fix(devin): wait out a stated rate-limit reset and replay the turn

scripts/test-layout/layout.json

@@ -1527,7 +1527,10 @@
     "ws-native-steering.test.ts": "responses",
     "ws-steering-stability.test.ts": "responses",
     "ws-steering-completion.test.ts": "responses",
-    "ws-steering-smoke.test.ts": "responses"
+    "ws-steering-smoke.test.ts": "responses",
+    "combo-authoritative-reset.test.ts": "codex-integration",
+    "devin-stated-reset-hardening.test.ts": "providers",
+    "devin-stated-reset-retry.test.ts": "providers"
   },
   "migrated": [
     "adapters",

src/adapters/devin.ts

@@ -9,7 +9,7 @@
 import type { AdapterEvent, OcxAssistantMessage, OcxContentPart, OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxTool, OcxToolCall, OcxToolResultMessage, OcxUsage } from "../types";
 import { namespacedToolName } from "../types";
 import type { IncomingMeta, ProviderAdapter } from "./base";
-import { streamChatEvents, allocateCascadeId, CloudChatError, type ChatHistoryItem, type ToolDef } from "./devin/cloud-direct";
+import { streamChatEventsWithResetRetry, allocateCascadeId, CloudChatError, type ChatHistoryItem, type ToolDef } from "./devin/cloud-direct";
 import type { ContentPart } from "./devin/cloud-direct/chat";
 import { getCachedCatalog, type CacheEntry } from "./devin/cloud-direct/catalog";
 import { collapseDevinModelUid } from "./devin/live-models";
@@ -563,7 +563,11 @@ export function createDevinAdapter(
         const maxInputTokens = resolveDevinMaxInputTokens(
           provider, modelUid, catalog?.byUid.get(modelUid)?.contextWindow,
         );
-        for await (const event of streamChatEvents({
+        // The reset-retry wrapper waits out a 429 that states its own recovery
+        // delay ("limit will reset in 35 seconds") and replays the identical
+        // request — but only while zero events have been yielded, so a
+        // post-output failure still takes the terminal path untouched.
+        for await (const event of streamChatEventsWithResetRetry({
           apiKey,
           apiServerUrl: host,
           modelUid,

src/adapters/devin/cloud-direct/index.ts

@@ -49,6 +49,13 @@ export {
   type ToolDef,
 } from './chat.js';
 
+export {
+  streamChatEventsWithResetRetry,
+  STATED_RESET_MAX_REPLAYS,
+  STATED_RESET_MAX_WAIT_MS,
+  type StatedResetRetryOptions,
+} from './stated-reset-retry.js';
+
 export {
   mintUserJwt,
   getCachedUserJwt,

src/adapters/devin/cloud-direct/stated-reset-retry.ts

@@ -0,0 +1,103 @@
+/**
+ * Same-target retry of an explicit, pre-output 429 refusal with a stated
+ * recovery delay. No event-producing attempt is ever automatically replayed.
+ * This is a refusal-specific policy, not a claim that every eventless POST
+ * is idempotent; ambiguous transport failures still propagate unchanged.
+ */
+import { parseRetryAfterFromMessage } from '../../../lib/retry-delay.js';
+import { abortError, sleepWithAbort } from '../../../lib/upstream-retry.js';
+import { CloudChatError, streamChatEvents, type CloudChatEvent, type CloudChatRequest } from './chat.js';
+
+/** 1 initial attempt plus at most 2 replays. */
+export const STATED_RESET_MAX_REPLAYS = 2;
+/** Default cumulative wait allowance for one invocation (30 minutes). */
+export const STATED_RESET_MAX_WAIT_MS = 1_800_000;
+/** Absolute maximum cumulative allowance, including explicit overrides. */
+export const STATED_RESET_WAIT_CEILING_MS = 3_600_000;
+
+function statedResetMaxWaitMs(): number {
+  const raw = process.env.OPENCODEX_DEVIN_STATED_RESET_WAIT_MS?.trim();
+  if (!raw) return STATED_RESET_MAX_WAIT_MS;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed) || parsed < 0) return STATED_RESET_MAX_WAIT_MS;
+  // Zero explicitly disables local waiting. Values above one hour are capped.
+  return Math.min(Math.floor(parsed), STATED_RESET_WAIT_CEILING_MS);
+}
+export const statedResetMaxWaitMsForTests = statedResetMaxWaitMs;
+
+export interface StatedResetRetryOptions {
+  /** Test seam: defaults to the real cloud stream. */
+  stream?: (req: CloudChatRequest) => AsyncGenerator<CloudChatEvent>;
+  /** Test seam: must either honour the whole delay or reject on cancellation. */
+  sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
+  maxReplays?: number;
+  /** CUMULATIVE wait allowance, not a fresh allowance on every failure. */
+  maxWaitMs?: number;
+}
+
+function replayLimit(value: number | undefined): number {
+  if (value === undefined) return STATED_RESET_MAX_REPLAYS;
+  if (!Number.isInteger(value) || value < 0 || value > STATED_RESET_MAX_REPLAYS) {
+    throw new RangeError('maxReplays must be an integer from 0 to 2');
+  }
+  return value;
+}
+
+function waitLimit(value: number | undefined): number {
+  if (value === undefined) return statedResetMaxWaitMs();
+  if (!Number.isFinite(value) || value < 0) {
+    throw new RangeError('maxWaitMs must be a finite non-negative number');
+  }
+  return Math.min(Math.floor(value), STATED_RESET_WAIT_CEILING_MS);
+}
+
+export async function* streamChatEventsWithResetRetry(
+  req: CloudChatRequest,
+  options?: StatedResetRetryOptions,
+): AsyncGenerator<CloudChatEvent> {
+  const stream = options?.stream ?? streamChatEvents;
+  const sleep = options?.sleep ?? sleepWithAbort;
+  const maxReplays = replayLimit(options?.maxReplays);
+  const maxWaitMs = waitLimit(options?.maxWaitMs);
+  let replays = 0;
+  let waitedMs = 0;
+  while (true) {
+    // Check again after sleeping: cancellation can race with timer completion.
+    // A pre-aborted request must not even enter a custom transport.
+    if (req.signal?.aborted) throw abortError(req.signal);
+    let yielded = false;
+    try {
+      for await (const event of stream(req)) {
+        // Latch before yielding, so a consumer-injected error is post-output.
+        yielded = true;
+        yield event;
+      }
+      return;
+    } catch (error) {
+      if (req.signal?.aborted) throw abortError(req.signal);
+      const waitSec = !yielded
+        && error instanceof CloudChatError
+        && error.status === 429
+        ? parseRetryAfterFromMessage(error.message)
+        : undefined;
+      const waitMs = waitSec === undefined ? undefined : waitSec * 1000;
+      if (
+        waitMs === undefined
+        || replays >= maxReplays
+        || waitMs > maxWaitMs - waitedMs
+      ) {
+        // Never shorten a provider's minimum delay to fit the local budget.
+        // Keep the original refusal so outer policy can preserve its metadata.
+        throw error;
+      }
+      replays += 1;
+      // Charge the complete scheduled wait once, before sleeping. This is a
+      // sleep allowance, not a wall-clock deadline on generation or timer
+      // scheduling: waking a few milliseconds late must not reject an already
+      // approved one-hour retry. No later wait can spend this allowance again.
+      waitedMs += waitMs;
+      await sleep(waitMs, req.signal);
+      if (req.signal?.aborted) throw abortError(req.signal);
+    }
+  }
+}

src/combos/failover.ts

@@ -118,23 +118,29 @@ function parseHttpDate(value: string, now: number): number | undefined {
 export function parseRetryAfterMs(
   value: string | null | undefined,
   now = Date.now(),
-  options?: { preserveImmediate?: boolean },
+  options?: { preserveImmediate?: boolean; preserveServerDelay?: boolean },
 ): number | undefined {
   const text = value?.trim();
   if (!text) return undefined;
+  // A local wait ceiling must not make an explicit upstream reset expire early.
+  // Keep legacy bounded parsing for other callers. The opt-in stores a timestamp;
+  // the combo picker still independently limits how long a live request waits.
+  const maximum = options?.preserveServerDelay === true
+    ? Number.MAX_SAFE_INTEGER - Math.max(0, now)
+    : MAX_COOLDOWN_MS;
   if (/^\d+(?:\.\d+)?$/.test(text)) {
     const seconds = Number(text);
     if (
       Number.isFinite(seconds)
       && (seconds > 0 || (options?.preserveImmediate && seconds === 0))
     ) {
-      return Math.min(Math.max(Math.ceil(seconds * 1000), 1), MAX_COOLDOWN_MS);
+      return Math.min(Math.max(Math.ceil(seconds * 1000), 1), maximum);
     }
   }
   const timestamp = parseHttpDate(text, now);
   if (timestamp === undefined) return undefined;
   const delay = timestamp - now;
-  if (delay > 0) return Math.min(delay, MAX_COOLDOWN_MS);
+  if (delay > 0) return Math.min(delay, maximum);
   return options?.preserveImmediate ? 1 : undefined;
 }
 
@@ -215,7 +221,11 @@ export function coolComboTarget(
   // A server-provided Retry-After is authoritative, including an immediate `0` directive.
   // A quota reset is the next-most-specific signal (#3256); configured and default cooldowns
   // are only fallbacks when upstream supplied neither usable value.
-  const cooldownMs = parseRetryAfterMs(options?.retryAfter, now, { preserveImmediate: true })
+  const serverDelayMs = parseRetryAfterMs(options?.retryAfter, now, {
+    preserveImmediate: true,
+    preserveServerDelay: true,
+  });
+  const cooldownMs = serverDelayMs
     ?? parseResetCooldownMs(options?.resetAt, now)
     ?? options?.cooldownMs
     ?? (isTransientRequestRateLimit({
@@ -224,7 +234,9 @@ export function coolComboTarget(
       message: options?.message,
     }) ? COMBO_REQUEST_RATE_COOLDOWN_MS : DEFAULT_COOLDOWN_MS);
   targetCooldowns.set(cooldownMapKey(comboId, target), {
-    cooldownUntil: now + Math.min(Math.max(cooldownMs, 1), MAX_COOLDOWN_MS),
+    // Only the locally chosen fallback is capped at ten minutes. An explicit
+    // server lower bound (including one hour) remains authoritative.
+    cooldownUntil: now + (serverDelayMs ?? Math.min(Math.max(cooldownMs, 1), MAX_COOLDOWN_MS)),
   });
   sweepExpiredOnWrite(now);
 }

src/lib/errors.ts

@@ -1,3 +1,5 @@
+import { parseRetryAfterFromMessage } from "./retry-delay";
+
 export interface OcxErrorPayload {
   message: string;
   type: string;
@@ -376,21 +378,7 @@ export function isRateLimitOrQuotaFailureMessage(message: string): boolean {
   return normalized.toLowerCase().includes("usage limit");
 }
 
-/** Best-effort parse of a retry delay embedded in an upstream error message. */
-export function parseRetryAfterFromMessage(message: string): number | undefined {
-  const patterns = [
-    /try again in (\d+(?:\.\d+)?)\s*s(?:ec(?:ond)?s?)?/i,
-    /retry after (\d+(?:\.\d+)?)\s*s(?:ec(?:ond)?s?)?/i,
-    /retry[- ]after[:\s]+(\d+)/i,
-  ];
-  for (const pattern of patterns) {
-    const match = message.match(pattern);
-    if (!match?.[1]) continue;
-    const seconds = Number.parseFloat(match[1]);
-    if (Number.isFinite(seconds) && seconds > 0) return Math.ceil(seconds);
-  }
-  return undefined;
-}
+export { parseRetryAfterFromMessage };
 
 /** Infer HTTP status from adapter terminal error text (provider-agnostic keyword matching). */
 export function inferHttpStatusFromAdapterMessage(message: string): number {

src/lib/retry-delay.ts

@@ -0,0 +1,69 @@
+/**
+ * Parse a provider's relative retry hint without mistaking a unit prefix for a
+ * complete unit. Kept independent of error classification and combo routing.
+ */
+const UNIT_SECONDS: Readonly<Record<string, number>> = {
+  ms: 0.001, msec: 0.001, msecs: 0.001, millisecond: 0.001, milliseconds: 0.001,
+  s: 1, sec: 1, secs: 1, second: 1, seconds: 1,
+  m: 60, min: 60, mins: 60, minute: 60, minutes: 60,
+  h: 3600, hr: 3600, hrs: 3600, hour: 3600, hours: 3600,
+  d: 86400, day: 86400, days: 86400,
+};
+
+// Read the WHOLE word before looking it up. In particular, ms is not m and
+// "months" must not accidentally become minutes. Digits may follow a unit so
+// compact, unambiguous durations such as 1h30m remain supported.
+const COMPONENT = /^(\d+(?:\.\d+)?)\s*([a-z]+)(?![a-z_])/i;
+const SEPARATOR = /^(?:\s*,\s*(?:and\s+)?|\s+and\s+|\s*\+\s*|\s*)/i;
+const MAX_COMPONENTS = 16;
+
+function durationSeconds(tail: string, allowBareSeconds: boolean): number | undefined {
+  let rest = tail.trimStart();
+  let seconds = 0;
+  let components = 0;
+  while (true) {
+    const component = COMPONENT.exec(rest);
+    if (!component) {
+      if (components !== 0 || !allowBareSeconds) return undefined;
+      // A header-style bare number means seconds. Never salvage the numeric
+      // prefix of an unsupported unit, exponent, signed value or clock time.
+      const bare = /^(\d+(?:\.\d+)?)(?![\w.:+-])(?=\s*(?:$|[.,;!?)\]}]))/.exec(rest);
+      if (!bare) return undefined;
+      seconds = Number(bare[1]);
+      break;
+    }
+    const unit = UNIT_SECONDS[component[2]!.toLowerCase()];
+    if (unit === undefined) return undefined;
+    seconds += Number(component[1]) * unit;
+    if (!Number.isFinite(seconds) || ++components > MAX_COMPONENTS) return undefined;
+    rest = rest.slice(component[0].length);
+    const separator = SEPARATOR.exec(rest)![0];
+    const next = rest.slice(separator.length);
+    if (!/^[+-]?(?:\d|\.\d)/.test(next)) break;
+    // A numeric continuation is part of this duration; a malformed second
+    // component must reject the hint, not silently shorten it to the first.
+    rest = next;
+  }
+  const rounded = Math.ceil(seconds);
+  return Number.isSafeInteger(rounded) && rounded > 0 ? rounded : undefined;
+}
+
+/**
+ * Supports reset(s) in, try again in and Retry-After/retry after hints; accepts
+ * compound durations and rounds UP once after summing all components.
+ * A bare number is permitted only for header-style Retry-After hints, never
+ * for "reset in 2026". When a message declares several usable lower bounds,
+ * honour the longest one rather than re-entering a still-live quota window.
+ */
+export function parseRetryAfterFromMessage(message: string): number | undefined {
+  const hints = /\b(try\s+again\s+in|retry[- ]after|resets?\s+in)\s*:?\s*/gi;
+  let result: number | undefined;
+  for (const hint of message.matchAll(hints)) {
+    const seconds = durationSeconds(
+      message.slice(hint.index! + hint[0].length),
+      /^retry[- ]after$/i.test(hint[1]!),
+    );
+    if (seconds !== undefined) result = Math.max(result ?? 0, seconds);
+  }
+  return result;
+}

tests/codex-integration/combo-authoritative-reset.test.ts

@@ -0,0 +1,42 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import {
+  clearComboTargetCooldowns,
+  coolComboTarget,
+  earliestComboCooldownExpiry,
+  isComboTargetInCooldown,
+  parseRetryAfterMs,
+} from "../../src/combos/failover";
+
+const target = { provider: "devin", model: "swe-2" };
+const combo = "stated-reset-hardening-test";
+const now = Date.UTC(2026, 8, 18, 8, 0, 0);
+afterEach(() => clearComboTargetCooldowns(combo));
+
+describe("explicit server cooldown versus local wait allowance", () => {
+  test("keeps legacy bounded parsing unless preserving a server lower bound", () => {
+    expect(parseRetryAfterMs("3600", now)).toBe(600_000);
+    expect(parseRetryAfterMs("3600", now, { preserveServerDelay: true })).toBe(3_600_000);
+  });
+
+  test.each([301, 780, 1260, 3600, 7200])("keeps the full %i-second server delay", seconds => {
+    coolComboTarget(combo, target, { now, retryAfter: String(seconds) });
+    expect(earliestComboCooldownExpiry(combo, [target], now)).toBe(now + seconds * 1000);
+    expect(isComboTargetInCooldown(combo, target, now + seconds * 1000 - 1)).toBe(true);
+    expect(isComboTargetInCooldown(combo, target, now + seconds * 1000)).toBe(false);
+  });
+
+  test("HTTP-date reset also keeps the full hour", () => {
+    coolComboTarget(combo, target, { now, retryAfter: new Date(now + 3_600_000).toUTCString() });
+    expect(earliestComboCooldownExpiry(combo, [target], now)).toBe(now + 3_600_000);
+  });
+
+  test("a configured fallback is still bounded to ten minutes", () => {
+    coolComboTarget(combo, target, { now, cooldownMs: 99_000_000 });
+    expect(earliestComboCooldownExpiry(combo, [target], now)).toBe(now + 600_000);
+  });
+
+  test("explicit immediate retry stays immediate", () => {
+    coolComboTarget(combo, target, { now, retryAfter: "0" });
+    expect(earliestComboCooldownExpiry(combo, [target], now)).toBe(now + 1);
+  });
+});

tests/fixtures/test-layout-expected.json

@@ -1359,5 +1359,8 @@
   "ws-native-steering.test.ts": "responses",
   "ws-steering-stability.test.ts": "responses",
   "ws-steering-completion.test.ts": "responses",
-  "ws-steering-smoke.test.ts": "responses"
+  "ws-steering-smoke.test.ts": "responses",
+  "combo-authoritative-reset.test.ts": "codex-integration",
+  "devin-stated-reset-hardening.test.ts": "providers",
+  "devin-stated-reset-retry.test.ts": "providers"
 }