diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 162b4ba20f..ac18b1ad04 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1527,7 +1527,10 @@ "ws-native-steering.test.ts": "responses", "ws-steering-stability.test.ts": "responses", "ws-steering-completion.test.ts": "responses", - "ws-steering-smoke.test.ts": "responses" + "ws-steering-smoke.test.ts": "responses", + "combo-authoritative-reset.test.ts": "codex-integration", + "devin-stated-reset-hardening.test.ts": "providers", + "devin-stated-reset-retry.test.ts": "providers" }, "migrated": [ "adapters", diff --git a/src/adapters/devin.ts b/src/adapters/devin.ts index 9b3a971fa7..0ff3b7da33 100644 --- a/src/adapters/devin.ts +++ b/src/adapters/devin.ts @@ -9,7 +9,7 @@ import type { AdapterEvent, OcxAssistantMessage, OcxContentPart, OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxTool, OcxToolCall, OcxToolResultMessage, OcxUsage } from "../types"; import { namespacedToolName } from "../types"; import type { IncomingMeta, ProviderAdapter } from "./base"; -import { streamChatEvents, allocateCascadeId, CloudChatError, type ChatHistoryItem, type ToolDef } from "./devin/cloud-direct"; +import { streamChatEventsWithResetRetry, allocateCascadeId, CloudChatError, type ChatHistoryItem, type ToolDef } from "./devin/cloud-direct"; import type { ContentPart } from "./devin/cloud-direct/chat"; import { getCachedCatalog, type CacheEntry } from "./devin/cloud-direct/catalog"; import { collapseDevinModelUid } from "./devin/live-models"; @@ -563,7 +563,11 @@ export function createDevinAdapter( const maxInputTokens = resolveDevinMaxInputTokens( provider, modelUid, catalog?.byUid.get(modelUid)?.contextWindow, ); - for await (const event of streamChatEvents({ + // The reset-retry wrapper waits out a 429 that states its own recovery + // delay ("limit will reset in 35 seconds") and replays the identical + // request — but only while zero events have been yielded, so a + // post-output failure still takes the terminal path untouched. + for await (const event of streamChatEventsWithResetRetry({ apiKey, apiServerUrl: host, modelUid, diff --git a/src/adapters/devin/cloud-direct/index.ts b/src/adapters/devin/cloud-direct/index.ts index 98dfcea9de..f4dbab563f 100644 --- a/src/adapters/devin/cloud-direct/index.ts +++ b/src/adapters/devin/cloud-direct/index.ts @@ -49,6 +49,13 @@ export { type ToolDef, } from './chat.js'; +export { + streamChatEventsWithResetRetry, + STATED_RESET_MAX_REPLAYS, + STATED_RESET_MAX_WAIT_MS, + type StatedResetRetryOptions, +} from './stated-reset-retry.js'; + export { mintUserJwt, getCachedUserJwt, diff --git a/src/adapters/devin/cloud-direct/stated-reset-retry.ts b/src/adapters/devin/cloud-direct/stated-reset-retry.ts new file mode 100644 index 0000000000..f500cd4b19 --- /dev/null +++ b/src/adapters/devin/cloud-direct/stated-reset-retry.ts @@ -0,0 +1,103 @@ +/** + * Same-target retry of an explicit, pre-output 429 refusal with a stated + * recovery delay. No event-producing attempt is ever automatically replayed. + * This is a refusal-specific policy, not a claim that every eventless POST + * is idempotent; ambiguous transport failures still propagate unchanged. + */ +import { parseRetryAfterFromMessage } from '../../../lib/retry-delay.js'; +import { abortError, sleepWithAbort } from '../../../lib/upstream-retry.js'; +import { CloudChatError, streamChatEvents, type CloudChatEvent, type CloudChatRequest } from './chat.js'; + +/** 1 initial attempt plus at most 2 replays. */ +export const STATED_RESET_MAX_REPLAYS = 2; +/** Default cumulative wait allowance for one invocation (30 minutes). */ +export const STATED_RESET_MAX_WAIT_MS = 1_800_000; +/** Absolute maximum cumulative allowance, including explicit overrides. */ +export const STATED_RESET_WAIT_CEILING_MS = 3_600_000; + +function statedResetMaxWaitMs(): number { + const raw = process.env.OPENCODEX_DEVIN_STATED_RESET_WAIT_MS?.trim(); + if (!raw) return STATED_RESET_MAX_WAIT_MS; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed < 0) return STATED_RESET_MAX_WAIT_MS; + // Zero explicitly disables local waiting. Values above one hour are capped. + return Math.min(Math.floor(parsed), STATED_RESET_WAIT_CEILING_MS); +} +export const statedResetMaxWaitMsForTests = statedResetMaxWaitMs; + +export interface StatedResetRetryOptions { + /** Test seam: defaults to the real cloud stream. */ + stream?: (req: CloudChatRequest) => AsyncGenerator; + /** Test seam: must either honour the whole delay or reject on cancellation. */ + sleep?: (ms: number, signal?: AbortSignal) => Promise; + maxReplays?: number; + /** CUMULATIVE wait allowance, not a fresh allowance on every failure. */ + maxWaitMs?: number; +} + +function replayLimit(value: number | undefined): number { + if (value === undefined) return STATED_RESET_MAX_REPLAYS; + if (!Number.isInteger(value) || value < 0 || value > STATED_RESET_MAX_REPLAYS) { + throw new RangeError('maxReplays must be an integer from 0 to 2'); + } + return value; +} + +function waitLimit(value: number | undefined): number { + if (value === undefined) return statedResetMaxWaitMs(); + if (!Number.isFinite(value) || value < 0) { + throw new RangeError('maxWaitMs must be a finite non-negative number'); + } + return Math.min(Math.floor(value), STATED_RESET_WAIT_CEILING_MS); +} + +export async function* streamChatEventsWithResetRetry( + req: CloudChatRequest, + options?: StatedResetRetryOptions, +): AsyncGenerator { + const stream = options?.stream ?? streamChatEvents; + const sleep = options?.sleep ?? sleepWithAbort; + const maxReplays = replayLimit(options?.maxReplays); + const maxWaitMs = waitLimit(options?.maxWaitMs); + let replays = 0; + let waitedMs = 0; + while (true) { + // Check again after sleeping: cancellation can race with timer completion. + // A pre-aborted request must not even enter a custom transport. + if (req.signal?.aborted) throw abortError(req.signal); + let yielded = false; + try { + for await (const event of stream(req)) { + // Latch before yielding, so a consumer-injected error is post-output. + yielded = true; + yield event; + } + return; + } catch (error) { + if (req.signal?.aborted) throw abortError(req.signal); + const waitSec = !yielded + && error instanceof CloudChatError + && error.status === 429 + ? parseRetryAfterFromMessage(error.message) + : undefined; + const waitMs = waitSec === undefined ? undefined : waitSec * 1000; + if ( + waitMs === undefined + || replays >= maxReplays + || waitMs > maxWaitMs - waitedMs + ) { + // Never shorten a provider's minimum delay to fit the local budget. + // Keep the original refusal so outer policy can preserve its metadata. + throw error; + } + replays += 1; + // Charge the complete scheduled wait once, before sleeping. This is a + // sleep allowance, not a wall-clock deadline on generation or timer + // scheduling: waking a few milliseconds late must not reject an already + // approved one-hour retry. No later wait can spend this allowance again. + waitedMs += waitMs; + await sleep(waitMs, req.signal); + if (req.signal?.aborted) throw abortError(req.signal); + } + } +} diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 21de270e93..da9ccdca37 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -118,23 +118,29 @@ function parseHttpDate(value: string, now: number): number | undefined { export function parseRetryAfterMs( value: string | null | undefined, now = Date.now(), - options?: { preserveImmediate?: boolean }, + options?: { preserveImmediate?: boolean; preserveServerDelay?: boolean }, ): number | undefined { const text = value?.trim(); if (!text) return undefined; + // A local wait ceiling must not make an explicit upstream reset expire early. + // Keep legacy bounded parsing for other callers. The opt-in stores a timestamp; + // the combo picker still independently limits how long a live request waits. + const maximum = options?.preserveServerDelay === true + ? Number.MAX_SAFE_INTEGER - Math.max(0, now) + : MAX_COOLDOWN_MS; if (/^\d+(?:\.\d+)?$/.test(text)) { const seconds = Number(text); if ( Number.isFinite(seconds) && (seconds > 0 || (options?.preserveImmediate && seconds === 0)) ) { - return Math.min(Math.max(Math.ceil(seconds * 1000), 1), MAX_COOLDOWN_MS); + return Math.min(Math.max(Math.ceil(seconds * 1000), 1), maximum); } } const timestamp = parseHttpDate(text, now); if (timestamp === undefined) return undefined; const delay = timestamp - now; - if (delay > 0) return Math.min(delay, MAX_COOLDOWN_MS); + if (delay > 0) return Math.min(delay, maximum); return options?.preserveImmediate ? 1 : undefined; } @@ -215,7 +221,11 @@ export function coolComboTarget( // A server-provided Retry-After is authoritative, including an immediate `0` directive. // A quota reset is the next-most-specific signal (#3256); configured and default cooldowns // are only fallbacks when upstream supplied neither usable value. - const cooldownMs = parseRetryAfterMs(options?.retryAfter, now, { preserveImmediate: true }) + const serverDelayMs = parseRetryAfterMs(options?.retryAfter, now, { + preserveImmediate: true, + preserveServerDelay: true, + }); + const cooldownMs = serverDelayMs ?? parseResetCooldownMs(options?.resetAt, now) ?? options?.cooldownMs ?? (isTransientRequestRateLimit({ @@ -224,7 +234,9 @@ export function coolComboTarget( message: options?.message, }) ? COMBO_REQUEST_RATE_COOLDOWN_MS : DEFAULT_COOLDOWN_MS); targetCooldowns.set(cooldownMapKey(comboId, target), { - cooldownUntil: now + Math.min(Math.max(cooldownMs, 1), MAX_COOLDOWN_MS), + // Only the locally chosen fallback is capped at ten minutes. An explicit + // server lower bound (including one hour) remains authoritative. + cooldownUntil: now + (serverDelayMs ?? Math.min(Math.max(cooldownMs, 1), MAX_COOLDOWN_MS)), }); sweepExpiredOnWrite(now); } diff --git a/src/lib/errors.ts b/src/lib/errors.ts index 7c4582c2fe..669e81b21f 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -1,3 +1,5 @@ +import { parseRetryAfterFromMessage } from "./retry-delay"; + export interface OcxErrorPayload { message: string; type: string; @@ -376,21 +378,7 @@ export function isRateLimitOrQuotaFailureMessage(message: string): boolean { return normalized.toLowerCase().includes("usage limit"); } -/** Best-effort parse of a retry delay embedded in an upstream error message. */ -export function parseRetryAfterFromMessage(message: string): number | undefined { - const patterns = [ - /try again in (\d+(?:\.\d+)?)\s*s(?:ec(?:ond)?s?)?/i, - /retry after (\d+(?:\.\d+)?)\s*s(?:ec(?:ond)?s?)?/i, - /retry[- ]after[:\s]+(\d+)/i, - ]; - for (const pattern of patterns) { - const match = message.match(pattern); - if (!match?.[1]) continue; - const seconds = Number.parseFloat(match[1]); - if (Number.isFinite(seconds) && seconds > 0) return Math.ceil(seconds); - } - return undefined; -} +export { parseRetryAfterFromMessage }; /** Infer HTTP status from adapter terminal error text (provider-agnostic keyword matching). */ export function inferHttpStatusFromAdapterMessage(message: string): number { diff --git a/src/lib/retry-delay.ts b/src/lib/retry-delay.ts new file mode 100644 index 0000000000..31251770e2 --- /dev/null +++ b/src/lib/retry-delay.ts @@ -0,0 +1,69 @@ +/** + * Parse a provider's relative retry hint without mistaking a unit prefix for a + * complete unit. Kept independent of error classification and combo routing. + */ +const UNIT_SECONDS: Readonly> = { + ms: 0.001, msec: 0.001, msecs: 0.001, millisecond: 0.001, milliseconds: 0.001, + s: 1, sec: 1, secs: 1, second: 1, seconds: 1, + m: 60, min: 60, mins: 60, minute: 60, minutes: 60, + h: 3600, hr: 3600, hrs: 3600, hour: 3600, hours: 3600, + d: 86400, day: 86400, days: 86400, +}; + +// Read the WHOLE word before looking it up. In particular, ms is not m and +// "months" must not accidentally become minutes. Digits may follow a unit so +// compact, unambiguous durations such as 1h30m remain supported. +const COMPONENT = /^(\d+(?:\.\d+)?)\s*([a-z]+)(?![a-z_])/i; +const SEPARATOR = /^(?:\s*,\s*(?:and\s+)?|\s+and\s+|\s*\+\s*|\s*)/i; +const MAX_COMPONENTS = 16; + +function durationSeconds(tail: string, allowBareSeconds: boolean): number | undefined { + let rest = tail.trimStart(); + let seconds = 0; + let components = 0; + while (true) { + const component = COMPONENT.exec(rest); + if (!component) { + if (components !== 0 || !allowBareSeconds) return undefined; + // A header-style bare number means seconds. Never salvage the numeric + // prefix of an unsupported unit, exponent, signed value or clock time. + const bare = /^(\d+(?:\.\d+)?)(?![\w.:+-])(?=\s*(?:$|[.,;!?)\]}]))/.exec(rest); + if (!bare) return undefined; + seconds = Number(bare[1]); + break; + } + const unit = UNIT_SECONDS[component[2]!.toLowerCase()]; + if (unit === undefined) return undefined; + seconds += Number(component[1]) * unit; + if (!Number.isFinite(seconds) || ++components > MAX_COMPONENTS) return undefined; + rest = rest.slice(component[0].length); + const separator = SEPARATOR.exec(rest)![0]; + const next = rest.slice(separator.length); + if (!/^[+-]?(?:\d|\.\d)/.test(next)) break; + // A numeric continuation is part of this duration; a malformed second + // component must reject the hint, not silently shorten it to the first. + rest = next; + } + const rounded = Math.ceil(seconds); + return Number.isSafeInteger(rounded) && rounded > 0 ? rounded : undefined; +} + +/** + * Supports reset(s) in, try again in and Retry-After/retry after hints; accepts + * compound durations and rounds UP once after summing all components. + * A bare number is permitted only for header-style Retry-After hints, never + * for "reset in 2026". When a message declares several usable lower bounds, + * honour the longest one rather than re-entering a still-live quota window. + */ +export function parseRetryAfterFromMessage(message: string): number | undefined { + const hints = /\b(try\s+again\s+in|retry[- ]after|resets?\s+in)\s*:?\s*/gi; + let result: number | undefined; + for (const hint of message.matchAll(hints)) { + const seconds = durationSeconds( + message.slice(hint.index! + hint[0].length), + /^retry[- ]after$/i.test(hint[1]!), + ); + if (seconds !== undefined) result = Math.max(result ?? 0, seconds); + } + return result; +} diff --git a/tests/codex-integration/combo-authoritative-reset.test.ts b/tests/codex-integration/combo-authoritative-reset.test.ts new file mode 100644 index 0000000000..001d8f8b3b --- /dev/null +++ b/tests/codex-integration/combo-authoritative-reset.test.ts @@ -0,0 +1,42 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + clearComboTargetCooldowns, + coolComboTarget, + earliestComboCooldownExpiry, + isComboTargetInCooldown, + parseRetryAfterMs, +} from "../../src/combos/failover"; + +const target = { provider: "devin", model: "swe-2" }; +const combo = "stated-reset-hardening-test"; +const now = Date.UTC(2026, 8, 18, 8, 0, 0); +afterEach(() => clearComboTargetCooldowns(combo)); + +describe("explicit server cooldown versus local wait allowance", () => { + test("keeps legacy bounded parsing unless preserving a server lower bound", () => { + expect(parseRetryAfterMs("3600", now)).toBe(600_000); + expect(parseRetryAfterMs("3600", now, { preserveServerDelay: true })).toBe(3_600_000); + }); + + test.each([301, 780, 1260, 3600, 7200])("keeps the full %i-second server delay", seconds => { + coolComboTarget(combo, target, { now, retryAfter: String(seconds) }); + expect(earliestComboCooldownExpiry(combo, [target], now)).toBe(now + seconds * 1000); + expect(isComboTargetInCooldown(combo, target, now + seconds * 1000 - 1)).toBe(true); + expect(isComboTargetInCooldown(combo, target, now + seconds * 1000)).toBe(false); + }); + + test("HTTP-date reset also keeps the full hour", () => { + coolComboTarget(combo, target, { now, retryAfter: new Date(now + 3_600_000).toUTCString() }); + expect(earliestComboCooldownExpiry(combo, [target], now)).toBe(now + 3_600_000); + }); + + test("a configured fallback is still bounded to ten minutes", () => { + coolComboTarget(combo, target, { now, cooldownMs: 99_000_000 }); + expect(earliestComboCooldownExpiry(combo, [target], now)).toBe(now + 600_000); + }); + + test("explicit immediate retry stays immediate", () => { + coolComboTarget(combo, target, { now, retryAfter: "0" }); + expect(earliestComboCooldownExpiry(combo, [target], now)).toBe(now + 1); + }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index efdee7fadc..952f8f56df 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1359,5 +1359,8 @@ "ws-native-steering.test.ts": "responses", "ws-steering-stability.test.ts": "responses", "ws-steering-completion.test.ts": "responses", - "ws-steering-smoke.test.ts": "responses" + "ws-steering-smoke.test.ts": "responses", + "combo-authoritative-reset.test.ts": "codex-integration", + "devin-stated-reset-hardening.test.ts": "providers", + "devin-stated-reset-retry.test.ts": "providers" } diff --git a/tests/providers/devin-stated-reset-hardening.test.ts b/tests/providers/devin-stated-reset-hardening.test.ts new file mode 100644 index 0000000000..0dac8ed0e3 --- /dev/null +++ b/tests/providers/devin-stated-reset-hardening.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, test } from "bun:test"; +import { CloudChatError, type CloudChatEvent, type CloudChatRequest } from "../../src/adapters/devin/cloud-direct"; +import { + streamChatEventsWithResetRetry, + statedResetMaxWaitMsForTests, +} from "../../src/adapters/devin/cloud-direct/stated-reset-retry"; + +const request = { + apiKey: "test-only", apiServerUrl: "https://example.invalid", modelUid: "swe-2", messages: [], +} as unknown as CloudChatRequest; + +const cap = (message: string) => new CloudChatError(message, "resource_exhausted", "test", 429); +async function drain(events: AsyncIterable): Promise { + for await (const _event of events) { /* Consume the real wrapper without a live RPC. */ } +} + +describe("Devin cumulative stated-reset allowance", () => { + test("two one-hour refusals cannot turn a one-hour allowance into two hours", async () => { + let calls = 0; + const waits: number[] = []; + const failure = cap("Your limit will reset in 1 hour"); + const stream = async function* (req: CloudChatRequest): AsyncGenerator { + expect(req).toBe(request); + calls += 1; + throw failure; + }; + await expect(drain(streamChatEventsWithResetRetry(request, { + stream, maxWaitMs: 3_600_000, sleep: async ms => { waits.push(ms); }, + }))).rejects.toBe(failure); + expect(calls).toBe(2); + expect(waits).toEqual([3_600_000]); + }); + + test("twenty plus ten minutes fits the default cumulative allowance", async () => { + let calls = 0; + const waits: number[] = []; + const stream = async function* (): AsyncGenerator { + calls += 1; + if (calls === 1) throw cap("reset in 20 minutes"); + if (calls === 2) throw cap("reset in 10 minutes"); + yield { kind: "text", text: "ok" }; + }; + await drain(streamChatEventsWithResetRetry(request, { + stream, maxWaitMs: 1_800_000, sleep: async ms => { waits.push(ms); }, + })); + expect(calls).toBe(3); + expect(waits).toEqual([1_200_000, 600_000]); + }); + + test("a compound duration is never shortened to fit the allowance", async () => { + let calls = 0; + const waits: number[] = []; + const stream = async function* (): AsyncGenerator { + calls += 1; + throw cap("reset in 5 minutes 30 seconds"); + }; + await expect(drain(streamChatEventsWithResetRetry(request, { + stream, maxWaitMs: 300_000, sleep: async ms => { waits.push(ms); }, + }))).rejects.toThrow("5 minutes 30 seconds"); + expect(calls).toBe(1); + expect(waits).toEqual([]); + }); + + test("a pre-aborted request does not enter the stream", async () => { + const controller = new AbortController(); + controller.abort(); + let calls = 0; + await expect(drain(streamChatEventsWithResetRetry({ ...request, signal: controller.signal }, { + stream: async function* (): AsyncGenerator { calls += 1; }, + }))).rejects.toHaveProperty("name", "AbortError"); + expect(calls).toBe(0); + }); + + test("cancellation at the sleep-completion boundary prevents replay", async () => { + const controller = new AbortController(); + let calls = 0; + const stream = async function* (): AsyncGenerator { + calls += 1; + throw cap("reset in 1 second"); + }; + await expect(drain(streamChatEventsWithResetRetry({ ...request, signal: controller.signal }, { + stream, sleep: async () => { controller.abort(); }, + }))).rejects.toHaveProperty("name", "AbortError"); + expect(calls).toBe(1); + }); + + test.each([ + { kind: "text", text: "partial" }, + { kind: "usage", promptTokens: 1 }, + { kind: "tool_call_start", id: "t1", name: "write_file" }, + { kind: "reasoning_signature", signature: "sig" }, + ] as CloudChatEvent[])("never retries after an event: %j", async event => { + let calls = 0; + const stream = async function* (): AsyncGenerator { + calls += 1; + yield event; + throw cap("reset in 1 second"); + }; + await expect(drain(streamChatEventsWithResetRetry(request, { stream }))).rejects.toThrow("reset in 1 second"); + expect(calls).toBe(1); + }); + + test("explicit zero disables waiting; overlarge overrides stay bounded", () => { + const name = "OPENCODEX_DEVIN_STATED_RESET_WAIT_MS"; + const previous = process.env[name]; + try { + process.env[name] = "0"; + expect(statedResetMaxWaitMsForTests()).toBe(0); + process.env[name] = "999999999"; + expect(statedResetMaxWaitMsForTests()).toBe(3_600_000); + } finally { + if (previous === undefined) delete process.env[name]; + else process.env[name] = previous; + } + }); +}); diff --git a/tests/providers/devin-stated-reset-retry.test.ts b/tests/providers/devin-stated-reset-retry.test.ts new file mode 100644 index 0000000000..18e7ef9ce6 --- /dev/null +++ b/tests/providers/devin-stated-reset-retry.test.ts @@ -0,0 +1,178 @@ +/** + * Devin stated-reset replay: a 429 trailer that names its own recovery delay + * ("Your limit will reset in 35 seconds") is waited out and the identical + * request replayed — but only while the stream produced zero events, and only + * within the replay/wait bounds. + */ +import { describe, expect, test } from "bun:test"; +import { CloudChatError, type CloudChatEvent, type CloudChatRequest } from "../../src/adapters/devin/cloud-direct"; +import { streamChatEventsWithResetRetry } from "../../src/adapters/devin/cloud-direct/stated-reset-retry"; + +const REQ = { apiKey: "k", apiServerUrl: "https://example.invalid", modelUid: "swe-2", messages: [] } as unknown as CloudChatRequest; + +function exhausting(message: string): () => AsyncGenerator { + return () => (async function* (): AsyncGenerator { + throw new CloudChatError(message, "resource_exhausted", "t", 429); + })(); +} + +async function* events(...items: CloudChatEvent[]): AsyncGenerator { + for (const item of items) yield item; +} + +async function drain(source: AsyncGenerator): Promise { + const out: CloudChatEvent[] = []; + for await (const event of source) out.push(event); + return out; +} + +describe("streamChatEventsWithResetRetry", () => { + test("waits the stated delay and replays a zero-event 429", async () => { + const waits: number[] = []; + let calls = 0; + const stream = () => { + calls += 1; + return calls === 1 + ? exhausting("Reached free model rate limit. Your limit will reset in 35 seconds.")() + : events({ kind: "text", text: "ok" } as CloudChatEvent, { kind: "finish", reason: "stop" } as CloudChatEvent); + }; + const out = await drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async (ms) => { waits.push(ms); }, + })); + expect(calls).toBe(2); + expect(waits).toEqual([35_000]); + expect(out.map(e => e.kind)).toEqual(["text", "finish"]); + }); + + test("does not replay once any event was yielded", async () => { + const stream = () => (async function* (): AsyncGenerator { + yield { kind: "text", text: "partial" } as CloudChatEvent; + throw new CloudChatError("Your limit will reset in 35 seconds", "resource_exhausted", "t", 429); + })(); + await expect(drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async () => { throw new Error("sleep must not run"); }, + }))).rejects.toThrow("35 seconds"); + }); + + test("does not replay a 429 that states no parseable delay", async () => { + let calls = 0; + const stream = () => { + calls += 1; + return exhausting("Reached free model rate limit. Upgrade to Max for higher limits.")(); + }; + await expect(drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async () => {}, + }))).rejects.toThrow("Upgrade to Max"); + expect(calls).toBe(1); + }); + + test("does not replay a non-429 CloudChatError", async () => { + let calls = 0; + const stream = () => { + calls += 1; + return (async function* (): AsyncGenerator { + throw new CloudChatError("unauthenticated", "unauthenticated", "t", 401); + })(); + }; + await expect(drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async () => {}, + }))).rejects.toThrow("unauthenticated"); + expect(calls).toBe(1); + }); + + test("stops replaying at the replay cap", async () => { + const waits: number[] = []; + let calls = 0; + const stream = () => { + calls += 1; + return exhausting("Your limit will reset in 10 seconds")(); + }; + await expect(drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async (ms) => { waits.push(ms); }, + maxReplays: 2, + }))).rejects.toThrow("reset in 10 seconds"); + expect(calls).toBe(3); + expect(waits).toEqual([10_000, 10_000]); + }); + + test("a stated window beyond the wait ceiling surfaces instead of holding", async () => { + let calls = 0; + const stream = () => { + calls += 1; + return exhausting("Your limit will reset in 13 minutes")(); + }; + await expect(drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async () => {}, + maxWaitMs: 300_000, + }))).rejects.toThrow("13 minutes"); + expect(calls).toBe(1); + }); + + test("a 21-minute stated window replays under the default ceiling", async () => { + // Observed upstream windows reach ~21 minutes; the default ceiling is 30. + const waits: number[] = []; + let calls = 0; + const stream = () => { + calls += 1; + return calls === 1 + ? exhausting("Your limit will reset in 21 minutes")() + : events({ kind: "finish", reason: "stop" } as CloudChatEvent); + }; + const out = await drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async (ms) => { waits.push(ms); }, + })); + expect(calls).toBe(2); + expect(waits).toEqual([1_260_000]); + expect(out.map(e => e.kind)).toEqual(["finish"]); + }); + + test("the wait ceiling honors the env override", async () => { + const prev = process.env.OPENCODEX_DEVIN_STATED_RESET_WAIT_MS; + process.env.OPENCODEX_DEVIN_STATED_RESET_WAIT_MS = "5000"; + try { + let calls = 0; + const stream = () => { + calls += 1; + return exhausting("Your limit will reset in 35 seconds")(); + }; + await expect(drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async () => {}, + }))).rejects.toThrow("35 seconds"); + expect(calls).toBe(1); + } finally { + if (prev === undefined) delete process.env.OPENCODEX_DEVIN_STATED_RESET_WAIT_MS; + else process.env.OPENCODEX_DEVIN_STATED_RESET_WAIT_MS = prev; + } + }); + + test("an abort during the wait propagates instead of replaying", async () => { + let calls = 0; + const stream = () => { + calls += 1; + return exhausting("Your limit will reset in 35 seconds")(); + }; + await expect(drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async () => { throw new DOMException("The operation was aborted", "AbortError"); }, + }))).rejects.toThrow("aborted"); + expect(calls).toBe(1); + }); + + test("a non-CloudChatError passes straight through", async () => { + const stream = () => (async function* (): AsyncGenerator { + throw new Error("socket reset"); + })(); + await expect(drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async () => {}, + }))).rejects.toThrow("socket reset"); + }); +}); diff --git a/tests/server/retry-after-429.test.ts b/tests/server/retry-after-429.test.ts index 8264399419..8930ae5532 100644 --- a/tests/server/retry-after-429.test.ts +++ b/tests/server/retry-after-429.test.ts @@ -90,6 +90,58 @@ describe("resolveClientRetryAfter (#507)", () => { includeDefault: false, })).toBe("9"); }); + + test("reads the reset delay a Cognition-style trailer states in seconds", () => { + // Devin cloud error resource_exhausted: "... Your limit will reset in 35 + // seconds." Previously fell through to the synthetic 2s default, so a + // retry fired straight back into the live cap. + expect(resolveClientRetryAfter({ + status: 429, + message: "Devin cloud error resource_exhausted: Reached free model rate limit. Upgrade to Max for higher limits, or switch to a different model. Your limit will reset in 35 seconds. (trace ID: 814519e)", + })).toBe("35"); + }); + + test("reads a stated reset in minutes and hours, not just seconds", () => { + expect(resolveClientRetryAfter({ + status: 429, + message: "Your limit will reset in 13 minutes", + })).toBe("780"); + expect(resolveClientRetryAfter({ + status: 429, + message: "quota window resets in 1 hour", + })).toBe("3600"); + }); + + test("a stated reset feeds cooldown metadata too (includeDefault:false)", () => { + expect(resolveClientRetryAfter({ + status: 429, + message: "Your limit will reset in 35 seconds", + includeDefault: false, + })).toBe("35"); + }); + + test("reset phrasing without a time unit is not a delay", () => { + // "reset in 2026" names a year, not a wait. + expect(resolveClientRetryAfter({ + status: 429, + message: "Your limit will reset in 2026", + })).toBe(DEFAULT_RETRYABLE_429_RETRY_AFTER_SEC); + }); + + test("existing phrasings still parse with their original units", () => { + expect(resolveClientRetryAfter({ + status: 429, + message: "Throttled. Please try again in 7s.", + })).toBe("7"); + expect(resolveClientRetryAfter({ + status: 429, + message: "retry after 2 minutes", + })).toBe("120"); + expect(resolveClientRetryAfter({ + status: 429, + message: "Retry-After: 30", + })).toBe("30"); + }); }); describe("formatErrorResponse Retry-After (#507)", () => { diff --git a/tests/server/retry-delay-hardening.test.ts b/tests/server/retry-delay-hardening.test.ts new file mode 100644 index 0000000000..e0e43d51c5 --- /dev/null +++ b/tests/server/retry-delay-hardening.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, test } from "bun:test"; +import { adapterFailureFromMessage, parseRetryAfterFromMessage } from "../../src/lib/errors"; + +describe("stated reset duration boundaries", () => { + test("the error adapter retains a local parser binding after extraction", () => { + const failure = adapterFailureFromMessage("Devin rate limit: reset in 5 minutes 30 seconds"); + expect(failure.httpStatus).toBe(429); + expect(failure.error.message).toBe("Devin rate limit: reset in 5 minutes 30 seconds Please try again in 330s."); + expect(failure.error.code).toBe("rate_limit_exceeded"); + }); + + test.each([ + ["reset in 5 minutes 30 seconds", 330], + ["reset in 1 hour, 5 minutes and 30 seconds", 3930], + ["reset in 1h30m", 5400], + ["try again in 500ms", 1], + ["retry after 1500 milliseconds", 2], + ["reset in 1 minute 500 milliseconds", 61], + ["retry after 7.2s", 8], + ["Retry-After: 30", 30], + ["Retry-After: 0.1", 1], + ["Your limit RESETS IN 21 MINUTES", 1260], + ["try again in 2s. Your limit will reset in 21 minutes.", 1260], + ] as const)("%s -> %i seconds", (message, expected) => { + expect(parseRetryAfterFromMessage(message)).toBe(expected); + }); + + test.each([ + "reset in 2026", + "reset in -5 minutes", + "reset in 5 minutes -30 seconds", + "reset in 5 minutes 30 bananas", + "reset in 5 minutes 30", + "reset in 5 months", + "retry after 3 monkeys", + "try again in 1e3s", + "Retry-After: 3:30", + "Retry-After: 123abc", + "reset in 0 seconds", + "reset in 999999999999999999999999999999999999999999 hours", + ])("does not salvage a misleading partial duration: %s", message => { + expect(parseRetryAfterFromMessage(message)).toBeUndefined(); + }); +});