From 94e29a183bc8066712d9a97d9da38456b59e4842 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:52:35 +0900 Subject: [PATCH 1/3] fix(devin): wait out a stated rate-limit reset and replay the turn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cognition free-tier caps answer resource_exhausted trailers that name their own recovery delay ("Your limit will reset in 35 seconds"). The delay was never parsed: clients got the synthetic 2s Retry-After and combo cooldowns fell back to the 60s default, so a retry fired straight back into the live cap and the turn died inside a window it could have waited out. parseRetryAfterFromMessage now reads "reset in N " phrasing plus minute/hour units on the existing hints, so the stated delay drives the client Retry-After and the cooldown metadata. streamChatEventsWithResetRetry waits the stated delay and replays the identical request, but only while the stream has yielded zero events — after any output a replay could double billable side effects, so those failures keep their terminal path. Waits are bounded by a replay cap and a per-wait ceiling; longer stated windows surface with the parsed delay intact. --- src/adapters/devin.ts | 8 +- src/adapters/devin/cloud-direct/index.ts | 7 + .../devin/cloud-direct/stated-reset-retry.ts | 99 ++++++++++ src/lib/errors.ts | 39 +++- .../devin-stated-reset-retry.test.ts | 178 ++++++++++++++++++ tests/server/retry-after-429.test.ts | 52 +++++ 6 files changed, 374 insertions(+), 9 deletions(-) create mode 100644 src/adapters/devin/cloud-direct/stated-reset-retry.ts create mode 100644 tests/providers/devin-stated-reset-retry.test.ts diff --git a/src/adapters/devin.ts b/src/adapters/devin.ts index 9b3a971fa7..0ff3b7da33 100644 --- a/src/adapters/devin.ts +++ b/src/adapters/devin.ts @@ -9,7 +9,7 @@ import type { AdapterEvent, OcxAssistantMessage, OcxContentPart, OcxMessage, OcxParsedRequest, OcxProviderConfig, OcxTool, OcxToolCall, OcxToolResultMessage, OcxUsage } from "../types"; import { namespacedToolName } from "../types"; import type { IncomingMeta, ProviderAdapter } from "./base"; -import { streamChatEvents, allocateCascadeId, CloudChatError, type ChatHistoryItem, type ToolDef } from "./devin/cloud-direct"; +import { streamChatEventsWithResetRetry, allocateCascadeId, CloudChatError, type ChatHistoryItem, type ToolDef } from "./devin/cloud-direct"; import type { ContentPart } from "./devin/cloud-direct/chat"; import { getCachedCatalog, type CacheEntry } from "./devin/cloud-direct/catalog"; import { collapseDevinModelUid } from "./devin/live-models"; @@ -563,7 +563,11 @@ export function createDevinAdapter( const maxInputTokens = resolveDevinMaxInputTokens( provider, modelUid, catalog?.byUid.get(modelUid)?.contextWindow, ); - for await (const event of streamChatEvents({ + // The reset-retry wrapper waits out a 429 that states its own recovery + // delay ("limit will reset in 35 seconds") and replays the identical + // request — but only while zero events have been yielded, so a + // post-output failure still takes the terminal path untouched. + for await (const event of streamChatEventsWithResetRetry({ apiKey, apiServerUrl: host, modelUid, diff --git a/src/adapters/devin/cloud-direct/index.ts b/src/adapters/devin/cloud-direct/index.ts index 98dfcea9de..f4dbab563f 100644 --- a/src/adapters/devin/cloud-direct/index.ts +++ b/src/adapters/devin/cloud-direct/index.ts @@ -49,6 +49,13 @@ export { type ToolDef, } from './chat.js'; +export { + streamChatEventsWithResetRetry, + STATED_RESET_MAX_REPLAYS, + STATED_RESET_MAX_WAIT_MS, + type StatedResetRetryOptions, +} from './stated-reset-retry.js'; + export { mintUserJwt, getCachedUserJwt, diff --git a/src/adapters/devin/cloud-direct/stated-reset-retry.ts b/src/adapters/devin/cloud-direct/stated-reset-retry.ts new file mode 100644 index 0000000000..69dd883d4c --- /dev/null +++ b/src/adapters/devin/cloud-direct/stated-reset-retry.ts @@ -0,0 +1,99 @@ +/** + * Same-target replay for a 429 whose trailer states its own recovery delay. + * + * Cognition's free-tier cap answers `resource_exhausted` trailers that name + * the wait: "Your limit will reset in 35 seconds". Surfaced raw, that turn + * dies — the Codex client does not retry 429s, and the stated cooldown was + * never read, so nothing waited the window the upstream itself announced. + * + * This wrapper waits the stated delay and replays the identical request, but + * only while the replay is provably safe: the stream must have yielded ZERO + * events. After the first event the turn may have billable side effects and + * client-visible output, so a replay would double them — those failures keep + * their terminal path. The wait is bounded twice (replays and a per-wait + * ceiling) so a long stated window still surfaces instead of holding the + * turn; the surfaced error now carries the parsed delay for whatever the + * client decides next. + */ + +import { parseRetryAfterFromMessage } from '../../../lib/errors.js'; +import { sleepWithAbort } from '../../../lib/upstream-retry.js'; +import { CloudChatError, streamChatEvents, type CloudChatEvent, type CloudChatRequest } from './chat.js'; + +/** Total replays after the first failure: 1 initial + 2 replays = 3 sends. */ +export const STATED_RESET_MAX_REPLAYS = 2; +/** + * Per-wait ceiling, default 30 minutes: observed Cognition stated windows run + * from seconds through ~21 minutes, so a 5-minute ceiling would still fail the + * common case. A stated window beyond this surfaces the failure (with the + * parsed delay intact) rather than pinning a turn for the full window. + * Override with OPENCODEX_DEVIN_STATED_RESET_WAIT_MS, bounded by + * STATED_RESET_WAIT_CEILING_MS so a stray value cannot wedge a turn forever. + */ +export const STATED_RESET_MAX_WAIT_MS = 1_800_000; +/** Upper bound for the override: one hour is the longest wait worth holding. */ +export const STATED_RESET_WAIT_CEILING_MS = 3_600_000; + +function statedResetMaxWaitMs(): number { + const raw = process.env.OPENCODEX_DEVIN_STATED_RESET_WAIT_MS?.trim(); + if (!raw) return STATED_RESET_MAX_WAIT_MS; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) return STATED_RESET_MAX_WAIT_MS; + return Math.min(parsed, STATED_RESET_WAIT_CEILING_MS); +} +/** Test seam for the wait ceiling; the resolver itself stays private. */ +export const statedResetMaxWaitMsForTests = statedResetMaxWaitMs; + +export interface StatedResetRetryOptions { + /** Test seam: the event source. Defaults to the real cloud stream. */ + stream?: (req: CloudChatRequest) => AsyncGenerator; + /** Test seam: abort-aware sleep. */ + sleep?: (ms: number, signal?: AbortSignal) => Promise; + maxReplays?: number; + maxWaitMs?: number; +} + +/** + * `streamChatEvents` plus a bounded stated-reset replay. Yields the same + * event stream; on a pre-output 429 carrying a parseable "reset in N" delay, + * sleeps that delay and re-issues the request instead of failing the turn. + */ +export async function* streamChatEventsWithResetRetry( + req: CloudChatRequest, + options?: StatedResetRetryOptions, +): AsyncGenerator { + const stream = options?.stream ?? streamChatEvents; + const sleep = options?.sleep ?? sleepWithAbort; + const maxReplays = options?.maxReplays ?? STATED_RESET_MAX_REPLAYS; + const maxWaitMs = options?.maxWaitMs ?? statedResetMaxWaitMs(); + let replays = 0; + while (true) { + // Set BEFORE the yield: an error surfacing at the yield point (consumer + // throw) must read as post-output, and any event at all means the turn + // may already have had effects a replay would duplicate. + let yielded = false; + try { + for await (const event of stream(req)) { + yielded = true; + yield event; + } + return; + } catch (error) { + const waitSec = !yielded + && error instanceof CloudChatError + && error.status === 429 + ? parseRetryAfterFromMessage(error.message) + : undefined; + if ( + waitSec === undefined + || replays >= maxReplays + || waitSec * 1000 > maxWaitMs + ) { + throw error; + } + replays += 1; + // Throws on client abort; the adapter's catch reports the cancellation. + await sleep(waitSec * 1000, req.signal); + } + } +} diff --git a/src/lib/errors.ts b/src/lib/errors.ts index 7c4582c2fe..e08f5b5010 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -376,17 +376,42 @@ export function isRateLimitOrQuotaFailureMessage(message: string): boolean { return normalized.toLowerCase().includes("usage limit"); } -/** Best-effort parse of a retry delay embedded in an upstream error message. */ +/** + * Time units an upstream may write into a retry hint. The first letter decides + * the multiplier, so the alternations below never need a lookup table. + */ +const RETRY_HINT_UNIT_SECONDS: Readonly> = { s: 1, m: 60, h: 3600 }; + +/** + * Best-effort parse of a retry delay embedded in an upstream error message. + * + * Recognised phrasings, all case-insensitive: + * - "try again in 7s" / "retry after 30s" — seconds were the only unit the + * original patterns understood; minutes and hours now parse too. + * - "Retry-After: 30" — header syntax inside a body; a bare number stays + * seconds per RFC 9110, but an explicit unit is honoured when present. + * - "Your limit will reset in 35 seconds" — Cognition/Devin's free-tier cap + * states its own recovery delay in the trailer message. Without this the + * stated wait was discarded: clients got the synthetic 2s fallback and + * combo cooldowns fell back to the 60s default, so a retry fired straight + * back into the live cap. The unit is REQUIRED here — "reset in 2026" + * names a year, not a delay. + */ export function parseRetryAfterFromMessage(message: string): number | undefined { const patterns = [ - /try again in (\d+(?:\.\d+)?)\s*s(?:ec(?:ond)?s?)?/i, - /retry after (\d+(?:\.\d+)?)\s*s(?:ec(?:ond)?s?)?/i, - /retry[- ]after[:\s]+(\d+)/i, + /try again in (\d+(?:\.\d+)?)\s*(s(?:econds?|ecs?)?|m(?:inutes?|ins?)?|h(?:ours?|rs?)?)/i, + /retry after (\d+(?:\.\d+)?)\s*(s(?:econds?|ecs?)?|m(?:inutes?|ins?)?|h(?:ours?|rs?)?)/i, + // Header syntax tolerates a missing unit: a bare Retry-After number is + // already seconds by definition. + /retry[- ]after[:\s]+(\d+(?:\.\d+)?)(?:\s*(s(?:econds?|ecs?)?|m(?:inutes?|ins?)?|h(?:ours?|rs?)?))?/i, + /\breset[s]?\s+in\s+(\d+(?:\.\d+)?)\s*(s(?:econds?|ecs?)?|m(?:inutes?|ins?)?|h(?:ours?|rs?)?)\b/i, ]; - for (const pattern of patterns) { - const match = message.match(pattern); + for (const re of patterns) { + const match = message.match(re); if (!match?.[1]) continue; - const seconds = Number.parseFloat(match[1]); + const amount = Number.parseFloat(match[1]); + const unit = match[2]?.[0]?.toLowerCase() ?? "s"; + const seconds = amount * (RETRY_HINT_UNIT_SECONDS[unit] ?? 1); if (Number.isFinite(seconds) && seconds > 0) return Math.ceil(seconds); } return undefined; diff --git a/tests/providers/devin-stated-reset-retry.test.ts b/tests/providers/devin-stated-reset-retry.test.ts new file mode 100644 index 0000000000..18e7ef9ce6 --- /dev/null +++ b/tests/providers/devin-stated-reset-retry.test.ts @@ -0,0 +1,178 @@ +/** + * Devin stated-reset replay: a 429 trailer that names its own recovery delay + * ("Your limit will reset in 35 seconds") is waited out and the identical + * request replayed — but only while the stream produced zero events, and only + * within the replay/wait bounds. + */ +import { describe, expect, test } from "bun:test"; +import { CloudChatError, type CloudChatEvent, type CloudChatRequest } from "../../src/adapters/devin/cloud-direct"; +import { streamChatEventsWithResetRetry } from "../../src/adapters/devin/cloud-direct/stated-reset-retry"; + +const REQ = { apiKey: "k", apiServerUrl: "https://example.invalid", modelUid: "swe-2", messages: [] } as unknown as CloudChatRequest; + +function exhausting(message: string): () => AsyncGenerator { + return () => (async function* (): AsyncGenerator { + throw new CloudChatError(message, "resource_exhausted", "t", 429); + })(); +} + +async function* events(...items: CloudChatEvent[]): AsyncGenerator { + for (const item of items) yield item; +} + +async function drain(source: AsyncGenerator): Promise { + const out: CloudChatEvent[] = []; + for await (const event of source) out.push(event); + return out; +} + +describe("streamChatEventsWithResetRetry", () => { + test("waits the stated delay and replays a zero-event 429", async () => { + const waits: number[] = []; + let calls = 0; + const stream = () => { + calls += 1; + return calls === 1 + ? exhausting("Reached free model rate limit. Your limit will reset in 35 seconds.")() + : events({ kind: "text", text: "ok" } as CloudChatEvent, { kind: "finish", reason: "stop" } as CloudChatEvent); + }; + const out = await drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async (ms) => { waits.push(ms); }, + })); + expect(calls).toBe(2); + expect(waits).toEqual([35_000]); + expect(out.map(e => e.kind)).toEqual(["text", "finish"]); + }); + + test("does not replay once any event was yielded", async () => { + const stream = () => (async function* (): AsyncGenerator { + yield { kind: "text", text: "partial" } as CloudChatEvent; + throw new CloudChatError("Your limit will reset in 35 seconds", "resource_exhausted", "t", 429); + })(); + await expect(drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async () => { throw new Error("sleep must not run"); }, + }))).rejects.toThrow("35 seconds"); + }); + + test("does not replay a 429 that states no parseable delay", async () => { + let calls = 0; + const stream = () => { + calls += 1; + return exhausting("Reached free model rate limit. Upgrade to Max for higher limits.")(); + }; + await expect(drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async () => {}, + }))).rejects.toThrow("Upgrade to Max"); + expect(calls).toBe(1); + }); + + test("does not replay a non-429 CloudChatError", async () => { + let calls = 0; + const stream = () => { + calls += 1; + return (async function* (): AsyncGenerator { + throw new CloudChatError("unauthenticated", "unauthenticated", "t", 401); + })(); + }; + await expect(drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async () => {}, + }))).rejects.toThrow("unauthenticated"); + expect(calls).toBe(1); + }); + + test("stops replaying at the replay cap", async () => { + const waits: number[] = []; + let calls = 0; + const stream = () => { + calls += 1; + return exhausting("Your limit will reset in 10 seconds")(); + }; + await expect(drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async (ms) => { waits.push(ms); }, + maxReplays: 2, + }))).rejects.toThrow("reset in 10 seconds"); + expect(calls).toBe(3); + expect(waits).toEqual([10_000, 10_000]); + }); + + test("a stated window beyond the wait ceiling surfaces instead of holding", async () => { + let calls = 0; + const stream = () => { + calls += 1; + return exhausting("Your limit will reset in 13 minutes")(); + }; + await expect(drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async () => {}, + maxWaitMs: 300_000, + }))).rejects.toThrow("13 minutes"); + expect(calls).toBe(1); + }); + + test("a 21-minute stated window replays under the default ceiling", async () => { + // Observed upstream windows reach ~21 minutes; the default ceiling is 30. + const waits: number[] = []; + let calls = 0; + const stream = () => { + calls += 1; + return calls === 1 + ? exhausting("Your limit will reset in 21 minutes")() + : events({ kind: "finish", reason: "stop" } as CloudChatEvent); + }; + const out = await drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async (ms) => { waits.push(ms); }, + })); + expect(calls).toBe(2); + expect(waits).toEqual([1_260_000]); + expect(out.map(e => e.kind)).toEqual(["finish"]); + }); + + test("the wait ceiling honors the env override", async () => { + const prev = process.env.OPENCODEX_DEVIN_STATED_RESET_WAIT_MS; + process.env.OPENCODEX_DEVIN_STATED_RESET_WAIT_MS = "5000"; + try { + let calls = 0; + const stream = () => { + calls += 1; + return exhausting("Your limit will reset in 35 seconds")(); + }; + await expect(drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async () => {}, + }))).rejects.toThrow("35 seconds"); + expect(calls).toBe(1); + } finally { + if (prev === undefined) delete process.env.OPENCODEX_DEVIN_STATED_RESET_WAIT_MS; + else process.env.OPENCODEX_DEVIN_STATED_RESET_WAIT_MS = prev; + } + }); + + test("an abort during the wait propagates instead of replaying", async () => { + let calls = 0; + const stream = () => { + calls += 1; + return exhausting("Your limit will reset in 35 seconds")(); + }; + await expect(drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async () => { throw new DOMException("The operation was aborted", "AbortError"); }, + }))).rejects.toThrow("aborted"); + expect(calls).toBe(1); + }); + + test("a non-CloudChatError passes straight through", async () => { + const stream = () => (async function* (): AsyncGenerator { + throw new Error("socket reset"); + })(); + await expect(drain(streamChatEventsWithResetRetry(REQ, { + stream, + sleep: async () => {}, + }))).rejects.toThrow("socket reset"); + }); +}); diff --git a/tests/server/retry-after-429.test.ts b/tests/server/retry-after-429.test.ts index 8264399419..8930ae5532 100644 --- a/tests/server/retry-after-429.test.ts +++ b/tests/server/retry-after-429.test.ts @@ -90,6 +90,58 @@ describe("resolveClientRetryAfter (#507)", () => { includeDefault: false, })).toBe("9"); }); + + test("reads the reset delay a Cognition-style trailer states in seconds", () => { + // Devin cloud error resource_exhausted: "... Your limit will reset in 35 + // seconds." Previously fell through to the synthetic 2s default, so a + // retry fired straight back into the live cap. + expect(resolveClientRetryAfter({ + status: 429, + message: "Devin cloud error resource_exhausted: Reached free model rate limit. Upgrade to Max for higher limits, or switch to a different model. Your limit will reset in 35 seconds. (trace ID: 814519e)", + })).toBe("35"); + }); + + test("reads a stated reset in minutes and hours, not just seconds", () => { + expect(resolveClientRetryAfter({ + status: 429, + message: "Your limit will reset in 13 minutes", + })).toBe("780"); + expect(resolveClientRetryAfter({ + status: 429, + message: "quota window resets in 1 hour", + })).toBe("3600"); + }); + + test("a stated reset feeds cooldown metadata too (includeDefault:false)", () => { + expect(resolveClientRetryAfter({ + status: 429, + message: "Your limit will reset in 35 seconds", + includeDefault: false, + })).toBe("35"); + }); + + test("reset phrasing without a time unit is not a delay", () => { + // "reset in 2026" names a year, not a wait. + expect(resolveClientRetryAfter({ + status: 429, + message: "Your limit will reset in 2026", + })).toBe(DEFAULT_RETRYABLE_429_RETRY_AFTER_SEC); + }); + + test("existing phrasings still parse with their original units", () => { + expect(resolveClientRetryAfter({ + status: 429, + message: "Throttled. Please try again in 7s.", + })).toBe("7"); + expect(resolveClientRetryAfter({ + status: 429, + message: "retry after 2 minutes", + })).toBe("120"); + expect(resolveClientRetryAfter({ + status: 429, + message: "Retry-After: 30", + })).toBe("30"); + }); }); describe("formatErrorResponse Retry-After (#507)", () => { From 9e883fa38cd324f59bfaca740562121e8f5c8dbb Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Fri, 18 Sep 2026 18:13:33 +0900 Subject: [PATCH 2/3] fix(devin): bound cumulative reset waits and preserve server cooldowns Parse complete and compound retry durations without treating milliseconds as minutes. Keep the errors.ts local parser binding as well as its public export. Bound scheduled waits cumulatively within a stated-reset wrapper invocation; preserve explicit refusal errors, cancellation, zero-wait opt-out and the no-output replay boundary. Store explicit Retry-After cooldowns without truncating them to the local ten-minute fallback. Add regression coverage for parser integration, cumulative waits, abort races, post-output refusals and authoritative numeric/HTTP-date cooldowns. Validation in this environment: 90 offline core checks; 43 proposed regression cases executed via a Node/TypeScript module harness with external routing/state/RPC dependencies stubbed; strict typecheck of errors.ts and retry-delay.ts. Full Bun/repository typecheck and live Devin validation remain pending. Long-lived SSE/preflight and cross-layer send/wait budgeting are not implemented by this commit. --- .../devin/cloud-direct/stated-reset-retry.ts | 98 ++++++++------- src/combos/failover.ts | 22 +++- src/lib/errors.ts | 43 +------ src/lib/retry-delay.ts | 69 +++++++++++ .../combo-authoritative-reset.test.ts | 42 +++++++ .../devin-stated-reset-hardening.test.ts | 116 ++++++++++++++++++ tests/server/retry-delay-hardening.test.ts | 44 +++++++ 7 files changed, 342 insertions(+), 92 deletions(-) create mode 100644 src/lib/retry-delay.ts create mode 100644 tests/codex-integration/combo-authoritative-reset.test.ts create mode 100644 tests/providers/devin-stated-reset-hardening.test.ts create mode 100644 tests/server/retry-delay-hardening.test.ts diff --git a/src/adapters/devin/cloud-direct/stated-reset-retry.ts b/src/adapters/devin/cloud-direct/stated-reset-retry.ts index 69dd883d4c..f500cd4b19 100644 --- a/src/adapters/devin/cloud-direct/stated-reset-retry.ts +++ b/src/adapters/devin/cloud-direct/stated-reset-retry.ts @@ -1,99 +1,103 @@ /** - * Same-target replay for a 429 whose trailer states its own recovery delay. - * - * Cognition's free-tier cap answers `resource_exhausted` trailers that name - * the wait: "Your limit will reset in 35 seconds". Surfaced raw, that turn - * dies — the Codex client does not retry 429s, and the stated cooldown was - * never read, so nothing waited the window the upstream itself announced. - * - * This wrapper waits the stated delay and replays the identical request, but - * only while the replay is provably safe: the stream must have yielded ZERO - * events. After the first event the turn may have billable side effects and - * client-visible output, so a replay would double them — those failures keep - * their terminal path. The wait is bounded twice (replays and a per-wait - * ceiling) so a long stated window still surfaces instead of holding the - * turn; the surfaced error now carries the parsed delay for whatever the - * client decides next. + * Same-target retry of an explicit, pre-output 429 refusal with a stated + * recovery delay. No event-producing attempt is ever automatically replayed. + * This is a refusal-specific policy, not a claim that every eventless POST + * is idempotent; ambiguous transport failures still propagate unchanged. */ - -import { parseRetryAfterFromMessage } from '../../../lib/errors.js'; -import { sleepWithAbort } from '../../../lib/upstream-retry.js'; +import { parseRetryAfterFromMessage } from '../../../lib/retry-delay.js'; +import { abortError, sleepWithAbort } from '../../../lib/upstream-retry.js'; import { CloudChatError, streamChatEvents, type CloudChatEvent, type CloudChatRequest } from './chat.js'; -/** Total replays after the first failure: 1 initial + 2 replays = 3 sends. */ +/** 1 initial attempt plus at most 2 replays. */ export const STATED_RESET_MAX_REPLAYS = 2; -/** - * Per-wait ceiling, default 30 minutes: observed Cognition stated windows run - * from seconds through ~21 minutes, so a 5-minute ceiling would still fail the - * common case. A stated window beyond this surfaces the failure (with the - * parsed delay intact) rather than pinning a turn for the full window. - * Override with OPENCODEX_DEVIN_STATED_RESET_WAIT_MS, bounded by - * STATED_RESET_WAIT_CEILING_MS so a stray value cannot wedge a turn forever. - */ +/** Default cumulative wait allowance for one invocation (30 minutes). */ export const STATED_RESET_MAX_WAIT_MS = 1_800_000; -/** Upper bound for the override: one hour is the longest wait worth holding. */ +/** Absolute maximum cumulative allowance, including explicit overrides. */ export const STATED_RESET_WAIT_CEILING_MS = 3_600_000; function statedResetMaxWaitMs(): number { const raw = process.env.OPENCODEX_DEVIN_STATED_RESET_WAIT_MS?.trim(); if (!raw) return STATED_RESET_MAX_WAIT_MS; const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) return STATED_RESET_MAX_WAIT_MS; - return Math.min(parsed, STATED_RESET_WAIT_CEILING_MS); + if (!Number.isFinite(parsed) || parsed < 0) return STATED_RESET_MAX_WAIT_MS; + // Zero explicitly disables local waiting. Values above one hour are capped. + return Math.min(Math.floor(parsed), STATED_RESET_WAIT_CEILING_MS); } -/** Test seam for the wait ceiling; the resolver itself stays private. */ export const statedResetMaxWaitMsForTests = statedResetMaxWaitMs; export interface StatedResetRetryOptions { - /** Test seam: the event source. Defaults to the real cloud stream. */ + /** Test seam: defaults to the real cloud stream. */ stream?: (req: CloudChatRequest) => AsyncGenerator; - /** Test seam: abort-aware sleep. */ + /** Test seam: must either honour the whole delay or reject on cancellation. */ sleep?: (ms: number, signal?: AbortSignal) => Promise; maxReplays?: number; + /** CUMULATIVE wait allowance, not a fresh allowance on every failure. */ maxWaitMs?: number; } -/** - * `streamChatEvents` plus a bounded stated-reset replay. Yields the same - * event stream; on a pre-output 429 carrying a parseable "reset in N" delay, - * sleeps that delay and re-issues the request instead of failing the turn. - */ +function replayLimit(value: number | undefined): number { + if (value === undefined) return STATED_RESET_MAX_REPLAYS; + if (!Number.isInteger(value) || value < 0 || value > STATED_RESET_MAX_REPLAYS) { + throw new RangeError('maxReplays must be an integer from 0 to 2'); + } + return value; +} + +function waitLimit(value: number | undefined): number { + if (value === undefined) return statedResetMaxWaitMs(); + if (!Number.isFinite(value) || value < 0) { + throw new RangeError('maxWaitMs must be a finite non-negative number'); + } + return Math.min(Math.floor(value), STATED_RESET_WAIT_CEILING_MS); +} + export async function* streamChatEventsWithResetRetry( req: CloudChatRequest, options?: StatedResetRetryOptions, ): AsyncGenerator { const stream = options?.stream ?? streamChatEvents; const sleep = options?.sleep ?? sleepWithAbort; - const maxReplays = options?.maxReplays ?? STATED_RESET_MAX_REPLAYS; - const maxWaitMs = options?.maxWaitMs ?? statedResetMaxWaitMs(); + const maxReplays = replayLimit(options?.maxReplays); + const maxWaitMs = waitLimit(options?.maxWaitMs); let replays = 0; + let waitedMs = 0; while (true) { - // Set BEFORE the yield: an error surfacing at the yield point (consumer - // throw) must read as post-output, and any event at all means the turn - // may already have had effects a replay would duplicate. + // Check again after sleeping: cancellation can race with timer completion. + // A pre-aborted request must not even enter a custom transport. + if (req.signal?.aborted) throw abortError(req.signal); let yielded = false; try { for await (const event of stream(req)) { + // Latch before yielding, so a consumer-injected error is post-output. yielded = true; yield event; } return; } catch (error) { + if (req.signal?.aborted) throw abortError(req.signal); const waitSec = !yielded && error instanceof CloudChatError && error.status === 429 ? parseRetryAfterFromMessage(error.message) : undefined; + const waitMs = waitSec === undefined ? undefined : waitSec * 1000; if ( - waitSec === undefined + waitMs === undefined || replays >= maxReplays - || waitSec * 1000 > maxWaitMs + || waitMs > maxWaitMs - waitedMs ) { + // Never shorten a provider's minimum delay to fit the local budget. + // Keep the original refusal so outer policy can preserve its metadata. throw error; } replays += 1; - // Throws on client abort; the adapter's catch reports the cancellation. - await sleep(waitSec * 1000, req.signal); + // Charge the complete scheduled wait once, before sleeping. This is a + // sleep allowance, not a wall-clock deadline on generation or timer + // scheduling: waking a few milliseconds late must not reject an already + // approved one-hour retry. No later wait can spend this allowance again. + waitedMs += waitMs; + await sleep(waitMs, req.signal); + if (req.signal?.aborted) throw abortError(req.signal); } } } diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 21de270e93..da9ccdca37 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -118,23 +118,29 @@ function parseHttpDate(value: string, now: number): number | undefined { export function parseRetryAfterMs( value: string | null | undefined, now = Date.now(), - options?: { preserveImmediate?: boolean }, + options?: { preserveImmediate?: boolean; preserveServerDelay?: boolean }, ): number | undefined { const text = value?.trim(); if (!text) return undefined; + // A local wait ceiling must not make an explicit upstream reset expire early. + // Keep legacy bounded parsing for other callers. The opt-in stores a timestamp; + // the combo picker still independently limits how long a live request waits. + const maximum = options?.preserveServerDelay === true + ? Number.MAX_SAFE_INTEGER - Math.max(0, now) + : MAX_COOLDOWN_MS; if (/^\d+(?:\.\d+)?$/.test(text)) { const seconds = Number(text); if ( Number.isFinite(seconds) && (seconds > 0 || (options?.preserveImmediate && seconds === 0)) ) { - return Math.min(Math.max(Math.ceil(seconds * 1000), 1), MAX_COOLDOWN_MS); + return Math.min(Math.max(Math.ceil(seconds * 1000), 1), maximum); } } const timestamp = parseHttpDate(text, now); if (timestamp === undefined) return undefined; const delay = timestamp - now; - if (delay > 0) return Math.min(delay, MAX_COOLDOWN_MS); + if (delay > 0) return Math.min(delay, maximum); return options?.preserveImmediate ? 1 : undefined; } @@ -215,7 +221,11 @@ export function coolComboTarget( // A server-provided Retry-After is authoritative, including an immediate `0` directive. // A quota reset is the next-most-specific signal (#3256); configured and default cooldowns // are only fallbacks when upstream supplied neither usable value. - const cooldownMs = parseRetryAfterMs(options?.retryAfter, now, { preserveImmediate: true }) + const serverDelayMs = parseRetryAfterMs(options?.retryAfter, now, { + preserveImmediate: true, + preserveServerDelay: true, + }); + const cooldownMs = serverDelayMs ?? parseResetCooldownMs(options?.resetAt, now) ?? options?.cooldownMs ?? (isTransientRequestRateLimit({ @@ -224,7 +234,9 @@ export function coolComboTarget( message: options?.message, }) ? COMBO_REQUEST_RATE_COOLDOWN_MS : DEFAULT_COOLDOWN_MS); targetCooldowns.set(cooldownMapKey(comboId, target), { - cooldownUntil: now + Math.min(Math.max(cooldownMs, 1), MAX_COOLDOWN_MS), + // Only the locally chosen fallback is capped at ten minutes. An explicit + // server lower bound (including one hour) remains authoritative. + cooldownUntil: now + (serverDelayMs ?? Math.min(Math.max(cooldownMs, 1), MAX_COOLDOWN_MS)), }); sweepExpiredOnWrite(now); } diff --git a/src/lib/errors.ts b/src/lib/errors.ts index e08f5b5010..669e81b21f 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -1,3 +1,5 @@ +import { parseRetryAfterFromMessage } from "./retry-delay"; + export interface OcxErrorPayload { message: string; type: string; @@ -376,46 +378,7 @@ export function isRateLimitOrQuotaFailureMessage(message: string): boolean { return normalized.toLowerCase().includes("usage limit"); } -/** - * Time units an upstream may write into a retry hint. The first letter decides - * the multiplier, so the alternations below never need a lookup table. - */ -const RETRY_HINT_UNIT_SECONDS: Readonly> = { s: 1, m: 60, h: 3600 }; - -/** - * Best-effort parse of a retry delay embedded in an upstream error message. - * - * Recognised phrasings, all case-insensitive: - * - "try again in 7s" / "retry after 30s" — seconds were the only unit the - * original patterns understood; minutes and hours now parse too. - * - "Retry-After: 30" — header syntax inside a body; a bare number stays - * seconds per RFC 9110, but an explicit unit is honoured when present. - * - "Your limit will reset in 35 seconds" — Cognition/Devin's free-tier cap - * states its own recovery delay in the trailer message. Without this the - * stated wait was discarded: clients got the synthetic 2s fallback and - * combo cooldowns fell back to the 60s default, so a retry fired straight - * back into the live cap. The unit is REQUIRED here — "reset in 2026" - * names a year, not a delay. - */ -export function parseRetryAfterFromMessage(message: string): number | undefined { - const patterns = [ - /try again in (\d+(?:\.\d+)?)\s*(s(?:econds?|ecs?)?|m(?:inutes?|ins?)?|h(?:ours?|rs?)?)/i, - /retry after (\d+(?:\.\d+)?)\s*(s(?:econds?|ecs?)?|m(?:inutes?|ins?)?|h(?:ours?|rs?)?)/i, - // Header syntax tolerates a missing unit: a bare Retry-After number is - // already seconds by definition. - /retry[- ]after[:\s]+(\d+(?:\.\d+)?)(?:\s*(s(?:econds?|ecs?)?|m(?:inutes?|ins?)?|h(?:ours?|rs?)?))?/i, - /\breset[s]?\s+in\s+(\d+(?:\.\d+)?)\s*(s(?:econds?|ecs?)?|m(?:inutes?|ins?)?|h(?:ours?|rs?)?)\b/i, - ]; - for (const re of patterns) { - const match = message.match(re); - if (!match?.[1]) continue; - const amount = Number.parseFloat(match[1]); - const unit = match[2]?.[0]?.toLowerCase() ?? "s"; - const seconds = amount * (RETRY_HINT_UNIT_SECONDS[unit] ?? 1); - if (Number.isFinite(seconds) && seconds > 0) return Math.ceil(seconds); - } - return undefined; -} +export { parseRetryAfterFromMessage }; /** Infer HTTP status from adapter terminal error text (provider-agnostic keyword matching). */ export function inferHttpStatusFromAdapterMessage(message: string): number { diff --git a/src/lib/retry-delay.ts b/src/lib/retry-delay.ts new file mode 100644 index 0000000000..31251770e2 --- /dev/null +++ b/src/lib/retry-delay.ts @@ -0,0 +1,69 @@ +/** + * Parse a provider's relative retry hint without mistaking a unit prefix for a + * complete unit. Kept independent of error classification and combo routing. + */ +const UNIT_SECONDS: Readonly> = { + ms: 0.001, msec: 0.001, msecs: 0.001, millisecond: 0.001, milliseconds: 0.001, + s: 1, sec: 1, secs: 1, second: 1, seconds: 1, + m: 60, min: 60, mins: 60, minute: 60, minutes: 60, + h: 3600, hr: 3600, hrs: 3600, hour: 3600, hours: 3600, + d: 86400, day: 86400, days: 86400, +}; + +// Read the WHOLE word before looking it up. In particular, ms is not m and +// "months" must not accidentally become minutes. Digits may follow a unit so +// compact, unambiguous durations such as 1h30m remain supported. +const COMPONENT = /^(\d+(?:\.\d+)?)\s*([a-z]+)(?![a-z_])/i; +const SEPARATOR = /^(?:\s*,\s*(?:and\s+)?|\s+and\s+|\s*\+\s*|\s*)/i; +const MAX_COMPONENTS = 16; + +function durationSeconds(tail: string, allowBareSeconds: boolean): number | undefined { + let rest = tail.trimStart(); + let seconds = 0; + let components = 0; + while (true) { + const component = COMPONENT.exec(rest); + if (!component) { + if (components !== 0 || !allowBareSeconds) return undefined; + // A header-style bare number means seconds. Never salvage the numeric + // prefix of an unsupported unit, exponent, signed value or clock time. + const bare = /^(\d+(?:\.\d+)?)(?![\w.:+-])(?=\s*(?:$|[.,;!?)\]}]))/.exec(rest); + if (!bare) return undefined; + seconds = Number(bare[1]); + break; + } + const unit = UNIT_SECONDS[component[2]!.toLowerCase()]; + if (unit === undefined) return undefined; + seconds += Number(component[1]) * unit; + if (!Number.isFinite(seconds) || ++components > MAX_COMPONENTS) return undefined; + rest = rest.slice(component[0].length); + const separator = SEPARATOR.exec(rest)![0]; + const next = rest.slice(separator.length); + if (!/^[+-]?(?:\d|\.\d)/.test(next)) break; + // A numeric continuation is part of this duration; a malformed second + // component must reject the hint, not silently shorten it to the first. + rest = next; + } + const rounded = Math.ceil(seconds); + return Number.isSafeInteger(rounded) && rounded > 0 ? rounded : undefined; +} + +/** + * Supports reset(s) in, try again in and Retry-After/retry after hints; accepts + * compound durations and rounds UP once after summing all components. + * A bare number is permitted only for header-style Retry-After hints, never + * for "reset in 2026". When a message declares several usable lower bounds, + * honour the longest one rather than re-entering a still-live quota window. + */ +export function parseRetryAfterFromMessage(message: string): number | undefined { + const hints = /\b(try\s+again\s+in|retry[- ]after|resets?\s+in)\s*:?\s*/gi; + let result: number | undefined; + for (const hint of message.matchAll(hints)) { + const seconds = durationSeconds( + message.slice(hint.index! + hint[0].length), + /^retry[- ]after$/i.test(hint[1]!), + ); + if (seconds !== undefined) result = Math.max(result ?? 0, seconds); + } + return result; +} diff --git a/tests/codex-integration/combo-authoritative-reset.test.ts b/tests/codex-integration/combo-authoritative-reset.test.ts new file mode 100644 index 0000000000..001d8f8b3b --- /dev/null +++ b/tests/codex-integration/combo-authoritative-reset.test.ts @@ -0,0 +1,42 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + clearComboTargetCooldowns, + coolComboTarget, + earliestComboCooldownExpiry, + isComboTargetInCooldown, + parseRetryAfterMs, +} from "../../src/combos/failover"; + +const target = { provider: "devin", model: "swe-2" }; +const combo = "stated-reset-hardening-test"; +const now = Date.UTC(2026, 8, 18, 8, 0, 0); +afterEach(() => clearComboTargetCooldowns(combo)); + +describe("explicit server cooldown versus local wait allowance", () => { + test("keeps legacy bounded parsing unless preserving a server lower bound", () => { + expect(parseRetryAfterMs("3600", now)).toBe(600_000); + expect(parseRetryAfterMs("3600", now, { preserveServerDelay: true })).toBe(3_600_000); + }); + + test.each([301, 780, 1260, 3600, 7200])("keeps the full %i-second server delay", seconds => { + coolComboTarget(combo, target, { now, retryAfter: String(seconds) }); + expect(earliestComboCooldownExpiry(combo, [target], now)).toBe(now + seconds * 1000); + expect(isComboTargetInCooldown(combo, target, now + seconds * 1000 - 1)).toBe(true); + expect(isComboTargetInCooldown(combo, target, now + seconds * 1000)).toBe(false); + }); + + test("HTTP-date reset also keeps the full hour", () => { + coolComboTarget(combo, target, { now, retryAfter: new Date(now + 3_600_000).toUTCString() }); + expect(earliestComboCooldownExpiry(combo, [target], now)).toBe(now + 3_600_000); + }); + + test("a configured fallback is still bounded to ten minutes", () => { + coolComboTarget(combo, target, { now, cooldownMs: 99_000_000 }); + expect(earliestComboCooldownExpiry(combo, [target], now)).toBe(now + 600_000); + }); + + test("explicit immediate retry stays immediate", () => { + coolComboTarget(combo, target, { now, retryAfter: "0" }); + expect(earliestComboCooldownExpiry(combo, [target], now)).toBe(now + 1); + }); +}); diff --git a/tests/providers/devin-stated-reset-hardening.test.ts b/tests/providers/devin-stated-reset-hardening.test.ts new file mode 100644 index 0000000000..0dac8ed0e3 --- /dev/null +++ b/tests/providers/devin-stated-reset-hardening.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, test } from "bun:test"; +import { CloudChatError, type CloudChatEvent, type CloudChatRequest } from "../../src/adapters/devin/cloud-direct"; +import { + streamChatEventsWithResetRetry, + statedResetMaxWaitMsForTests, +} from "../../src/adapters/devin/cloud-direct/stated-reset-retry"; + +const request = { + apiKey: "test-only", apiServerUrl: "https://example.invalid", modelUid: "swe-2", messages: [], +} as unknown as CloudChatRequest; + +const cap = (message: string) => new CloudChatError(message, "resource_exhausted", "test", 429); +async function drain(events: AsyncIterable): Promise { + for await (const _event of events) { /* Consume the real wrapper without a live RPC. */ } +} + +describe("Devin cumulative stated-reset allowance", () => { + test("two one-hour refusals cannot turn a one-hour allowance into two hours", async () => { + let calls = 0; + const waits: number[] = []; + const failure = cap("Your limit will reset in 1 hour"); + const stream = async function* (req: CloudChatRequest): AsyncGenerator { + expect(req).toBe(request); + calls += 1; + throw failure; + }; + await expect(drain(streamChatEventsWithResetRetry(request, { + stream, maxWaitMs: 3_600_000, sleep: async ms => { waits.push(ms); }, + }))).rejects.toBe(failure); + expect(calls).toBe(2); + expect(waits).toEqual([3_600_000]); + }); + + test("twenty plus ten minutes fits the default cumulative allowance", async () => { + let calls = 0; + const waits: number[] = []; + const stream = async function* (): AsyncGenerator { + calls += 1; + if (calls === 1) throw cap("reset in 20 minutes"); + if (calls === 2) throw cap("reset in 10 minutes"); + yield { kind: "text", text: "ok" }; + }; + await drain(streamChatEventsWithResetRetry(request, { + stream, maxWaitMs: 1_800_000, sleep: async ms => { waits.push(ms); }, + })); + expect(calls).toBe(3); + expect(waits).toEqual([1_200_000, 600_000]); + }); + + test("a compound duration is never shortened to fit the allowance", async () => { + let calls = 0; + const waits: number[] = []; + const stream = async function* (): AsyncGenerator { + calls += 1; + throw cap("reset in 5 minutes 30 seconds"); + }; + await expect(drain(streamChatEventsWithResetRetry(request, { + stream, maxWaitMs: 300_000, sleep: async ms => { waits.push(ms); }, + }))).rejects.toThrow("5 minutes 30 seconds"); + expect(calls).toBe(1); + expect(waits).toEqual([]); + }); + + test("a pre-aborted request does not enter the stream", async () => { + const controller = new AbortController(); + controller.abort(); + let calls = 0; + await expect(drain(streamChatEventsWithResetRetry({ ...request, signal: controller.signal }, { + stream: async function* (): AsyncGenerator { calls += 1; }, + }))).rejects.toHaveProperty("name", "AbortError"); + expect(calls).toBe(0); + }); + + test("cancellation at the sleep-completion boundary prevents replay", async () => { + const controller = new AbortController(); + let calls = 0; + const stream = async function* (): AsyncGenerator { + calls += 1; + throw cap("reset in 1 second"); + }; + await expect(drain(streamChatEventsWithResetRetry({ ...request, signal: controller.signal }, { + stream, sleep: async () => { controller.abort(); }, + }))).rejects.toHaveProperty("name", "AbortError"); + expect(calls).toBe(1); + }); + + test.each([ + { kind: "text", text: "partial" }, + { kind: "usage", promptTokens: 1 }, + { kind: "tool_call_start", id: "t1", name: "write_file" }, + { kind: "reasoning_signature", signature: "sig" }, + ] as CloudChatEvent[])("never retries after an event: %j", async event => { + let calls = 0; + const stream = async function* (): AsyncGenerator { + calls += 1; + yield event; + throw cap("reset in 1 second"); + }; + await expect(drain(streamChatEventsWithResetRetry(request, { stream }))).rejects.toThrow("reset in 1 second"); + expect(calls).toBe(1); + }); + + test("explicit zero disables waiting; overlarge overrides stay bounded", () => { + const name = "OPENCODEX_DEVIN_STATED_RESET_WAIT_MS"; + const previous = process.env[name]; + try { + process.env[name] = "0"; + expect(statedResetMaxWaitMsForTests()).toBe(0); + process.env[name] = "999999999"; + expect(statedResetMaxWaitMsForTests()).toBe(3_600_000); + } finally { + if (previous === undefined) delete process.env[name]; + else process.env[name] = previous; + } + }); +}); diff --git a/tests/server/retry-delay-hardening.test.ts b/tests/server/retry-delay-hardening.test.ts new file mode 100644 index 0000000000..e0e43d51c5 --- /dev/null +++ b/tests/server/retry-delay-hardening.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, test } from "bun:test"; +import { adapterFailureFromMessage, parseRetryAfterFromMessage } from "../../src/lib/errors"; + +describe("stated reset duration boundaries", () => { + test("the error adapter retains a local parser binding after extraction", () => { + const failure = adapterFailureFromMessage("Devin rate limit: reset in 5 minutes 30 seconds"); + expect(failure.httpStatus).toBe(429); + expect(failure.error.message).toBe("Devin rate limit: reset in 5 minutes 30 seconds Please try again in 330s."); + expect(failure.error.code).toBe("rate_limit_exceeded"); + }); + + test.each([ + ["reset in 5 minutes 30 seconds", 330], + ["reset in 1 hour, 5 minutes and 30 seconds", 3930], + ["reset in 1h30m", 5400], + ["try again in 500ms", 1], + ["retry after 1500 milliseconds", 2], + ["reset in 1 minute 500 milliseconds", 61], + ["retry after 7.2s", 8], + ["Retry-After: 30", 30], + ["Retry-After: 0.1", 1], + ["Your limit RESETS IN 21 MINUTES", 1260], + ["try again in 2s. Your limit will reset in 21 minutes.", 1260], + ] as const)("%s -> %i seconds", (message, expected) => { + expect(parseRetryAfterFromMessage(message)).toBe(expected); + }); + + test.each([ + "reset in 2026", + "reset in -5 minutes", + "reset in 5 minutes -30 seconds", + "reset in 5 minutes 30 bananas", + "reset in 5 minutes 30", + "reset in 5 months", + "retry after 3 monkeys", + "try again in 1e3s", + "Retry-After: 3:30", + "Retry-After: 123abc", + "reset in 0 seconds", + "reset in 999999999999999999999999999999999999999999 hours", + ])("does not salvage a misleading partial duration: %s", message => { + expect(parseRetryAfterFromMessage(message)).toBeUndefined(); + }); +}); From 6d435fa2c2a1428f57e45af7c6c78703621df4ad Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 18 Sep 2026 18:45:39 +0900 Subject: [PATCH 3/3] test(layout): register the new Devin and combo reset test files `tests/test-layout-tooling.test.ts` failed its membership oracle: three of the new test files resolve through neither the explicit map nor a regex seed, so nothing placed them. `combo-authoritative-reset.test.ts` collides with the `combo-` seeds, which point at `routing`, and the `devin-stated-reset-*` pair has no seed at all. Registered in both `scripts/test-layout/layout.json` and `tests/fixtures/test-layout-expected.json`, which the oracle compares against each other as well as against the tree. Also rebased onto current `dev`. Co-authored-by: luvs01 --- scripts/test-layout/layout.json | 5 ++++- tests/fixtures/test-layout-expected.json | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 162b4ba20f..ac18b1ad04 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1527,7 +1527,10 @@ "ws-native-steering.test.ts": "responses", "ws-steering-stability.test.ts": "responses", "ws-steering-completion.test.ts": "responses", - "ws-steering-smoke.test.ts": "responses" + "ws-steering-smoke.test.ts": "responses", + "combo-authoritative-reset.test.ts": "codex-integration", + "devin-stated-reset-hardening.test.ts": "providers", + "devin-stated-reset-retry.test.ts": "providers" }, "migrated": [ "adapters", diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index efdee7fadc..952f8f56df 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1359,5 +1359,8 @@ "ws-native-steering.test.ts": "responses", "ws-steering-stability.test.ts": "responses", "ws-steering-completion.test.ts": "responses", - "ws-steering-smoke.test.ts": "responses" + "ws-steering-smoke.test.ts": "responses", + "combo-authoritative-reset.test.ts": "codex-integration", + "devin-stated-reset-hardening.test.ts": "providers", + "devin-stated-reset-retry.test.ts": "providers" }