test(responses): pin the #4546 incident as one system, not five fixes (#4546)

scripts/test-layout/layout.json

@@ -171,6 +171,7 @@
     "responses-core-modules.test.ts": "responses",
     "responses-spend-ledger-wiring.test.ts": "responses",
     "responses-send-budget-errors.test.ts": "responses",
+    "responses-4546-incident-regression.test.ts": "responses",
     "chat-responses-control-integration.test.ts": "responses",
     "coding-agent-tool-result-images.test.ts": "adapters",
     "hub-usage.test.ts": "server",

src/lib/spend-reservation-ledger.ts

@@ -670,23 +670,22 @@ export function createSpendReservationLedger(options: {
       }
     }
     // A reservation that survived replay has no owner left. The process that made it is gone,
-    // so nothing in this one can ever settle it, and leaving it live holds its tokens against
-    // the scope forever -- a ceiling that only ever tightens, which is the opposite of the
-    // bound this store exists to keep. Deleting the entry is not the alternative: that would
-    // hand the same send id a second reservation.
+    // so nothing in this one can ever settle it, and leaving it live means the send stays
+    // pending forever against a scope that can never resolve it. Deleting the entry is not the
+    // alternative either: that would hand the same send id a second reservation.
     //
-    // The distinction is the one the rest of the module already draws. An UNDISPATCHED
-    // reservation never reached the wire, so it is abandoned and its tokens come back. A
-    // DISPATCHED one may already have been billed, so it becomes unresolved spend. Both are
-    // appended, so the file agrees with memory and the next restart has nothing left to do.
+    // Both live states resolve to UNRESOLVED, including an undispatched one. The tempting
+    // distinction -- open never reached the wire, so give its tokens back -- assumes the
+    // journal is complete up to the crash, and the torn-tail handling above says it is not: a
+    // send can dispatch and die before its dispatch record lands. Abandoning that reservation
+    // returns tokens for a send that may have been billed, and worse, it RESETS a ceiling that
+    // had already fired. An exhausted scope staying exhausted across a restart is the whole
+    // reason this store is on disk.
     const reconciledAt = now();
     for (const [send, reservation] of reservations) {
       if (!isLive(reservation.status)) continue;
-      const abandoned = reservation.status === "open";
-      applyResolve(send, abandoned ? "abandoned" : "lost", 0, reconciledAt);
-      append(abandoned
-        ? { v: 1, kind: "abandon", send, at: reconciledAt }
-        : { v: 1, kind: "lost", send, at: reconciledAt });
+      applyResolve(send, "lost", 0, reconciledAt);
+      append({ v: 1, kind: "lost", send, at: reconciledAt });
     }
   }
 

src/server/responses/core.ts

@@ -57,12 +57,8 @@ export async function handleResponses(
       visionDescribeTerminal: options.visionDescribeTerminal === true
         || req.headers.get("x-opencodex-vision-describe") === "1",
       translatorBudget,
-      // Created once at genuine ingress; a combo child arrives with the parent's holder already
-      // in options and must not start a fresh allowance.
-      // The spend observer is installed with it, for the same reason: a child inherits the
-      // parent's ledger entries instead of opening a second set for the same physical sends.
-      sendBudget: options.sendBudget
-        ?? createRequestExecutionBudget(undefined, undefined, attachRequestSpendTracker(req, logCtx)),
+      // Once at ingress, spend observer included: a combo child inherits the parent's holder.
+      sendBudget: options.sendBudget ?? createRequestExecutionBudget(undefined, undefined, attachRequestSpendTracker(req, logCtx)),
     });
     return ownsBudget ? finalizeOwnedTranslatorBudget(response, translatorBudget) : response;
   } catch (error) {

src/server/responses/request-spend.ts

@@ -44,8 +44,14 @@ export function createRequestSpendTracker(
     "provider" | "accountLogLabel" | "usageLogInputTokens" | "spendOutputCeilingTokens"
   >,
   rootId: string | undefined,
-  ledger: SpendReservationLedger = sharedSpendLedger(),
+  injected?: SpendReservationLedger,
 ): RequestSpendTracker {
+  // Resolved on the first CHARGE, not when the request is built. The shared ledger opens a
+  // journal under the OpenCodex home, and a request that never dispatches -- refused at
+  // admission, answered locally, cancelled before its first send -- has no business creating
+  // one. It also means the home in effect at dispatch is the one that gets written.
+  let ledgerRef: SpendReservationLedger | undefined = injected;
+  const ledger = (): SpendReservationLedger => (ledgerRef ??= sharedSpendLedger());
   // Every send this request still owes the ledger an answer for, oldest first.
   const live: string[] = [];
   let refusals = 0;
@@ -56,17 +62,17 @@ export function createRequestSpendTracker(
    * A booking is only marked dispatched once a LATER send exists, because that later send
    * proves the earlier one left. The newest booking stays open until it is settled, so a
    * reservation the budget hands back -- a rotation that found no alternate, a rebuild
-   * abandoned before the wire -- can still be released for free. The cost of that choice is
-   * bounded and stated: a hard crash between reserving and sending replays as abandoned rather
-   * than unresolved, for at most one send per request.
+   * abandoned before the wire -- can still be released for free while this process is alive.
+   * A crash resolves every surviving reservation as unresolved spend regardless of this mark,
+   * because a journal that lost its tail cannot prove a send never left.
    */
   const confirmOlderSends = (): void => {
-    for (let index = 0; index < live.length - 1; index += 1) ledger.markDispatched(live[index] as string);
+    for (let index = 0; index < live.length - 1; index += 1) ledger().markDispatched(live[index] as string);
   };
   return {
     charge(): boolean {
       const sendId = randomUUID();
-      const decision = ledger.reserve({
+      const decision = ledger().reserve({
         sendId,
         scopes: {
           ...(rootId !== undefined ? { rootId } : {}),
@@ -80,7 +86,12 @@ export function createRequestSpendTracker(
       });
       if (!decision.reserved) {
         refusals += 1;
-        return false;
+        // Only an operator's configured ceiling refuses a dispatch. Every other denial --
+        // capacity, durability, a journal this process could not prove complete -- means the
+        // ledger cannot ACCOUNT for this send, which is not a reason to refuse one. An
+        // unconfigured install keeps the count caps it already had and is not newly refused,
+        // and a degraded ledger must not become an outage.
+        return decision.denial.reason !== "spend-limit-exceeded";
       }
       live.push(sendId);
       confirmOlderSends();
@@ -91,7 +102,7 @@ export function createRequestSpendTracker(
       if (sendId === undefined) return;
       // Undispatched, so this returns the tokens. If the send was already confirmed by a later
       // one, `abandon` refuses and unresolved is the only honest outcome left.
-      if (!ledger.abandon(sendId)) ledger.markLost(sendId);
+      if (!ledger().abandon(sendId)) ledger().markLost(sendId);
     },
     settle(usage: TerminalSpendUsage | undefined): void {
       if (resolved) return;
@@ -100,16 +111,16 @@ export function createRequestSpendTracker(
       if (terminal !== undefined) {
         const reported = typeof usage?.inputTokens === "number" || typeof usage?.outputTokens === "number";
         if (reported) {
-          ledger.settle(terminal, {
+          ledger().settle(terminal, {
             inputTokens: usage?.inputTokens ?? 0,
             outputTokens: usage?.outputTokens ?? 0,
           });
         } else {
           // The response never reported usage. It may still have been billed.
-          ledger.markLost(terminal);
+          ledger().markLost(terminal);
         }
       }
-      for (const sendId of live.splice(0)) ledger.markLost(sendId);
+      for (const sendId of live.splice(0)) ledger().markLost(sendId);
     },
     get refusals(): number { return refusals; },
   };

structure/transports/responses.md

@@ -794,19 +794,21 @@ forget to book. The previous attempt at this wiring shipped the whole reserve/di
 vocabulary with no caller at all (#4707), which is the failure mode this shape rules out.
 
 A booking is confirmed dispatched only once a LATER send exists, because that later send proves
-the earlier one left. The newest booking stays open, so a reservation the budget hands back can
-still be released for free. The stated cost: a hard crash between reserving and sending replays
-as abandoned rather than unresolved, for at most one send per request.
+the earlier one left. The newest booking stays open, so a reservation the budget hands back
+during this process's lifetime can still be released for free.
 
 Settlement follows what the request learned. The terminal usage belongs to the last send that
 left, so that one settles with the real figure; every earlier send failed without reporting usage
 of its own and may still have been billed, so it becomes unresolved spend rather than free. A
 request that reports no usage at all leaves all of them unresolved.
 
-Replay resolves what nobody is left to settle: an undispatched reservation is abandoned and a
-dispatched one becomes unresolved, both journaled so a second restart has nothing to redo.
-Without it a reservation whose process died held its tokens against the scope forever, which is a
-ceiling that only tightens. `tests/responses/responses-spend-ledger-wiring.test.ts` pins the
+Replay resolves what nobody is left to settle, and resolves it as unresolved spend whatever state
+it was in. Giving an undispatched one its tokens back would assume the journal is complete up to
+the crash, and the torn-tail rule says it is not: a send can dispatch and die before its dispatch
+record lands. It would also reset a ceiling that had already fired, and an exhausted scope
+staying exhausted across a restart is the whole reason this store is on disk. Both are journaled,
+so a second restart has nothing to redo.
+`tests/responses/responses-spend-ledger-wiring.test.ts` pins the
 booking, the settlement split, the refund, a ceiling that refuses a dispatch rather than
 describing it afterwards, and the restart.
 

tests/fixtures/test-layout-expected.json

@@ -3,6 +3,7 @@
   "responses-core-modules.test.ts": "responses",
   "responses-spend-ledger-wiring.test.ts": "responses",
   "responses-send-budget-errors.test.ts": "responses",
+  "responses-4546-incident-regression.test.ts": "responses",
   "chat-responses-control-integration.test.ts": "responses",
   "coding-agent-tool-result-images.test.ts": "adapters",
   "hub-usage.test.ts": "server",

tests/lib/transient-budget-scope-source.test.ts

@@ -47,7 +47,10 @@ describe("transient send budget stays request-scoped", () => {
     expect(core.match(/const sendBudget = options\.sendBudget \?\? createRequestExecutionBudget\(\);/g))
       .toHaveLength(1);
     // Genuine ingress mints it; a child arrives with the parent's and must not replace it.
-    expect(core).toContain("sendBudget: options.sendBudget ?? createRequestExecutionBudget(),");
+    expect(core).toContain("sendBudget: options.sendBudget ?? createRequestExecutionBudget(");
+    // ...and the durable spend observer is installed WITH it, for the same reason: a child that
+    // inherited the holder must not open a second set of ledger entries for the same sends.
+    expect(core).toContain("attachRequestSpendTracker(req, logCtx)");
     // The regressed shape: a counter local to one call frame, which a combo child restarts.
     expect(core).not.toContain("let transientSendsUsed = 0;");
     expect(core.match(/const remainingTransientSendBudget = \(budget: number\): number =>/g)).toHaveLength(1);